initial-test-for-submission.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b2956898",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-10-10T06:38:07.750968Z",
     "iopub.status.busy": "2023-10-10T06:38:07.750263Z",
     "iopub.status.idle": "2023-10-10T06:38:08.137580Z",
     "shell.execute_reply": "2023-10-10T06:38:08.135664Z"
    },
    "papermill": {
     "duration": 0.39715,
     "end_time": "2023-10-10T06:38:08.140610",
     "exception": false,
     "start_time": "2023-10-10T06:38:07.743460",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/optiver-trading-at-the-close/public_timeseries_testing_util.py\n",
      "/kaggle/input/optiver-trading-at-the-close/train.csv\n",
      "/kaggle/input/optiver-trading-at-the-close/example_test_files/sample_submission.csv\n",
      "/kaggle/input/optiver-trading-at-the-close/example_test_files/revealed_targets.csv\n",
      "/kaggle/input/optiver-trading-at-the-close/example_test_files/test.csv\n",
      "/kaggle/input/optiver-trading-at-the-close/optiver2023/competition.cpython-310-x86_64-linux-gnu.so\n",
      "/kaggle/input/optiver-trading-at-the-close/optiver2023/__init__.py\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "43db6df5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-10T06:38:08.153539Z",
     "iopub.status.busy": "2023-10-10T06:38:08.153083Z",
     "iopub.status.idle": "2023-10-10T06:38:11.482549Z",
     "shell.execute_reply": "2023-10-10T06:38:11.481418Z"
    },
    "papermill": {
     "duration": 3.338247,
     "end_time": "2023-10-10T06:38:11.485687",
     "exception": false,
     "start_time": "2023-10-10T06:38:08.147440",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from itertools import groupby\n",
    "from sklearn.model_selection import train_test_split\n",
    "import lightgbm as lgb\n",
    "import gc\n",
    "from itertools import combinations\n",
    "import plotly.express as px\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from warnings import simplefilter\n",
    "simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5af6da01",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-10T06:38:11.496141Z",
     "iopub.status.busy": "2023-10-10T06:38:11.495194Z",
     "iopub.status.idle": "2023-10-10T06:38:33.506282Z",
     "shell.execute_reply": "2023-10-10T06:38:33.504540Z"
    },
    "papermill": {
     "duration": 22.019833,
     "end_time": "2023-10-10T06:38:33.509650",
     "exception": false,
     "start_time": "2023-10-10T06:38:11.489817",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('/kaggle/input/optiver-trading-at-the-close/train.csv')\n",
    "revealed_targets = pd.read_csv('/kaggle/input/optiver-trading-at-the-close/example_test_files/revealed_targets.csv')\n",
    "test = pd.read_csv('/kaggle/input/optiver-trading-at-the-close/example_test_files/test.csv')\n",
    "sample_submission = pd.read_csv('/kaggle/input/optiver-trading-at-the-close/example_test_files/sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "984bd322",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-10T06:38:33.521111Z",
     "iopub.status.busy": "2023-10-10T06:38:33.520581Z",
     "iopub.status.idle": "2023-10-10T06:39:01.129366Z",
     "shell.execute_reply": "2023-10-10T06:39:01.128426Z"
    },
    "papermill": {
     "duration": 27.61779,
     "end_time": "2023-10-10T06:39:01.132241",
     "exception": false,
     "start_time": "2023-10-10T06:38:33.514451",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "median_vol = train.groupby('stock_id')['bid_size'].median() + train.groupby('stock_id')['ask_size'].median()\n",
    "train['bid_size_std_30'] = train.groupby(['stock_id'], as_index=False)['bid_size'].rolling(30, min_periods=1).std().sort_index()['bid_size']\n",
    "train['bid_size_std_10'] = train.groupby(['stock_id'], as_index=False)['bid_size'].rolling(10, min_periods=1).std().sort_index()['bid_size']\n",
    "train['bid_size_std_5'] = train.groupby(['stock_id'], as_index=False)['bid_size'].rolling(5, min_periods=1).std().sort_index()['bid_size']\n",
    "train['ask_size_std_30'] = train.groupby(['stock_id'], as_index=False)['ask_size'].rolling(30, min_periods=1).std().sort_index()['ask_size']\n",
    "train['ask_size_std_10'] = train.groupby(['stock_id'], as_index=False)['ask_size'].rolling(10, min_periods=1).std().sort_index()['ask_size']\n",
    "train['ask_size_std_5'] = train.groupby(['stock_id'], as_index=False)['ask_size'].rolling(5, min_periods=1).std().sort_index()['ask_size']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fc62896a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-10T06:39:01.142133Z",
     "iopub.status.busy": "2023-10-10T06:39:01.141738Z",
     "iopub.status.idle": "2023-10-10T06:39:01.154838Z",
     "shell.execute_reply": "2023-10-10T06:39:01.153554Z"
    },
    "papermill": {
     "duration": 0.02072,
     "end_time": "2023-10-10T06:39:01.157005",
     "exception": false,
     "start_time": "2023-10-10T06:39:01.136285",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def feat_eng(df):\n",
    "    \n",
    "    cols = [c for c in df.columns if c not in ['row_id', 'time_id', 'data_id']]\n",
    "    df = df[cols]\n",
    "    df['bid_plus_ask_sizes'] = df['bid_size'] + df['ask_size']\n",
    "    df['median_vol'] = df['stock_id'].map(median_vol.to_dict())\n",
    "    df['high_volume'] = np.where(df['bid_plus_ask_sizes'] > df['median_vol'], 1, 0) \n",
    "    df['high_volume_ratio'] = np.where(df['bid_plus_ask_sizes'] > df['median_vol'], 1, 0)*df['bid_plus_ask_sizes']/df['bid_plus_ask_sizes'].sum()\n",
    "    df['imbalance_ratio'] = df['imbalance_size'] / df['matched_size']\n",
    "    df['high_volume_imbalance_size'] = df['high_volume']*df['imbalance_size']\n",
    "    df['low_volume_matched_size'] = df['high_volume']*df['matched_size']\n",
    "    df['mid_price'] = (df['ask_price'] + df['bid_price'])/2\n",
    "    df['wwap'] = df.eval('(bid_size*bid_price+ask_size*ask_price)/(bid_size+ask_size)')\n",
    "    \n",
    "    df['imb_s1'] = df.eval('(bid_size-ask_size)/(bid_size+ask_size)')\n",
    "    df['imb_s2'] = df.eval('(imbalance_size-matched_size)/(matched_size+imbalance_size)')\n",
    "\n",
    "    df['ask_x_size'] = df.eval('ask_size*ask_price')\n",
    "    df['bid_x_size'] = df.eval('bid_size*bid_price')\n",
    "        \n",
    "    df['ask_minus_bid'] = df['ask_x_size'] - df['bid_x_size'] \n",
    "    \n",
    "    df['mix_diff_price'] = df['ask_minus_bid'].div(df['bid_plus_ask_sizes'])\n",
    "    \n",
    "    df[\"bid_size_over_ask_size\"] = df[\"bid_size\"].div(df[\"ask_size\"])\n",
    "    df[\"bid_price_over_ask_price\"] = df[\"bid_price\"].div(df[\"ask_price\"])\n",
    "    \n",
    "    prices = ['reference_price','far_price', 'near_price', 'ask_price', 'bid_price', 'wap']\n",
    "    \n",
    "    for c in combinations(prices, 2):\n",
    "        \n",
    "        df[f'{c[0]}_minus_{c[1]}'] = (df[f'{c[0]}'] - df[f'{c[1]}']).astype(np.float32)\n",
    "        df[f'{c[0]}_times_{c[1]}'] = (df[f'{c[0]}'] * df[f'{c[1]}']).astype(np.float32)\n",
    "        df[f'{c[0]}_{c[1]}_imb'] = df.eval(f'({c[0]}-{c[1]})/({c[0]}+{c[1]})')\n",
    "\n",
    "    for c in combinations(prices, 3):\n",
    "        \n",
    "        max_ = df[list(c)].max(axis=1)\n",
    "        min_ = df[list(c)].min(axis=1)\n",
    "        mid_ = df[list(c)].sum(axis=1)-min_-max_\n",
    "\n",
    "        df[f'{c[0]}_{c[1]}_{c[2]}_imb2'] = (max_-mid_)/(mid_-min_)\n",
    "    \n",
    "        \n",
    "    gc.collect()\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e360a220",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-10T06:39:01.165780Z",
     "iopub.status.busy": "2023-10-10T06:39:01.165414Z",
     "iopub.status.idle": "2023-10-10T06:40:13.055575Z",
     "shell.execute_reply": "2023-10-10T06:40:13.054012Z"
    },
    "papermill": {
     "duration": 71.901424,
     "end_time": "2023-10-10T06:40:13.062066",
     "exception": false,
     "start_time": "2023-10-10T06:39:01.160642",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 58.9 s, sys: 12.8 s, total: 1min 11s\n",
      "Wall time: 1min 11s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "y = train['target'].values\n",
    "X = feat_eng(train.drop(columns='target'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "436e3a5a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-10T06:40:13.072335Z",
     "iopub.status.busy": "2023-10-10T06:40:13.071859Z",
     "iopub.status.idle": "2023-10-10T06:40:51.615189Z",
     "shell.execute_reply": "2023-10-10T06:40:51.614280Z"
    },
    "papermill": {
     "duration": 38.555123,
     "end_time": "2023-10-10T06:40:51.621215",
     "exception": false,
     "start_time": "2023-10-10T06:40:13.066092",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 49s, sys: 5.72 s, total: 1min 54s\n",
      "Wall time: 38.5 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LGBMRegressor(colsample_bytree=0.95, learning_rate=0.01, max_depth=3,\n",
       "              min_child_weight=31, n_estimators=10, random_state=51,\n",
       "              reg_alpha=0.01, subsample_for_bin=500000)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LGBMRegressor</label><div class=\"sk-toggleable__content\"><pre>LGBMRegressor(colsample_bytree=0.95, learning_rate=0.01, max_depth=3,\n",
       "              min_child_weight=31, n_estimators=10, random_state=51,\n",
       "              reg_alpha=0.01, subsample_for_bin=500000)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LGBMRegressor(colsample_bytree=0.95, learning_rate=0.01, max_depth=3,\n",
       "              min_child_weight=31, n_estimators=10, random_state=51,\n",
       "              reg_alpha=0.01, subsample_for_bin=500000)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "params = {'n_estimators': 10, 'learning_rate': 0.01, 'max_depth': 3, 'colsample_bytree': 0.95, 'min_child_weight': 31,\n",
    "         'subsample_for_bin':500000, 'reg_alpha':0.01}\n",
    "m = lgb.LGBMRegressor(**params, random_state=51)\n",
    "\n",
    "m.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "35220a03",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-10T06:40:51.633346Z",
     "iopub.status.busy": "2023-10-10T06:40:51.630710Z",
     "iopub.status.idle": "2023-10-10T06:40:53.479625Z",
     "shell.execute_reply": "2023-10-10T06:40:53.478590Z"
    },
    "papermill": {
     "duration": 1.85648,
     "end_time": "2023-10-10T06:40:53.481647",
     "exception": false,
     "start_time": "2023-10-10T06:40:51.625167",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns with poor contribution Index(['stock_id', 'near_price_times_wap', 'near_price_minus_wap',\n",
      "       'near_price_bid_price_imb', 'near_price_times_bid_price',\n",
      "       'near_price_minus_bid_price', 'near_price_ask_price_imb',\n",
      "       'near_price_times_ask_price', 'near_price_minus_ask_price',\n",
      "       'far_price_wap_imb', 'near_price_wap_imb', 'far_price_times_wap',\n",
      "       'far_price_bid_price_imb', 'far_price_times_bid_price',\n",
      "       'far_price_minus_bid_price', 'far_price_ask_price_imb',\n",
      "       'far_price_times_ask_price', 'far_price_minus_ask_price',\n",
      "       'far_price_near_price_imb', 'far_price_times_near_price',\n",
      "       'far_price_minus_near_price', 'far_price_minus_wap',\n",
      "       'ask_price_minus_bid_price', 'ask_price_times_bid_price',\n",
      "       'ask_price_bid_price_imb', 'near_price_ask_price_wap_imb2',\n",
      "       'near_price_ask_price_bid_price_imb2', 'far_price_bid_price_wap_imb2',\n",
      "       'far_price_ask_price_wap_imb2', 'far_price_ask_price_bid_price_imb2',\n",
      "       'far_price_near_price_wap_imb2', 'far_price_near_price_bid_price_imb2',\n",
      "       'far_price_near_price_ask_price_imb2',\n",
      "       'reference_price_bid_price_wap_imb2',\n",
      "       'reference_price_ask_price_wap_imb2',\n",
      "       'reference_price_ask_price_bid_price_imb2',\n",
      "       'reference_price_near_price_wap_imb2',\n",
      "       'reference_price_near_price_bid_price_imb2',\n",
      "       'reference_price_near_price_ask_price_imb2',\n",
      "       'reference_price_far_price_wap_imb2',\n",
      "       'reference_price_far_price_bid_price_imb2',\n",
      "       'reference_price_far_price_ask_price_imb2',\n",
      "       'reference_price_far_price_near_price_imb2', 'bid_price_wap_imb',\n",
      "       'bid_price_times_wap', 'ask_price_times_wap', 'reference_price_wap_imb',\n",
      "       'near_price_bid_price_wap_imb2', 'reference_price_times_wap',\n",
      "       'high_volume', 'imbalance_ratio', 'high_volume_ratio',\n",
      "       'reference_price_bid_price_imb', 'imbalance_buy_sell_flag',\n",
      "       'reference_price', 'matched_size', 'far_price', 'bid_plus_ask_sizes',\n",
      "       'high_volume_imbalance_size', 'ask_size_std_5', 'ask_size_std_30',\n",
      "       'bid_size_std_5', 'bid_size_std_10', 'bid_size_std_30', 'wap',\n",
      "       'ask_size', 'ask_price', 'bid_size', 'ask_size_std_10',\n",
      "       'low_volume_matched_size', 'mid_price', 'wwap',\n",
      "       'reference_price_times_bid_price', 'reference_price_minus_bid_price',\n",
      "       'reference_price_ask_price_imb', 'reference_price_times_ask_price',\n",
      "       'reference_price_minus_ask_price', 'reference_price_near_price_imb',\n",
      "       'reference_price_times_near_price', 'reference_price_minus_near_price',\n",
      "       'reference_price_far_price_imb', 'reference_price_times_far_price',\n",
      "       'reference_price_minus_far_price', 'bid_price_over_ask_price',\n",
      "       'date_id', 'seconds_in_bucket', 'ask_minus_bid', 'bid_x_size',\n",
      "       'ask_x_size', 'imb_s2', 'imbalance_size', 'bid_price', 'near_price',\n",
      "       'ask_price_bid_price_wap_imb2', 'imb_s1', 'bid_size_over_ask_size',\n",
      "       'ask_price_wap_imb', 'mix_diff_price', 'bid_price_minus_wap'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "        <script type=\"text/javascript\">\n",
       "        window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
       "        if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
       "        if (typeof require !== 'undefined') {\n",
       "        require.undef(\"plotly\");\n",
       "        requirejs.config({\n",
       "            paths: {\n",
       "                'plotly': ['https://cdn.plot.ly/plotly-2.24.1.min']\n",
       "            }\n",
       "        });\n",
       "        require(['plotly'], function(Plotly) {\n",
       "            window._Plotly = Plotly;\n",
       "        });\n",
       "        }\n",
       "        </script>\n",
       "        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>                            <div id=\"aa14c63a-6c73-48b1-a0c2-1f4d4af00056\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"aa14c63a-6c73-48b1-a0c2-1f4d4af00056\")) {                    Plotly.newPlot(                        \"aa14c63a-6c73-48b1-a0c2-1f4d4af00056\",                        [{\"alignmentgroup\":\"True\",\"hovertemplate\":\"x=%{x}\\u003cbr\\u003ey=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"pattern\":{\"shape\":\"\"}},\"name\":\"\",\"offsetgroup\":\"\",\"orientation\":\"h\",\"showlegend\":false,\"textposition\":\"auto\",\"x\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,4,5,8,8,10,14,17],\"xaxis\":\"x\",\"y\":[\"stock_id\",\"near_price_times_wap\",\"near_price_minus_wap\",\"near_price_bid_price_imb\",\"near_price_times_bid_price\",\"near_price_minus_bid_price\",\"near_price_ask_price_imb\",\"near_price_times_ask_price\",\"near_price_minus_ask_price\",\"far_price_wap_imb\",\"near_price_wap_imb\",\"far_price_times_wap\",\"far_price_bid_price_imb\",\"far_price_times_bid_price\",\"far_price_minus_bid_price\",\"far_price_ask_price_imb\",\"far_price_times_ask_price\",\"far_price_minus_ask_price\",\"far_price_near_price_imb\",\"far_price_times_near_price\",\"far_price_minus_near_price\",\"far_price_minus_wap\",\"ask_price_minus_bid_price\",\"ask_price_times_bid_price\",\"ask_price_bid_price_imb\",\"near_price_ask_price_wap_imb2\",\"near_price_ask_price_bid_price_imb2\",\"far_price_bid_price_wap_imb2\",\"far_price_ask_price_wap_imb2\",\"far_price_ask_price_bid_price_imb2\",\"far_price_near_price_wap_imb2\",\"far_price_near_price_bid_price_imb2\",\"far_price_near_price_ask_price_imb2\",\"reference_price_bid_price_wap_imb2\",\"reference_price_ask_price_wap_imb2\",\"reference_price_ask_price_bid_price_imb2\",\"reference_price_near_price_wap_imb2\",\"reference_price_near_price_bid_price_imb2\",\"reference_price_near_price_ask_price_imb2\",\"reference_price_far_price_wap_imb2\",\"reference_price_far_price_bid_price_imb2\",\"reference_price_far_price_ask_price_imb2\",\"reference_price_far_price_near_price_imb2\",\"bid_price_wap_imb\",\"bid_price_times_wap\",\"ask_price_times_wap\",\"reference_price_wap_imb\",\"near_price_bid_price_wap_imb2\",\"reference_price_times_wap\",\"high_volume\",\"imbalance_ratio\",\"high_volume_ratio\",\"reference_price_bid_price_imb\",\"imbalance_buy_sell_flag\",\"reference_price\",\"matched_size\",\"far_price\",\"bid_plus_ask_sizes\",\"high_volume_imbalance_size\",\"ask_size_std_5\",\"ask_size_std_30\",\"bid_size_std_5\",\"bid_size_std_10\",\"bid_size_std_30\",\"wap\",\"ask_size\",\"ask_price\",\"bid_size\",\"ask_size_std_10\",\"low_volume_matched_size\",\"mid_price\",\"wwap\",\"reference_price_times_bid_price\",\"reference_price_minus_bid_price\",\"reference_price_ask_price_imb\",\"reference_price_times_ask_price\",\"reference_price_minus_ask_price\",\"reference_price_near_price_imb\",\"reference_price_times_near_price\",\"reference_price_minus_near_price\",\"reference_price_far_price_imb\",\"reference_price_times_far_price\",\"reference_price_minus_far_price\",\"bid_price_over_ask_price\",\"date_id\",\"seconds_in_bucket\",\"ask_minus_bid\",\"bid_x_size\",\"ask_x_size\",\"imb_s2\",\"imbalance_size\",\"bid_price\",\"near_price\",\"ask_price_bid_price_wap_imb2\",\"imb_s1\",\"bid_size_over_ask_size\",\"ask_price_wap_imb\",\"mix_diff_price\",\"bid_price_minus_wap\",\"ask_price_minus_wap\",\"reference_price_minus_wap\",\"median_vol\"],\"yaxis\":\"y\",\"type\":\"bar\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"x\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"y\"}},\"legend\":{\"tracegroupgap\":0},\"margin\":{\"t\":60},\"barmode\":\"relative\"},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('aa14c63a-6c73-48b1-a0c2-1f4d4af00056');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "feat_imp = pd.Series(m.feature_importances_, index=X.columns).sort_values()\n",
    "print('Columns with poor contribution', feat_imp[feat_imp<10].index)\n",
    "fig = px.bar(x=feat_imp, y=feat_imp.index, orientation='h')\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cf189baa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-10T06:40:53.493745Z",
     "iopub.status.busy": "2023-10-10T06:40:53.492346Z",
     "iopub.status.idle": "2023-10-10T06:40:54.132186Z",
     "shell.execute_reply": "2023-10-10T06:40:54.130945Z"
    },
    "papermill": {
     "duration": 0.649202,
     "end_time": "2023-10-10T06:40:54.135388",
     "exception": false,
     "start_time": "2023-10-10T06:40:53.486186",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test = feat_eng(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b5b10109",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-10T06:40:54.147346Z",
     "iopub.status.busy": "2023-10-10T06:40:54.146960Z",
     "iopub.status.idle": "2023-10-10T06:40:54.153385Z",
     "shell.execute_reply": "2023-10-10T06:40:54.151944Z"
    },
    "papermill": {
     "duration": 0.01523,
     "end_time": "2023-10-10T06:40:54.155839",
     "exception": false,
     "start_time": "2023-10-10T06:40:54.140609",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def zero_sum(prices, volumes):\n",
    "    \n",
    "#    I got this idea from https://github.com/gotoConversion/goto_conversion/\n",
    "    \n",
    "    std_error = np.sqrt(volumes)\n",
    "    step = np.sum(prices)/np.sum(std_error)\n",
    "    out = prices-std_error*step\n",
    "    \n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "eedbf1b5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-10T06:40:54.168926Z",
     "iopub.status.busy": "2023-10-10T06:40:54.167773Z",
     "iopub.status.idle": "2023-10-10T06:40:54.191466Z",
     "shell.execute_reply": "2023-10-10T06:40:54.190362Z"
    },
    "papermill": {
     "duration": 0.032934,
     "end_time": "2023-10-10T06:40:54.194437",
     "exception": false,
     "start_time": "2023-10-10T06:40:54.161503",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import optiver2023\n",
    "env = optiver2023.make_env()\n",
    "iter_test = env.iter_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d9cc9d2e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-10T06:40:54.206525Z",
     "iopub.status.busy": "2023-10-10T06:40:54.206187Z",
     "iopub.status.idle": "2023-10-10T06:40:54.217950Z",
     "shell.execute_reply": "2023-10-10T06:40:54.216856Z"
    },
    "papermill": {
     "duration": 0.020389,
     "end_time": "2023-10-10T06:40:54.219851",
     "exception": false,
     "start_time": "2023-10-10T06:40:54.199462",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Here is how I define rolling feature, however, it takes too much time to complete the test.\n",
    "# It would be much helpful if anyone could improve the efficiency\n",
    "stock_id = list(range(200))\n",
    "\n",
    "def rolling_window(a, window):\n",
    "    insert = np.full(window-1, a[0])\n",
    "    a = np.insert(a, 0, insert)\n",
    "    shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)\n",
    "    strides = a.strides + (a.strides[-1],)\n",
    "    return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)\n",
    "\n",
    "def ts_stddev(data, window=10):\n",
    "    value = rolling_window(np.array(data), window).std(axis=1)\n",
    "    return value\n",
    "\n",
    "def rolling_eng(test, test_all, counter):\n",
    "    bid_size_std_60 = []\n",
    "    bid_size_std_120 = []\n",
    "    bid_size_std_300 = []\n",
    "    ask_size_std_60 = []\n",
    "    ask_size_std_120 = []\n",
    "    ask_size_std_300 = []\n",
    "    for sid in stock_id:\n",
    "        bid_size = test_all[test_all['stock_id'] == sid]['bid_size']\n",
    "        ask_size = test_all[test_all['stock_id'] == sid]['ask_size']\n",
    "        bid_size_std_60.append(ts_stddev(bid_size, 30)[-1])\n",
    "        bid_size_std_120.append(ts_stddev(bid_size, 10)[-1])\n",
    "        bid_size_std_300.append(ts_stddev(bid_size, 5)[-1])\n",
    "        ask_size_std_60.append(ts_stddev(ask_size, 30)[-1])\n",
    "        ask_size_std_120.append(ts_stddev(ask_size, 10)[-1])\n",
    "        ask_size_std_300.append(ts_stddev(ask_size, 5)[-1])\n",
    "    test['bid_szie_std_30'] = np.array(bid_size_std_60)\n",
    "    test['bid_size_std_10'] = np.array(bid_size_std_120)\n",
    "    test['bid_size_std_50'] = np.array(bid_size_std_300)\n",
    "    test['ask_size_std_30'] = np.array(ask_size_std_60)\n",
    "    test['ask_size_std_10'] = np.array(ask_size_std_120)\n",
    "    test['ask_size_std_5'] = np.array(ask_size_std_300)\n",
    "    return test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "89c5e7a1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-10T06:40:54.231972Z",
     "iopub.status.busy": "2023-10-10T06:40:54.231577Z",
     "iopub.status.idle": "2023-10-10T06:42:44.633718Z",
     "shell.execute_reply": "2023-10-10T06:42:44.632366Z"
    },
    "papermill": {
     "duration": 110.411921,
     "end_time": "2023-10-10T06:42:44.636858",
     "exception": false,
     "start_time": "2023-10-10T06:40:54.224937",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This version of the API is not optimized and should not be used to estimate the runtime of your code on the hidden test set.\n"
     ]
    }
   ],
   "source": [
    "counter = 0\n",
    "for (test, revealed_targets, sample_prediction) in iter_test:\n",
    "    test['counter'] = counter\n",
    "#     print(test['counter'])\n",
    "    if counter == 0:\n",
    "        test_all = test\n",
    "    else:\n",
    "        test_all = pd.concat([test_all, test], axis = 0)\n",
    "    \n",
    "    test_roll = rolling_eng(test, test_all, counter)\n",
    "    feat = feat_eng(test_roll)\n",
    "    sample_prediction['target'] = m.predict(feat.drop(columns = ['counter']))\n",
    "    sample_prediction['target'] = zero_sum(sample_prediction['target'], test.loc[:,'bid_size'] + test.loc[:,'ask_size'])\n",
    "    env.predict(sample_prediction)\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29a0ece1",
   "metadata": {
    "papermill": {
     "duration": 0.004508,
     "end_time": "2023-10-10T06:42:44.646634",
     "exception": false,
     "start_time": "2023-10-10T06:42:44.642126",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 281.511009,
   "end_time": "2023-10-10T06:42:45.777795",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-10-10T06:38:04.266786",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}