main-notebook-no-model.ipynb

{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Optiver Trading At The Close\n\n[Competition](https://www.kaggle.com/competitions/optiver-trading-at-the-close)\n\n[GitHub Repository](https://github.com/cvaisnor/DNN-Final-Project)\n\n[Google Slides Presentation](https://docs.google.com/presentation/d/1Xc5F1_NveFi1il3GqHej2aqVmklR_jZU4kx6ZJhiDEM/edit?usp=sharing)","metadata":{}},{"cell_type":"markdown","source":"# General Imports","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nkaggle_environment = True # True if running on Kaggle, don't forget to add the dataset!\n\nif kaggle_environment:\n    data_path = '/kaggle/input/'\nelse:\n    data_path = 'kaggle/input/'","metadata":{"execution":{"iopub.status.busy":"2023-10-17T21:29:46.442668Z","iopub.execute_input":"2023-10-17T21:29:46.443451Z","iopub.status.idle":"2023-10-17T21:29:46.826370Z","shell.execute_reply.started":"2023-10-17T21:29:46.443417Z","shell.execute_reply":"2023-10-17T21:29:46.825504Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"markdown","source":"# Setting File Structure","metadata":{}},{"cell_type":"code","source":"for dirname, _, filenames in os.walk(data_path):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-10-17T21:29:46.828481Z","iopub.execute_input":"2023-10-17T21:29:46.829068Z","iopub.status.idle":"2023-10-17T21:29:46.840764Z","shell.execute_reply.started":"2023-10-17T21:29:46.829013Z","shell.execute_reply":"2023-10-17T21:29:46.839513Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"/kaggle/input/optiver-trading-at-the-close/public_timeseries_testing_util.py\n/kaggle/input/optiver-trading-at-the-close/train.csv\n/kaggle/input/optiver-trading-at-the-close/example_test_files/sample_submission.csv\n/kaggle/input/optiver-trading-at-the-close/example_test_files/revealed_targets.csv\n/kaggle/input/optiver-trading-at-the-close/example_test_files/test.csv\n/kaggle/input/optiver-trading-at-the-close/optiver2023/competition.cpython-310-x86_64-linux-gnu.so\n/kaggle/input/optiver-trading-at-the-close/optiver2023/__init__.py\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Loading Data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(data_path + 'optiver-trading-at-the-close/train.csv')\nrevealed_targets = pd.read_csv(data_path + 'optiver-trading-at-the-close/example_test_files/revealed_targets.csv')\ntest = pd.read_csv(data_path + 'optiver-trading-at-the-close/example_test_files/test.csv')\nsample_submission = pd.read_csv(data_path + 'optiver-trading-at-the-close/example_test_files/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-10-17T21:29:46.842562Z","iopub.execute_input":"2023-10-17T21:29:46.843600Z","iopub.status.idle":"2023-10-17T21:30:06.373150Z","shell.execute_reply.started":"2023-10-17T21:29:46.843565Z","shell.execute_reply":"2023-10-17T21:30:06.371968Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering Section","metadata":{}},{"cell_type":"code","source":"# imports","metadata":{"execution":{"iopub.status.busy":"2023-10-17T21:30:06.374676Z","iopub.execute_input":"2023-10-17T21:30:06.375802Z","iopub.status.idle":"2023-10-17T21:30:06.380310Z","shell.execute_reply.started":"2023-10-17T21:30:06.375765Z","shell.execute_reply":"2023-10-17T21:30:06.379187Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"markdown","source":"All feature transformations should be combined into a generate_feature() function. This can then be applied to the test data will in the for loop for the Optiver API.","metadata":{}},{"cell_type":"code","source":"def generate_features(test):\n    return test","metadata":{"execution":{"iopub.status.busy":"2023-10-17T21:30:06.382593Z","iopub.execute_input":"2023-10-17T21:30:06.382936Z","iopub.status.idle":"2023-10-17T21:30:06.396015Z","shell.execute_reply.started":"2023-10-17T21:30:06.382907Z","shell.execute_reply":"2023-10-17T21:30:06.394500Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"markdown","source":"# Slicing\n\nGenerally, don't change the cell below. It is used to slice the data into training and validation sets.","metadata":{}},{"cell_type":"code","source":"# imports\nfrom sklearn.model_selection import train_test_split\n\ny = train['target'].values\nX = train.drop(columns='target')\n\n# train test split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T21:30:06.397370Z","iopub.execute_input":"2023-10-17T21:30:06.397849Z","iopub.status.idle":"2023-10-17T21:30:10.587219Z","shell.execute_reply.started":"2023-10-17T21:30:06.397816Z","shell.execute_reply":"2023-10-17T21:30:10.586217Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"markdown","source":"# Model Development","metadata":{}},{"cell_type":"code","source":"# imports\n\nclass Model:\n    def __init__(self) -> None:\n        pass\n\n    def predict(self, X):\n        return np.random.rand(len(X))","metadata":{"execution":{"iopub.status.busy":"2023-10-17T21:30:10.588439Z","iopub.execute_input":"2023-10-17T21:30:10.589632Z","iopub.status.idle":"2023-10-17T21:30:10.594763Z","shell.execute_reply.started":"2023-10-17T21:30:10.589587Z","shell.execute_reply":"2023-10-17T21:30:10.593758Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"model = Model()","metadata":{"execution":{"iopub.status.busy":"2023-10-17T21:30:10.596476Z","iopub.execute_input":"2023-10-17T21:30:10.596968Z","iopub.status.idle":"2023-10-17T21:30:10.611827Z","shell.execute_reply.started":"2023-10-17T21:30:10.596930Z","shell.execute_reply":"2023-10-17T21:30:10.610479Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"markdown","source":"# Generating Predictions\n\nOnly for Kaggle:\nWhen uploading notebook, make sure to add the Optiver data on the right side of the screen.","metadata":{}},{"cell_type":"code","source":"if kaggle_environment:\n    import optiver2023\n    env = optiver2023.make_env()\n    iter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2023-10-17T21:30:10.613395Z","iopub.execute_input":"2023-10-17T21:30:10.613776Z","iopub.status.idle":"2023-10-17T21:30:10.641355Z","shell.execute_reply.started":"2023-10-17T21:30:10.613732Z","shell.execute_reply":"2023-10-17T21:30:10.640398Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"if kaggle_environment:\n    # To count how many time the \"for loop\" runs.\n    counter = 0\n\n    # init 3 empty lists\n    test_ls, revealed_targets_ls, sample_prediction_ls = [], [], []\n\n    for (test, revealed_targets, sample_prediction) in iter_test:\n        # Append the dataframe that API return into the list.\n        test_ls.append(test.copy())\n        revealed_targets_ls.append(revealed_targets.copy())\n        sample_prediction_ls.append(sample_prediction.copy())\n\n        # Generate features\n        test = generate_features(test)\n\n        # Writes our predictions \n        sample_prediction[\"target\"] = model.predict(test)\n        \n        # This line submit our predictions.\n        env.predict(sample_prediction)\n        counter += 1\n\n    print('\\n', '=' * 50, sep=\"\")\n    print(f\"counter: {counter}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-17T21:30:10.642850Z","iopub.execute_input":"2023-10-17T21:30:10.643482Z","iopub.status.idle":"2023-10-17T21:30:11.480530Z","shell.execute_reply.started":"2023-10-17T21:30:10.643452Z","shell.execute_reply":"2023-10-17T21:30:11.479410Z"},"trusted":true},"execution_count":10,"outputs":[{"name":"stdout","text":"This version of the API is not optimized and should not be used to estimate the runtime of your code on the hidden test set.\n\n==================================================\ncounter: 165\n","output_type":"stream"}]}]}