Optiver - Trading at the Close - Inference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d111edbc",
   "metadata": {
    "papermill": {
     "duration": 0.006602,
     "end_time": "2023-12-19T18:46:43.046126",
     "exception": false,
     "start_time": "2023-12-19T18:46:43.039524",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Optiver - Trading at the Close"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b39989aa",
   "metadata": {
    "papermill": {
     "duration": 0.005312,
     "end_time": "2023-12-19T18:46:43.057464",
     "exception": false,
     "start_time": "2023-12-19T18:46:43.052152",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 1. Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a28d36f0",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-12-19T18:46:43.070899Z",
     "iopub.status.busy": "2023-12-19T18:46:43.070365Z",
     "iopub.status.idle": "2023-12-19T18:46:47.128534Z",
     "shell.execute_reply": "2023-12-19T18:46:47.127268Z"
    },
    "papermill": {
     "duration": 4.068696,
     "end_time": "2023-12-19T18:46:47.131689",
     "exception": false,
     "start_time": "2023-12-19T18:46:43.062993",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "from pathlib import Path\n",
    "import yaml\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import lightgbm as lgb\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a53a8910",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-19T18:46:47.145845Z",
     "iopub.status.busy": "2023-12-19T18:46:47.145329Z",
     "iopub.status.idle": "2023-12-19T18:46:47.151602Z",
     "shell.execute_reply": "2023-12-19T18:46:47.150459Z"
    },
    "papermill": {
     "duration": 0.016582,
     "end_time": "2023-12-19T18:46:47.154599",
     "exception": false,
     "start_time": "2023-12-19T18:46:47.138017",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "warnings.filterwarnings('ignore') \n",
    "\n",
    "competition_dataset_directory = Path('/kaggle/input/optiver-trading-at-the-close')\n",
    "external_dataset_directory = Path('/kaggle/input/optiver-trading-at-the-close-dataset')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3a6cc3c8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-19T18:46:47.168634Z",
     "iopub.status.busy": "2023-12-19T18:46:47.167396Z",
     "iopub.status.idle": "2023-12-19T18:47:10.373654Z",
     "shell.execute_reply": "2023-12-19T18:47:10.372224Z"
    },
    "papermill": {
     "duration": 23.216414,
     "end_time": "2023-12-19T18:47:10.376706",
     "exception": false,
     "start_time": "2023-12-19T18:46:47.160292",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stock_id</th>\n",
       "      <th>date_id</th>\n",
       "      <th>seconds_in_bucket</th>\n",
       "      <th>imbalance_size</th>\n",
       "      <th>reference_price</th>\n",
       "      <th>matched_size</th>\n",
       "      <th>far_price</th>\n",
       "      <th>near_price</th>\n",
       "      <th>bid_price</th>\n",
       "      <th>bid_size</th>\n",
       "      <th>...</th>\n",
       "      <th>bid_size_ask_size_ratio</th>\n",
       "      <th>reference_price_far_price_difference</th>\n",
       "      <th>reference_price_near_price_difference</th>\n",
       "      <th>reference_price_bid_price_difference</th>\n",
       "      <th>reference_price_ask_price_difference</th>\n",
       "      <th>reference_price_wap_difference</th>\n",
       "      <th>far_price_near_price_difference</th>\n",
       "      <th>bid_price_ask_price_difference</th>\n",
       "      <th>reference_price_matched_size</th>\n",
       "      <th>reference_price_imbalance_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>480.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.372506e+06</td>\n",
       "      <td>0.999894</td>\n",
       "      <td>18358364.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.999894</td>\n",
       "      <td>18186.189453</td>\n",
       "      <td>...</td>\n",
       "      <td>1.518781</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.000176</td>\n",
       "      <td>-0.000106</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.000176</td>\n",
       "      <td>18356418.00</td>\n",
       "      <td>5.371937e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>480.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000230</td>\n",
       "      <td>1463335.75</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.999966</td>\n",
       "      <td>949.200012</td>\n",
       "      <td>...</td>\n",
       "      <td>0.047584</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000264</td>\n",
       "      <td>-0.000474</td>\n",
       "      <td>0.000230</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.000738</td>\n",
       "      <td>1463672.25</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>480.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.668167e+06</td>\n",
       "      <td>1.000514</td>\n",
       "      <td>3178828.25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>194.020004</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003066</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000515</td>\n",
       "      <td>0.000052</td>\n",
       "      <td>0.000514</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.000463</td>\n",
       "      <td>3180462.25</td>\n",
       "      <td>2.669538e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.0</td>\n",
       "      <td>480.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.009214e+07</td>\n",
       "      <td>0.999887</td>\n",
       "      <td>26401518.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.999839</td>\n",
       "      <td>41308.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.999516</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000048</td>\n",
       "      <td>-0.000194</td>\n",
       "      <td>-0.000113</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.000242</td>\n",
       "      <td>26398534.00</td>\n",
       "      <td>1.009100e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>480.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.413386e+06</td>\n",
       "      <td>1.000016</td>\n",
       "      <td>14120504.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.999358</td>\n",
       "      <td>18225.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>9.084429</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000658</td>\n",
       "      <td>-0.000055</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.000713</td>\n",
       "      <td>14120730.00</td>\n",
       "      <td>5.413472e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10995</th>\n",
       "      <td>195.0</td>\n",
       "      <td>480.0</td>\n",
       "      <td>540.0</td>\n",
       "      <td>2.440723e+06</td>\n",
       "      <td>1.000317</td>\n",
       "      <td>28280362.00</td>\n",
       "      <td>0.999734</td>\n",
       "      <td>0.999734</td>\n",
       "      <td>1.000317</td>\n",
       "      <td>32257.039062</td>\n",
       "      <td>...</td>\n",
       "      <td>0.100847</td>\n",
       "      <td>0.000583</td>\n",
       "      <td>0.000583</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.000117</td>\n",
       "      <td>-0.000011</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.000117</td>\n",
       "      <td>28289326.00</td>\n",
       "      <td>2.441497e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10996</th>\n",
       "      <td>196.0</td>\n",
       "      <td>480.0</td>\n",
       "      <td>540.0</td>\n",
       "      <td>3.495105e+05</td>\n",
       "      <td>1.000643</td>\n",
       "      <td>9187699.00</td>\n",
       "      <td>1.000129</td>\n",
       "      <td>1.000386</td>\n",
       "      <td>1.000643</td>\n",
       "      <td>205108.406250</td>\n",
       "      <td>...</td>\n",
       "      <td>2.196184</td>\n",
       "      <td>0.000514</td>\n",
       "      <td>0.000257</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.000257</td>\n",
       "      <td>-0.000176</td>\n",
       "      <td>-0.000257</td>\n",
       "      <td>-0.000257</td>\n",
       "      <td>9193607.00</td>\n",
       "      <td>3.497352e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10997</th>\n",
       "      <td>197.0</td>\n",
       "      <td>480.0</td>\n",
       "      <td>540.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.995789</td>\n",
       "      <td>12725436.00</td>\n",
       "      <td>0.995789</td>\n",
       "      <td>0.995789</td>\n",
       "      <td>0.995789</td>\n",
       "      <td>16790.660156</td>\n",
       "      <td>...</td>\n",
       "      <td>0.093262</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.000094</td>\n",
       "      <td>-0.000008</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.000094</td>\n",
       "      <td>12671849.00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10998</th>\n",
       "      <td>198.0</td>\n",
       "      <td>480.0</td>\n",
       "      <td>540.0</td>\n",
       "      <td>1.000899e+06</td>\n",
       "      <td>0.999210</td>\n",
       "      <td>94773272.00</td>\n",
       "      <td>0.999210</td>\n",
       "      <td>0.999210</td>\n",
       "      <td>0.998970</td>\n",
       "      <td>125631.718750</td>\n",
       "      <td>...</td>\n",
       "      <td>0.187540</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000240</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000202</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.000240</td>\n",
       "      <td>94698400.00</td>\n",
       "      <td>1.000108e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10999</th>\n",
       "      <td>199.0</td>\n",
       "      <td>480.0</td>\n",
       "      <td>540.0</td>\n",
       "      <td>1.884286e+06</td>\n",
       "      <td>1.002129</td>\n",
       "      <td>24073678.00</td>\n",
       "      <td>1.000859</td>\n",
       "      <td>1.001494</td>\n",
       "      <td>1.002129</td>\n",
       "      <td>250081.437500</td>\n",
       "      <td>...</td>\n",
       "      <td>0.833139</td>\n",
       "      <td>0.001270</td>\n",
       "      <td>0.000635</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.000318</td>\n",
       "      <td>-0.000145</td>\n",
       "      <td>-0.000635</td>\n",
       "      <td>-0.000318</td>\n",
       "      <td>24124930.00</td>\n",
       "      <td>1.888297e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11000 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       stock_id  date_id  seconds_in_bucket  imbalance_size  reference_price  \\\n",
       "0           0.0    480.0                0.0    5.372506e+06         0.999894   \n",
       "1           1.0    480.0                0.0    0.000000e+00         1.000230   \n",
       "2           2.0    480.0                0.0    2.668167e+06         1.000514   \n",
       "3           3.0    480.0                0.0    1.009214e+07         0.999887   \n",
       "4           4.0    480.0                0.0    5.413386e+06         1.000016   \n",
       "...         ...      ...                ...             ...              ...   \n",
       "10995     195.0    480.0              540.0    2.440723e+06         1.000317   \n",
       "10996     196.0    480.0              540.0    3.495105e+05         1.000643   \n",
       "10997     197.0    480.0              540.0    0.000000e+00         0.995789   \n",
       "10998     198.0    480.0              540.0    1.000899e+06         0.999210   \n",
       "10999     199.0    480.0              540.0    1.884286e+06         1.002129   \n",
       "\n",
       "       matched_size  far_price  near_price  bid_price       bid_size  ...  \\\n",
       "0       18358364.00        NaN         NaN   0.999894   18186.189453  ...   \n",
       "1        1463335.75        NaN         NaN   0.999966     949.200012  ...   \n",
       "2        3178828.25        NaN         NaN   0.999999     194.020004  ...   \n",
       "3       26401518.00        NaN         NaN   0.999839   41308.000000  ...   \n",
       "4       14120504.00        NaN         NaN   0.999358   18225.000000  ...   \n",
       "...             ...        ...         ...        ...            ...  ...   \n",
       "10995   28280362.00   0.999734    0.999734   1.000317   32257.039062  ...   \n",
       "10996    9187699.00   1.000129    1.000386   1.000643  205108.406250  ...   \n",
       "10997   12725436.00   0.995789    0.995789   0.995789   16790.660156  ...   \n",
       "10998   94773272.00   0.999210    0.999210   0.998970  125631.718750  ...   \n",
       "10999   24073678.00   1.000859    1.001494   1.002129  250081.437500  ...   \n",
       "\n",
       "       bid_size_ask_size_ratio  reference_price_far_price_difference  \\\n",
       "0                     1.518781                                   NaN   \n",
       "1                     0.047584                                   NaN   \n",
       "2                     0.003066                                   NaN   \n",
       "3                     1.999516                                   NaN   \n",
       "4                     9.084429                                   NaN   \n",
       "...                        ...                                   ...   \n",
       "10995                 0.100847                              0.000583   \n",
       "10996                 2.196184                              0.000514   \n",
       "10997                 0.093262                              0.000000   \n",
       "10998                 0.187540                              0.000000   \n",
       "10999                 0.833139                              0.001270   \n",
       "\n",
       "       reference_price_near_price_difference  \\\n",
       "0                                        NaN   \n",
       "1                                        NaN   \n",
       "2                                        NaN   \n",
       "3                                        NaN   \n",
       "4                                        NaN   \n",
       "...                                      ...   \n",
       "10995                               0.000583   \n",
       "10996                               0.000257   \n",
       "10997                               0.000000   \n",
       "10998                               0.000000   \n",
       "10999                               0.000635   \n",
       "\n",
       "       reference_price_bid_price_difference  \\\n",
       "0                                  0.000000   \n",
       "1                                  0.000264   \n",
       "2                                  0.000515   \n",
       "3                                  0.000048   \n",
       "4                                  0.000658   \n",
       "...                                     ...   \n",
       "10995                              0.000000   \n",
       "10996                              0.000000   \n",
       "10997                              0.000000   \n",
       "10998                              0.000240   \n",
       "10999                              0.000000   \n",
       "\n",
       "       reference_price_ask_price_difference  reference_price_wap_difference  \\\n",
       "0                                 -0.000176                       -0.000106   \n",
       "1                                 -0.000474                        0.000230   \n",
       "2                                  0.000052                        0.000514   \n",
       "3                                 -0.000194                       -0.000113   \n",
       "4                                 -0.000055                        0.000016   \n",
       "...                                     ...                             ...   \n",
       "10995                             -0.000117                       -0.000011   \n",
       "10996                             -0.000257                       -0.000176   \n",
       "10997                             -0.000094                       -0.000008   \n",
       "10998                              0.000000                        0.000202   \n",
       "10999                             -0.000318                       -0.000145   \n",
       "\n",
       "       far_price_near_price_difference  bid_price_ask_price_difference  \\\n",
       "0                                  NaN                       -0.000176   \n",
       "1                                  NaN                       -0.000738   \n",
       "2                                  NaN                       -0.000463   \n",
       "3                                  NaN                       -0.000242   \n",
       "4                                  NaN                       -0.000713   \n",
       "...                                ...                             ...   \n",
       "10995                         0.000000                       -0.000117   \n",
       "10996                        -0.000257                       -0.000257   \n",
       "10997                         0.000000                       -0.000094   \n",
       "10998                         0.000000                       -0.000240   \n",
       "10999                        -0.000635                       -0.000318   \n",
       "\n",
       "       reference_price_matched_size  reference_price_imbalance_size  \n",
       "0                       18356418.00                    5.371937e+06  \n",
       "1                        1463672.25                    0.000000e+00  \n",
       "2                        3180462.25                    2.669538e+06  \n",
       "3                       26398534.00                    1.009100e+07  \n",
       "4                       14120730.00                    5.413472e+06  \n",
       "...                             ...                             ...  \n",
       "10995                   28289326.00                    2.441497e+06  \n",
       "10996                    9193607.00                    3.497352e+05  \n",
       "10997                   12671849.00                    0.000000e+00  \n",
       "10998                   94698400.00                    1.000108e+06  \n",
       "10999                   24124930.00                    1.888297e+06  \n",
       "\n",
       "[11000 rows x 30 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.read_csv(competition_dataset_directory / 'train.csv').drop(columns=['time_id', 'row_id']).astype(np.float32)\n",
    "df_train = df_train.loc[df_train['date_id'] == 480].reset_index(drop=True)\n",
    "\n",
    "# Drop imbalance_buy_sell_flag because it's only the sign of imbalance_size\n",
    "df_train['imbalance_buy_sell_size'] = df_train.eval('imbalance_size * imbalance_buy_sell_flag').astype(np.float32)\n",
    "df_train.drop(columns=['imbalance_buy_sell_flag'], inplace=True)\n",
    "\n",
    "# Column-wise size features\n",
    "df_train['imbalance_size_matched_size_ratio'] = df_train.eval('imbalance_size / matched_size').astype(np.float32)\n",
    "df_train['imbalance_size_matched_size_difference'] = df_train.eval('imbalance_size - matched_size').astype(np.float32)\n",
    "df_train['imbalance_size_matched_size_sum'] = df_train.eval('imbalance_size + matched_size').astype(np.float32)\n",
    "df_train['imbalance_size_bid_size_ratio'] = df_train.eval('imbalance_size / bid_size').astype(np.float32)\n",
    "df_train['matched_size_bid_size_ratio'] = df_train.eval('matched_size / bid_size').astype(np.float32)\n",
    "df_train['bid_size_ask_size_ratio'] = df_train.eval('bid_size / ask_size').astype(np.float32)\n",
    "\n",
    "# Column-wise price features\n",
    "df_train['reference_price_far_price_difference'] = df_train.eval('reference_price - far_price').astype(np.float32)\n",
    "df_train['reference_price_near_price_difference'] = df_train.eval('reference_price - near_price').astype(np.float32)\n",
    "df_train['reference_price_bid_price_difference'] = df_train.eval('reference_price - bid_price').astype(np.float32)\n",
    "df_train['reference_price_ask_price_difference'] = df_train.eval('reference_price - ask_price').astype(np.float32)\n",
    "df_train['reference_price_wap_difference'] = df_train.eval('reference_price - wap').astype(np.float32)\n",
    "df_train['far_price_near_price_difference'] = df_train.eval('far_price - near_price').astype(np.float32)\n",
    "df_train['bid_price_ask_price_difference'] = df_train.eval('bid_price - ask_price').astype(np.float32)\n",
    "\n",
    "# Column-wise price and size features\n",
    "df_train['reference_price_matched_size'] = df_train.eval('reference_price * matched_size')\n",
    "df_train['reference_price_imbalance_size'] = df_train.eval('reference_price * imbalance_size')\n",
    "\n",
    "df_train"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac458c55",
   "metadata": {
    "papermill": {
     "duration": 0.006014,
     "end_time": "2023-12-19T18:47:10.389261",
     "exception": false,
     "start_time": "2023-12-19T18:47:10.383247",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 2. Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "72011b5a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-19T18:47:10.404399Z",
     "iopub.status.busy": "2023-12-19T18:47:10.403946Z",
     "iopub.status.idle": "2023-12-19T18:47:10.413294Z",
     "shell.execute_reply": "2023-12-19T18:47:10.412037Z"
    },
    "papermill": {
     "duration": 0.019786,
     "end_time": "2023-12-19T18:47:10.415727",
     "exception": false,
     "start_time": "2023-12-19T18:47:10.395941",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def load_model(model_directory, model_file_names, model_type):\n",
    "    \n",
    "    \"\"\"\n",
    "    Load models from the given model directory\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    model_directory: str\n",
    "        Path of the model directory\n",
    "\n",
    "    model_file_names: str\n",
    "        Name of the model files\n",
    "\n",
    "    model_type: str (lightgbm or xgboost)\n",
    "        Model type\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    models: dict\n",
    "        Dictionary of models\n",
    "        \n",
    "    config: dict\n",
    "        Dictionary of configurations\n",
    "    \"\"\"\n",
    "\n",
    "    config = yaml.load(open(model_directory / 'config.yaml', 'r'), Loader=yaml.FullLoader)\n",
    "        \n",
    "    models = {}\n",
    "\n",
    "    for model_file_name in model_file_names:\n",
    "        if model_type == 'lightgbm':\n",
    "            model = lgb.Booster(model_file=model_directory / model_file_name)\n",
    "        elif model_type == 'xgboost':\n",
    "            model = xgb.Booster()\n",
    "            model.load_model(model_directory / model_file_name)\n",
    "        models[model_file_name] = model\n",
    "        print(f'{model_type} model is loaded from {model_directory / model_file_name}')\n",
    "\n",
    "    return models, config\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8332dee9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-19T18:47:10.430699Z",
     "iopub.status.busy": "2023-12-19T18:47:10.430276Z",
     "iopub.status.idle": "2023-12-19T18:47:16.559877Z",
     "shell.execute_reply": "2023-12-19T18:47:16.558341Z"
    },
    "papermill": {
     "duration": 6.140615,
     "end_time": "2023-12-19T18:47:16.562874",
     "exception": false,
     "start_time": "2023-12-19T18:47:10.422259",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "xgboost model is loaded from /kaggle/input/optiver-trading-at-the-close-dataset/xgboost_regression_v9/model_seed42.json\n"
     ]
    }
   ],
   "source": [
    "models, config = load_model(\n",
    "    model_directory=external_dataset_directory / 'xgboost_regression_v9',\n",
    "    model_file_names=[\n",
    "        'model_seed42.json'\n",
    "    ],\n",
    "    model_type='xgboost'\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60404fa4",
   "metadata": {
    "papermill": {
     "duration": 0.007189,
     "end_time": "2023-12-19T18:47:16.577125",
     "exception": false,
     "start_time": "2023-12-19T18:47:16.569936",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 3. Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a6836bfd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-19T18:47:16.592118Z",
     "iopub.status.busy": "2023-12-19T18:47:16.591692Z",
     "iopub.status.idle": "2023-12-19T18:47:16.601413Z",
     "shell.execute_reply": "2023-12-19T18:47:16.600190Z"
    },
    "papermill": {
     "duration": 0.020373,
     "end_time": "2023-12-19T18:47:16.604117",
     "exception": false,
     "start_time": "2023-12-19T18:47:16.583744",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Read precomputed stock weights\n",
    "with open(external_dataset_directory / 'stock_weights.json', mode='r') as f:\n",
    "    stock_weights = json.load(f)\n",
    "stock_weights = {int(stock_id): weight for stock_id, weight in stock_weights.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b2a4079b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-19T18:47:16.619451Z",
     "iopub.status.busy": "2023-12-19T18:47:16.619045Z",
     "iopub.status.idle": "2023-12-19T18:47:16.634433Z",
     "shell.execute_reply": "2023-12-19T18:47:16.633185Z"
    },
    "papermill": {
     "duration": 0.026284,
     "end_time": "2023-12-19T18:47:16.636971",
     "exception": false,
     "start_time": "2023-12-19T18:47:16.610687",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "columns_to_cast = [\n",
    "    'stock_id', 'date_id', 'seconds_in_bucket',\n",
    "    'imbalance_size', 'imbalance_buy_sell_flag', 'matched_size',\n",
    "    'reference_price', 'far_price', 'near_price',\n",
    "    'bid_price', 'bid_size', 'ask_price', 'ask_size', 'wap',\n",
    "]\n",
    "\n",
    "# Raw and column-wise features to concatenate on each iteration\n",
    "test_columns = [\n",
    "    # Raw features\n",
    "    'stock_id', 'date_id', 'seconds_in_bucket',\n",
    "    'imbalance_buy_sell_size', 'matched_size',\n",
    "    'reference_price', 'far_price', 'near_price',\n",
    "    'bid_price', 'ask_price', 'bid_size', 'ask_size', 'wap',\n",
    "\n",
    "    # Other features\n",
    "    'stock_weight',\n",
    "\n",
    "    # Column-wise size features\n",
    "    'imbalance_size_matched_size_ratio', 'imbalance_size_matched_size_sum', 'imbalance_size_matched_size_difference',\n",
    "    'imbalance_size_bid_size_ratio',\n",
    "    'matched_size_bid_size_ratio',\n",
    "    'bid_size_ask_size_ratio',\n",
    "    \n",
    "    # Column-wise price features\n",
    "    'reference_price_far_price_difference', 'reference_price_near_price_difference',\n",
    "    'reference_price_bid_price_difference', 'reference_price_ask_price_difference', 'reference_price_wap_difference',\n",
    "    'far_price_near_price_difference',\n",
    "    'bid_price_ask_price_difference',\n",
    "    \n",
    "    # Column-wise price and size features\n",
    "    'reference_price_matched_size', 'reference_price_imbalance_size'\n",
    "]\n",
    "\n",
    "# Row-wise differences\n",
    "difference_periods = [1, 2, 3, 4, 5, 10, 15]\n",
    "difference_columns = [\n",
    "    'imbalance_buy_sell_size', 'matched_size',\n",
    "    'reference_price', 'far_price', 'near_price',\n",
    "    'bid_price', 'bid_size', 'ask_price', 'ask_size', 'wap',\n",
    "\n",
    "    'imbalance_size_matched_size_sum',\n",
    "\n",
    "    'reference_price_far_price_difference',\n",
    "    'reference_price_near_price_difference',\n",
    "    'reference_price_bid_price_difference',\n",
    "    'reference_price_ask_price_difference',\n",
    "    'reference_price_wap_difference',\n",
    "    'bid_price_ask_price_difference'\n",
    "]\n",
    "\n",
    "# Row-wise ratios\n",
    "ratio_periods = [1, 2, 3, 4, 5, 10, 15]\n",
    "ratio_columns = [\n",
    "    'matched_size', 'imbalance_buy_sell_size',\n",
    "    'bid_size', 'ask_size'\n",
    "]\n",
    "\n",
    "# Rolling statistics\n",
    "rolling_windows = [3]\n",
    "rolling_columns = [\n",
    "    'imbalance_buy_sell_size', 'matched_size',\n",
    "    'reference_price', 'far_price', 'near_price',\n",
    "    'bid_price', 'bid_size', 'ask_price', 'ask_size', 'wap',\n",
    "\n",
    "    'imbalance_size_matched_size_sum',\n",
    "\n",
    "    'reference_price_far_price_difference',\n",
    "    'reference_price_near_price_difference',\n",
    "    'reference_price_bid_price_difference',\n",
    "    'reference_price_ask_price_difference',\n",
    "    'reference_price_wap_difference',\n",
    "    'bid_price_ask_price_difference'\n",
    "]\n",
    "\n",
    "# Current day open differences\n",
    "current_day_open_columns = [\n",
    "    'imbalance_buy_sell_size', 'matched_size',\n",
    "    'reference_price',\n",
    "    'bid_price', 'bid_size', 'ask_price', 'ask_size', 'wap',\n",
    "\n",
    "    'imbalance_size_matched_size_sum',\n",
    "\n",
    "    'reference_price_bid_price_difference',\n",
    "    'reference_price_ask_price_difference',\n",
    "    'reference_price_wap_difference',\n",
    "    'bid_price_ask_price_difference'\n",
    "]\n",
    "\n",
    "# Current day 30th differences\n",
    "current_day_30th_columns = [\n",
    "    'imbalance_buy_sell_size', 'matched_size'\n",
    "]\n",
    "\n",
    "# Previous day close difference\n",
    "previous_day_close_columns = [\n",
    "    'imbalance_buy_sell_size', 'matched_size',\n",
    "    'reference_price', 'far_price', 'near_price',\n",
    "    'bid_price', 'bid_size', 'ask_price', 'ask_size', 'wap',\n",
    "\n",
    "    'imbalance_size_matched_size_sum',\n",
    "]\n",
    "\n",
    "# Previous day high difference\n",
    "previous_day_high_columns = [\n",
    "    'imbalance_buy_sell_size', 'matched_size',\n",
    "    'reference_price', 'far_price', 'near_price',\n",
    "    'bid_price', 'bid_size', 'ask_price', 'ask_size', 'wap',\n",
    "]\n",
    "\n",
    "# Previous day shifts\n",
    "daily_shift_columns = [\n",
    "    'imbalance_buy_sell_size', 'matched_size', 'target'\n",
    "]\n",
    "\n",
    "# Stock ranks inside buckets\n",
    "stock_pct_rank_columns = [\n",
    "    'imbalance_buy_sell_size', 'matched_size',\n",
    "    'reference_price', 'far_price', 'near_price',\n",
    "    'bid_price', 'bid_size', 'ask_price', 'ask_size', 'wap'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b06060a7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-19T18:47:16.651334Z",
     "iopub.status.busy": "2023-12-19T18:47:16.650954Z",
     "iopub.status.idle": "2023-12-19T18:47:16.680120Z",
     "shell.execute_reply": "2023-12-19T18:47:16.678843Z"
    },
    "papermill": {
     "duration": 0.03953,
     "end_time": "2023-12-19T18:47:16.682896",
     "exception": false,
     "start_time": "2023-12-19T18:47:16.643366",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import optiver2023\n",
    "\n",
    "env = optiver2023.make_env()\n",
    "iter_test = env.iter_test()\n",
    "\n",
    "verbose = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1f2847c3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-19T18:47:16.698541Z",
     "iopub.status.busy": "2023-12-19T18:47:16.697928Z",
     "iopub.status.idle": "2023-12-19T18:50:04.018168Z",
     "shell.execute_reply": "2023-12-19T18:50:04.016768Z"
    },
    "papermill": {
     "duration": 167.332093,
     "end_time": "2023-12-19T18:50:04.021605",
     "exception": false,
     "start_time": "2023-12-19T18:47:16.689512",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This version of the API is not optimized and should not be used to estimate the runtime of your code on the hidden test set.\n"
     ]
    }
   ],
   "source": [
    "df_test_current_day = None\n",
    "df_test_previous_day = None\n",
    "df_test_current_day_open = None\n",
    "df_test_previous_day_close = None\n",
    "df_test_previous_day_high = None\n",
    "\n",
    "#df_predictions = []\n",
    "\n",
    "for batch_idx, (df_test_batch, df_revealed_targets_batch, df_submission_batch) in enumerate(iter_test):\n",
    "    \n",
    "    df_test_batch[columns_to_cast] = df_test_batch[columns_to_cast].astype(np.float32)\n",
    "        \n",
    "    # Merge precomputed stock weights\n",
    "    df_test_batch['stock_weight'] = df_test_batch['stock_id'].map(stock_weights).astype(np.float32)\n",
    "    \n",
    "    # Drop imbalance_buy_sell_flag because it's only the sign of imbalance_size\n",
    "    df_test_batch['imbalance_buy_sell_size'] = df_test_batch.eval('imbalance_size * imbalance_buy_sell_flag')\n",
    "    df_test_batch.drop(columns=['imbalance_buy_sell_flag'], inplace=True)\n",
    "\n",
    "    # Column-wise size features\n",
    "    df_test_batch['imbalance_size_matched_size_ratio'] = df_test_batch.eval('imbalance_size / matched_size')\n",
    "    df_test_batch['imbalance_size_matched_size_difference'] = df_test_batch.eval('imbalance_size - matched_size')\n",
    "    df_test_batch['imbalance_size_matched_size_sum'] = df_test_batch.eval('imbalance_size + matched_size')\n",
    "    df_test_batch['imbalance_size_bid_size_ratio'] = df_test_batch.eval('imbalance_size / bid_size')\n",
    "    df_test_batch['matched_size_bid_size_ratio'] = df_test_batch.eval('matched_size / bid_size')\n",
    "    df_test_batch['bid_size_ask_size_ratio'] = df_test_batch.eval('bid_size / ask_size')\n",
    "    \n",
    "    # Column-wise price features\n",
    "    df_test_batch['reference_price_far_price_difference'] = df_test_batch.eval('reference_price - far_price')\n",
    "    df_test_batch['reference_price_near_price_difference'] = df_test_batch.eval('reference_price - near_price')\n",
    "    df_test_batch['reference_price_bid_price_difference'] = df_test_batch.eval('reference_price - bid_price')\n",
    "    df_test_batch['reference_price_ask_price_difference'] = df_test_batch.eval('reference_price - ask_price')\n",
    "    df_test_batch['reference_price_wap_difference'] = df_test_batch.eval('reference_price - wap')\n",
    "    df_test_batch['far_price_near_price_difference'] = df_test_batch.eval('far_price - near_price')\n",
    "    df_test_batch['bid_price_ask_price_difference'] = df_test_batch.eval('bid_price - ask_price')\n",
    "    \n",
    "    # Column-wise price and size features\n",
    "    df_test_batch['reference_price_matched_size'] = df_test_batch.eval('reference_price * matched_size')\n",
    "    df_test_batch['reference_price_imbalance_size'] = df_test_batch.eval('reference_price * imbalance_size')\n",
    "    \n",
    "    current_seconds_in_bucket = df_test_batch['seconds_in_bucket'].values[0]\n",
    "    if current_seconds_in_bucket == 0:\n",
    "        \n",
    "        if df_test_current_day is not None:\n",
    "            # Revealed targets are merged to current day dataframe before new current day dataframe is created\n",
    "            df_revealed_targets_batch = df_revealed_targets_batch.loc[:, [\n",
    "                'stock_id', 'seconds_in_bucket', 'revealed_target'\n",
    "            ]].rename(columns={'revealed_target': 'target'}).astype(np.float32)\n",
    "            df_test_current_day = df_test_current_day.merge(df_revealed_targets_batch, on=['stock_id', 'seconds_in_bucket'], how='left')\n",
    "            # Current day dataframe becomes the previous day dataframe\n",
    "            df_test_previous_day = df_test_current_day.copy(deep=True)\n",
    "            if verbose:\n",
    "                print(f'Iteration {batch_idx} previous date is created from single date')\n",
    "        else:\n",
    "            # Training set date_id 480 becomes the previous day dataframe on the first iteration\n",
    "            df_test_previous_day = df_train.copy(deep=True)\n",
    "            if verbose:\n",
    "                print(f'Iteration {batch_idx} training set assigned to previous day')\n",
    "            \n",
    "        # Create the current day dataframe from the first batch\n",
    "        df_test_current_day = df_test_batch.loc[:, test_columns].copy(deep=True)\n",
    "        \n",
    "        # Create the static dataframes from the previous day dataframe\n",
    "        df_test_current_day_open = df_test_batch.loc[:, ['stock_id'] + current_day_open_columns].copy(deep=True).rename(columns={\n",
    "            column: f'{column}_current_day_open' for column in current_day_open_columns\n",
    "        })\n",
    "        df_test_previous_day_close = df_test_previous_day.groupby(['stock_id'])[previous_day_close_columns].last().rename(columns={\n",
    "            column: f'{column}_previous_day_close' for column in previous_day_close_columns\n",
    "        }).reset_index()\n",
    "        df_test_previous_day_high = df_test_previous_day.groupby(['stock_id'])[previous_day_high_columns].max().rename(columns={\n",
    "            column: f'{column}_previous_day_high' for column in previous_day_high_columns\n",
    "        }).reset_index()\n",
    "        \n",
    "        # Create 30th dataframe from the current as a placeholder with NaNs\n",
    "        df_test_current_day_30th = df_test_batch.loc[:, ['stock_id'] + current_day_30th_columns].copy(deep=True).rename(columns={\n",
    "            column: f'{column}_current_day_30th' for column in current_day_30th_columns\n",
    "        }).reset_index()\n",
    "        df_test_current_day_30th.loc[:, [f'{column}_current_day_30th' for column in current_day_30th_columns]] = np.nan\n",
    "        if verbose:\n",
    "            print(f'Iteration {batch_idx} current day 30th is created with nans')\n",
    "\n",
    "        if verbose:\n",
    "            print(f'Iteration {batch_idx} single date is first batch')\n",
    "                    \n",
    "    else:\n",
    "        # Concatenate test batch to current day dataframe\n",
    "        df_test_current_day = pd.concat((df_test_current_day, df_test_batch.loc[:, test_columns]), axis=0).reset_index(drop=True)\n",
    "        if verbose:\n",
    "            print(f'Iteration {batch_idx} single date concatenated')\n",
    "            \n",
    "        if current_seconds_in_bucket == 300:\n",
    "            df_test_current_day_30th = df_test_batch.loc[:, ['stock_id'] + current_day_30th_columns].copy(deep=True).rename(columns={\n",
    "                column: f'{column}_current_day_30th' for column in current_day_30th_columns\n",
    "            }).reset_index()\n",
    "            if verbose:\n",
    "                print(f'Iteration {batch_idx} current day 30th is updated with real values')\n",
    "            \n",
    "    for period in difference_periods:\n",
    "        df_test_diff_features = df_test_current_day.groupby(['stock_id'])[difference_columns].diff(periods=period).rename(columns={\n",
    "            column: f'{column}_diff_{period}' for column in difference_columns\n",
    "        })\n",
    "        df_test_diff_features = pd.concat((\n",
    "            df_test_current_day.loc[:, ['stock_id', 'date_id', 'seconds_in_bucket']],\n",
    "            df_test_diff_features\n",
    "        ), axis=1, ignore_index=False)\n",
    "        df_test_batch = df_test_batch.merge(df_test_diff_features, on=['stock_id', 'date_id', 'seconds_in_bucket'], how='left')\n",
    "        \n",
    "    for period in ratio_periods:\n",
    "        df_test_ratio_features = df_test_current_day.groupby(['stock_id'])[ratio_columns].pct_change(periods=period).rename(columns={\n",
    "            column: f'{column}_pct_change_{period}' for column in ratio_columns\n",
    "        })\n",
    "        df_test_ratio_features = pd.concat((\n",
    "            df_test_current_day.loc[:, ['stock_id', 'date_id', 'seconds_in_bucket']],\n",
    "            df_test_ratio_features\n",
    "        ), axis=1, ignore_index=False)\n",
    "        df_test_batch = df_test_batch.merge(df_test_ratio_features, on=['stock_id', 'date_id', 'seconds_in_bucket'], how='left')\n",
    "        \n",
    "    for window in rolling_windows:\n",
    "        df_rolling_features = df_test_current_day.groupby(['stock_id'])[\n",
    "            rolling_columns\n",
    "        ].rolling(window=window, min_periods=1).agg([\n",
    "            'mean', 'std'\n",
    "        ]).reset_index(level=(0,), drop=True).astype(np.float32)\n",
    "        df_rolling_features.columns = df_rolling_features.columns.map(f'_window_{window}_'.join).str.strip('_')\n",
    "        df_rolling_features = pd.concat((\n",
    "            df_test_current_day.loc[:, ['stock_id', 'date_id', 'seconds_in_bucket']],\n",
    "            df_rolling_features\n",
    "        ), axis=1, ignore_index=False)\n",
    "        df_test_batch = df_test_batch.merge(df_rolling_features, on=['stock_id', 'date_id', 'seconds_in_bucket'], how='left')\n",
    "       \n",
    "    # Current day open difference features\n",
    "    df_test_batch = df_test_batch.merge(df_test_current_day_open, on=['stock_id'], how='left')\n",
    "    df_test_batch[[f'{column}_current_day_open_difference' for column in current_day_open_columns]] =\\\n",
    "    df_test_batch[[f'{column}_current_day_open' for column in current_day_open_columns]] -\\\n",
    "    df_test_batch[current_day_open_columns].values\n",
    "    \n",
    "    # Current day 30th difference features\n",
    "    df_test_batch = df_test_batch.merge(df_test_current_day_30th, on=['stock_id'], how='left')\n",
    "    df_test_batch[[f'{column}_current_day_30th_difference' for column in current_day_30th_columns]] =\\\n",
    "    df_test_batch[[f'{column}_current_day_30th' for column in current_day_30th_columns]] -\\\n",
    "    df_test_batch[current_day_30th_columns].values\n",
    "    \n",
    "    # Previous day close difference features\n",
    "    df_test_batch = df_test_batch.merge(df_test_previous_day_close, on=['stock_id'], how='left')\n",
    "    df_test_batch[[f'{column}_previous_day_close_difference' for column in previous_day_close_columns]] =\\\n",
    "    df_test_batch[[f'{column}_previous_day_close' for column in previous_day_close_columns]] -\\\n",
    "    df_test_batch[previous_day_close_columns].values\n",
    "    \n",
    "    # Previous day high difference features\n",
    "    df_test_batch = df_test_batch.merge(df_test_previous_day_high, on=['stock_id'], how='left')\n",
    "    df_test_batch[[f'{column}_previous_day_high_difference' for column in previous_day_high_columns]] =\\\n",
    "    df_test_batch[[f'{column}_previous_day_high' for column in previous_day_high_columns]] -\\\n",
    "    df_test_batch[previous_day_high_columns].values\n",
    "\n",
    "    # Merge daily shift features\n",
    "    df_test_previous_day_values = df_test_previous_day.loc[\n",
    "        df_test_previous_day['seconds_in_bucket'] == current_seconds_in_bucket,\n",
    "        daily_shift_columns + ['stock_id']\n",
    "    ].rename(columns={column: f'{column}_daily_shift' for column in daily_shift_columns})\n",
    "    df_test_batch = df_test_batch.merge(\n",
    "        df_test_previous_day_values,\n",
    "        on=['stock_id'],\n",
    "        how='left'\n",
    "    )\n",
    "    \n",
    "    df_test_batch[[f'{column}_stock_pct_rank' for column in stock_pct_rank_columns]] = df_test_batch[stock_pct_rank_columns].rank(pct=True).astype(np.float32)\n",
    "    \n",
    "    df_test_batch.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "            \n",
    "    batch_predictions = np.zeros(df_test_batch.shape[0])\n",
    "    for model in list(models.values()):\n",
    "        batch_predictions += (model.predict(xgb.DMatrix(df_test_batch.loc[:, config['training']['features']])) / len(models))\n",
    "    \n",
    "    df_test_batch['prediction'] = batch_predictions\n",
    "    df_test_batch['weighted_prediction'] = df_test_batch['prediction'] * df_test_batch['stock_weight']\n",
    "    df_test_batch['target'] = df_test_batch['prediction'] - df_test_batch['weighted_prediction'].sum()\n",
    "    \n",
    "    #df_predictions.append(df_test_batch)\n",
    "    \n",
    "    df_submission_batch = df_submission_batch.drop(columns=['target']).merge(df_test_batch.loc[:, ['row_id', 'target']], on='row_id', how='left')    \n",
    "    env.predict(df_submission_batch)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b07d8d1",
   "metadata": {
    "papermill": {
     "duration": 0.006316,
     "end_time": "2023-12-19T18:50:04.034908",
     "exception": false,
     "start_time": "2023-12-19T18:50:04.028592",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "databundleVersionId": 7056235,
     "sourceId": 57891,
     "sourceType": "competition"
    },
    {
     "datasetId": 4002503,
     "sourceId": 7240567,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30579,
   "isGpuEnabled": false,
   "isInternetEnabled": false,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 206.487008,
   "end_time": "2023-12-19T18:50:04.968210",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-12-19T18:46:38.481202",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}