optiver-conv-just-imb-inference-cleanup.ipynb

{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":168.614017,"end_time":"2023-10-30T00:47:50.42774","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-10-30T00:45:01.813723","version":"2.4.0"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":57891,"databundleVersionId":7056235,"sourceType":"competition"},{"sourceId":7219327,"sourceType":"datasetVersion","datasetId":4178345}],"dockerImageVersionId":30626,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"colab":{"provenance":[],"machine_shape":"hm","gpuType":"T4"},"accelerator":"GPU"},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# NS\n#optiver-ver (for ensemble)\n\n#optiver-ver-nn-modeling/inference-v1\n#optiver-ver-rnn-modeling/inference-v1\n#optiver-ver-lgb-modeling/inference-v1\n\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.linear_model import LinearRegression\n# import lightgbm as lgb\n# import xgboost as xgb\nfrom sklearn.linear_model import Ridge\nfrom sklearn.metrics import mean_absolute_error\nimport matplotlib.pyplot as plt\nfrom scipy.stats import hmean\n\nfrom sklearn.ensemble import HistGradientBoostingRegressor\nimport itertools\nimport pickle\nimport joblib\nfrom itertools import combinations\nfrom tqdm import tqdm\n\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Input, Dense, Embedding, Flatten, Concatenate, GaussianNoise\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom sklearn.preprocessing import StandardScaler\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.experimental import CosineDecay\nfrom tensorflow.keras.optimizers.schedules import ExponentialDecay\nfrom tensorflow.keras.layers import concatenate,Dropout\nimport pickle\nfrom tensorflow.keras.models import load_model\nimport os\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import Huber\nfrom tensorflow.keras.metrics import MeanAbsoluteError\nfrom tensorflow.keras.callbacks import Callback\nimport random\nfrom tensorflow.keras.layers import Input, Embedding, Lambda, Reshape, LSTM, Dense, BatchNormalization, Dropout, concatenate\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.layers import ZeroPadding1D\nfrom tensorflow.keras.layers import Conv1D\nfrom tensorflow.keras.layers import RepeatVector\n\nfrom tensorflow.keras.layers import Input, Embedding, Lambda, Reshape, LSTM, Dense, BatchNormalization, Dropout, concatenate\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.layers import ZeroPadding1D, Activation\nfrom tensorflow.keras.layers import Conv1D\nfrom tensorflow.keras.layers import RepeatVector\nfrom tensorflow.keras.layers import MaxPooling1D, AveragePooling1D\nfrom tensorflow.keras.layers import Add\n\n#---------------------------------------------------------------------- setup dashboard ------------------------------------------------------------\n\nkaggle              = True\nis_inference        = True\nload_models         = True\nrun_pipeline        = False\ntrain_models        = False\nis_rnn              = True   #3\nonline_learning     = True\n\n\n\n\n#public-validation\n# dates_train = [0,390]\n# dates_test = [391,480]\n\n#full-inference\ndates_train = [0,480]\ndates_test = [-1,-1]\n\n\nnum_models ={'rnn':2}\n\n\nif kaggle:\n    train_path = \"/kaggle/input/optiver-trading-at-the-close/train.csv\"\n    models_path = \"/kaggle/input/optiver-rnn-just-imb-models/\"\nelse:\n    models_path = \"optiver-inference-models/\"\n    train_path = \"train.csv\"\n\nif dates_train[1]!=480:\n    models_path = \"vals-258/\"\n#---------------------------------------------------------------------- setup dashboard ------------------------------------------------------------\n\nsimulation_path = \"vals-258/\"","metadata":{"papermill":{"duration":13.419877,"end_time":"2023-10-30T00:45:18.598788","exception":false,"start_time":"2023-10-30T00:45:05.178911","status":"completed"},"tags":[],"id":"AHqylKuN78oc","execution":{"iopub.status.busy":"2023-12-18T01:16:23.844091Z","iopub.execute_input":"2023-12-18T01:16:23.844658Z","iopub.status.idle":"2023-12-18T01:16:39.729781Z","shell.execute_reply.started":"2023-12-18T01:16:23.844607Z","shell.execute_reply":"2023-12-18T01:16:39.728659Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"\nrnn_ep         = 1\nrnn_lr         = 0.001\nrnn_bs         = 2**12\nwindow_size    = 3","metadata":{"id":"0K9ZMhVz78of","execution":{"iopub.status.busy":"2023-12-18T01:16:39.731735Z","iopub.execute_input":"2023-12-18T01:16:39.732426Z","iopub.status.idle":"2023-12-18T01:16:39.740637Z","shell.execute_reply.started":"2023-12-18T01:16:39.732387Z","shell.execute_reply":"2023-12-18T01:16:39.737744Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"#@title read train\n\ncolab = False\nif colab:\n  from google.colab import drive\n  drive.mount('/content/drive')\n  train_path = '/content/drive/My Drive/optiver/train.csv'\n  models_path = \"/content/drive/My Drive/optiver/inference/optiver-rnn-ji/\"\nelse:\n    if not kaggle:\n        from public_timeseries_testing_util import MockApi\n\npd.set_option('mode.chained_assignment', None)\n\n\ndef convert_price_cols_float32(df):\n\n    # Columns containing 'price'\n    price_columns = [col for col in df.columns if 'price' in col]\n    df[price_columns] = df[price_columns].astype('float32')\n\n    # Columns containing 'wap'\n    wap_columns = [col for col in df.columns if 'wap' in col]\n    df[wap_columns] = df[wap_columns].astype('float32')\n\n    return df\n\ntrain = pd.read_csv(train_path).drop(['row_id', 'time_id'], axis = 1)\nnan_count = train['target'].isna().sum()\nprint(f\"The 'target' column has {nan_count} NaN values.\")\n\ntarget_median = train['target'].median()\ntrain['target'].fillna(target_median, inplace=True)\n\nprint(f\"converting prices columns to float32 values.\")\ntrain = convert_price_cols_float32(train)\n# ----------------------------- Reading train data -------------------------","metadata":{"papermill":{"duration":17.742575,"end_time":"2023-10-30T00:45:36.347807","exception":false,"start_time":"2023-10-30T00:45:18.605232","status":"completed"},"tags":[],"id":"chPvI9VC78oh","outputId":"82b55ddd-f37c-4ce1-c214-f608b1bb3f04","execution":{"iopub.status.busy":"2023-12-18T01:16:39.742084Z","iopub.execute_input":"2023-12-18T01:16:39.742745Z","iopub.status.idle":"2023-12-18T01:17:01.304481Z","shell.execute_reply.started":"2023-12-18T01:16:39.742694Z","shell.execute_reply":"2023-12-18T01:17:01.303033Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"The 'target' column has 88 NaN values.\nconverting prices columns to float32 values.\n","output_type":"stream"}]},{"cell_type":"code","source":"#@title functions\n\ndef split_by_date(df, dates):\n\n    df_start, df_end = dates\n    df = df[(df['date_id'] >= df_start) & (df['date_id'] <=df_end)].reset_index(drop=True)\n\n    return df\n\n\n\ndef lag_function(df, columns_to_lag, numbers_of_days_to_lag):\n\n    df_indexed = df.set_index(['stock_id', 'seconds_in_bucket', 'date_id'])\n\n    for column_to_lag in columns_to_lag:\n        for number_days_to_lag in numbers_of_days_to_lag:\n            df_indexed[f'lag{number_days_to_lag}_{column_to_lag}'] = df_indexed.groupby(level=['stock_id', 'seconds_in_bucket'])[column_to_lag].shift(number_days_to_lag)\n\n    df_indexed.reset_index(inplace=True)\n\n    return df_indexed\n\n\n\n\ndef compute_imbalances(df_, columns, prefix = ''):\n    \"\"\"Computes the differences and imbalances for pairs of columns and stores them in the DataFrame.\"\"\"\n    df = df_.copy()\n    for col1, col2 in combinations(columns, 2):\n\n        # Sort the columns lexicographically to ensure consistent ordering\n        col1, col2 = sorted([col1, col2])\n\n        # Compute imbalance directly without creating a temporary difference column\n        total = df[col1] + df[col2]\n        imbalance_column_name = f'{col1}_{col2}_imb{prefix}'\n\n        # Ensure we don't divide by zero\n        df[imbalance_column_name] = (df[col1] - df[col2]).divide(total, fill_value=np.nan)\n\n    return df\n\n\n\ndef save_pickle(data, file_path):\n\n    # Create the directory if it doesn't exist\n    directory = os.path.dirname(file_path)\n    if not os.path.exists(directory):\n        os.makedirs(directory)\n\n    # Save the pickle file\n    with open(file_path, 'wb') as file:\n        pickle.dump(data, file)\n\n    print(f\"Data saved to {file_path}\")\n    #example: save_pickle(all_data, 'k8/all_data.pkl')\n\ndef load_pickle(file_path):\n\n    # Load and return the data from the pickle file\n    if os.path.exists(file_path):\n        with open(file_path, 'rb') as file:\n            data = pickle.load(file)\n        return data\n    else:\n        raise FileNotFoundError(f\"No such file: {file_path}\")\n\n    #example: loaded_data = load_pickle('k8/all_data.pkl')\n\n","metadata":{"papermill":{"duration":0.048164,"end_time":"2023-10-30T00:45:36.402382","exception":false,"start_time":"2023-10-30T00:45:36.354218","status":"completed"},"tags":[],"cellView":"form","id":"Odw5lqHN78oi","execution":{"iopub.status.busy":"2023-12-18T01:17:01.306889Z","iopub.execute_input":"2023-12-18T01:17:01.308423Z","iopub.status.idle":"2023-12-18T01:17:01.326960Z","shell.execute_reply.started":"2023-12-18T01:17:01.308335Z","shell.execute_reply":"2023-12-18T01:17:01.325669Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"#@title pipeline\n\nraw_cols          = ['imbalance_size','matched_size','bid_size','ask_size','reference_price','far_price','near_price','bid_price','ask_price','wap','imbalance_buy_sell_flag']\n\ncolumns_prices    = ['reference_price','far_price','near_price','bid_price','ask_price','wap']\ncolumns_4prices   = ['reference_price','bid_price','ask_price','wap']\n\ncolumns_sizes     = ['imbalance_size','matched_size','bid_size','ask_size']\ncolumns_flag      = ['imbalance_buy_sell_flag']\n\n\nnum_of_target_lags  = 3\ntarget_lags         = list(range(1,num_of_target_lags+1))\n\n\n\n\n\ndef feature_pipeline(df):\n\n    if df.empty:\n        return pd.DataFrame()\n\n\n    df = compute_imbalances(df, columns_sizes,prefix='_sz_')\n    df = compute_imbalances(df, columns_prices,prefix = '_pr_')\n\n\n    print(f\"lagging target column for {len(target_lags)} lags.\")\n    df = lag_function(df, ['target'], target_lags)\n\n\n\n    print(\"Done...\")\n\n    return df","metadata":{"papermill":{"duration":0.020161,"end_time":"2023-10-30T00:45:36.45631","exception":false,"start_time":"2023-10-30T00:45:36.436149","status":"completed"},"tags":[],"id":"0rJWHorY78ol","execution":{"iopub.status.busy":"2023-12-18T01:17:01.331400Z","iopub.execute_input":"2023-12-18T01:17:01.332280Z","iopub.status.idle":"2023-12-18T01:17:01.361136Z","shell.execute_reply.started":"2023-12-18T01:17:01.332209Z","shell.execute_reply":"2023-12-18T01:17:01.359682Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"\ndef make_predictions(models, X_test,model = 'nn'):\n    if model == 'nn':\n        all_predictions = [model.predict(X_test, batch_size=16384) for model in models]\n    if model == 'lgb' or model == 'xgb' or model == 'cat':\n        all_predictions = [model.predict(X_test) for model in models]\n    prediction = np.mean(all_predictions, axis=0)\n    return prediction\n\ndef set_all_seeds(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\nclass BestScoresCallback(Callback):\n    def __init__(self):\n        super().__init__()\n        self.best_train_loss = float('inf')\n        self.best_val_loss = float('inf')\n\n    def on_epoch_end(self, epoch, logs=None):\n        train_loss = logs.get('loss', float('inf'))\n        val_loss = logs.get('val_loss', float('inf'))\n\n        if train_loss < self.best_train_loss:\n            self.best_train_loss = train_loss\n        if val_loss < self.best_val_loss:\n            self.best_val_loss = val_loss\n\n    def on_train_end(self, logs=None):\n        print(f\"Best training loss: {self.best_train_loss}, Best validation loss: {self.best_val_loss}\")\n","metadata":{"papermill":{"duration":0.03576,"end_time":"2023-10-30T00:45:36.517275","exception":false,"start_time":"2023-10-30T00:45:36.481515","status":"completed"},"tags":[],"cellView":"form","id":"inHoh5ka78ol","execution":{"iopub.status.busy":"2023-12-18T01:17:01.363796Z","iopub.execute_input":"2023-12-18T01:17:01.364300Z","iopub.status.idle":"2023-12-18T01:17:01.378375Z","shell.execute_reply.started":"2023-12-18T01:17:01.364253Z","shell.execute_reply":"2023-12-18T01:17:01.377354Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"#@title RNN second pass\n\ndef precompute_sequences(stock_data, window_size, rnn_numerical_features, rnn_categorical_features):\n    # Convert DataFrame columns to NumPy arrays\n    stock_data_num = stock_data[rnn_numerical_features].values\n    stock_data_cat = stock_data[rnn_categorical_features].values\n\n    # Pre-compute all sequences\n    all_sequences_num = [stock_data_num[max(0, i - window_size + 1):i + 1] for i in range(len(stock_data))]\n    all_sequences_cat = [stock_data_cat[max(0, i - window_size + 1):i + 1] for i in range(len(stock_data))]\n\n    # Add padding if necessary\n    padded_sequences_num = [np.pad(seq, ((window_size - len(seq), 0), (0, 0)), 'constant') for seq in all_sequences_num]\n    padded_sequences_cat = [np.pad(seq, ((window_size - len(seq), 0), (0, 0)), 'constant') for seq in all_sequences_cat]\n\n    # Combine numerical and categorical features\n    combined_sequences = np.array([np.concatenate([num, cat], axis=-1)\n                                   for num, cat in zip(padded_sequences_num, padded_sequences_cat)])\n\n    # Extract targets\n    targets = stock_data['target'].values\n\n    return combined_sequences, targets\n\ndef get_sequence(precomputed_data, time_step):\n    combined_sequences, targets = precomputed_data\n    return combined_sequences[time_step], targets[time_step]\n\n\n\ndef create_batches(data, window_size, rnn_numerical_features, rnn_categorical_features, max_time_steps=55):\n\n    grouped = data.groupby(['stock_id', 'date_id'])\n    all_batches = []\n    all_targets = []\n\n    for _, group in tqdm(grouped, desc=\"Processing groups\"):\n        # Precompute sequences for the current group\n        precomputed_data = precompute_sequences(group, window_size, rnn_numerical_features, rnn_categorical_features)\n\n        # Initialize containers for group sequences and targets\n        group_sequences = []\n        group_targets = []\n\n        # Iterate over the time steps and retrieve precomputed sequences\n        for time_step in range(max_time_steps):\n            sequence, target = get_sequence(precomputed_data, time_step)\n            if sequence.size > 0:\n                group_sequences.append(sequence)\n                group_targets.append(target)\n\n        # Extend the main batches with the group's sequences and targets\n        all_batches.extend(group_sequences)\n        all_targets.extend(group_targets)\n\n    return all_batches, all_targets\n\n\n\n\ndef compute_last_sequence(group, window_size, rnn_numerical_features, rnn_categorical_features):\n    # Convert DataFrame columns to NumPy arrays\n    stock_data_num      = group[rnn_numerical_features].values\n    stock_data_cat      = group[rnn_categorical_features].values\n    stock_data_target   = group['target'].values\n\n    # Find the index of the target second\n    target_index = len(group) - 1\n\n    # Extract the sequence for the target index\n    sequence_num = stock_data_num[max(0, target_index - window_size + 1):target_index + 1]\n    sequence_cat = stock_data_cat[max(0, target_index - window_size + 1):target_index + 1]\n\n    # Add padding if necessary\n    padded_sequence_num = np.pad(sequence_num, ((window_size - len(sequence_num), 0), (0, 0)), 'constant')\n    padded_sequence_cat = np.pad(sequence_cat, ((window_size - len(sequence_cat), 0), (0, 0)), 'constant')\n\n    # Combine numerical and categorical features\n    combined_sequence = np.concatenate([padded_sequence_num, padded_sequence_cat], axis=-1)\n\n    # Extract target\n    target = stock_data_target[-1]\n\n    return combined_sequence, target\n\n\ndef create_last_batches(data, window_size, rnn_numerical_features, rnn_categorical_features):\n\n    grouped = data.groupby(['stock_id'])\n    all_batches = []\n    all_targets = []\n\n    for _, group in grouped:\n        # Compute the sequence for the last data point in the current group\n        last_sequence, last_target = compute_last_sequence(group, window_size, rnn_numerical_features, rnn_categorical_features)\n\n        # Check if the sequence is valid (i.e., not empty)\n        if last_sequence.size > 0:\n            all_batches.append(last_sequence)\n            all_targets.append(last_target)\n\n    return all_batches, all_targets\n\n\n\n\n\ndef second_pass_for_rnn(df, rnn_numerical_features, rnn_categorical_features, window_size, is_inference=False):\n    # Check if the DataFrame is empty\n    global rnn_scaler,rnn_medians\n\n    if df.empty:\n        return None, None\n\n    # Work on a copy of the DataFrame to avoid changing the original df\n    df_copy = df.copy()\n\n    # Standard scaling for numerical features\n    if is_inference:\n        df_copy.fillna(rnn_medians, inplace=True)\n        df_copy[rnn_numerical_features] = rnn_scaler.transform(df_copy[rnn_numerical_features])\n\n    # Preprocess Data\n    df_copy['seconds_in_bucket'] = df_copy['seconds_in_bucket'] / 10\n    df_copy['imbalance_buy_sell_flag'] += 1\n\n    if is_inference:\n        df_copy_batches, df_copy_targets = create_last_batches(df_copy, window_size, rnn_numerical_features, rnn_categorical_features)\n    else:\n        df_copy_batches, df_copy_targets = create_batches(df_copy, window_size, rnn_numerical_features, rnn_categorical_features)\n\n    df_copy_batches = np.array(df_copy_batches)\n    df_copy_targets = np.array(df_copy_targets)\n\n\n    return df_copy_batches, df_copy_targets\n\n\ndef online_pass_for_rnn(df, rnn_numerical_features, rnn_categorical_features, window_size):\n    # Check if the DataFrame is empty\n    global rnn_scaler,rnn_medians\n\n    if df.empty:\n        return None, None\n\n    # Work on a copy of the DataFrame to avoid changing the original df\n    df_copy = df.copy()\n\n    # Standard scaling for numerical features\n    df_copy.fillna(rnn_medians, inplace=True)\n    df_copy[rnn_numerical_features] = rnn_scaler.transform(df_copy[rnn_numerical_features])\n\n    # Preprocess Data\n    df_copy['seconds_in_bucket'] = df_copy['seconds_in_bucket'] / 10\n    df_copy['imbalance_buy_sell_flag'] += 1\n\n\n    grouped = df_copy.groupby(['stock_id'])\n    all_batches = []\n    all_targets = []\n\n    for _, group in tqdm(grouped, desc=\"Processing groups\"):\n        # Precompute sequences for the current group\n        precomputed_data = precompute_sequences(group, window_size, rnn_numerical_features, rnn_categorical_features)\n\n        # Initialize containers for group sequences and targets\n        group_sequences = []\n        group_targets = []\n\n        # Iterate over the time steps and retrieve precomputed sequences\n        for time_step in range(55):\n            sequence, target = get_sequence(precomputed_data, time_step)\n            if sequence.size > 0:\n                group_sequences.append(sequence)\n                group_targets.append(target)\n\n        # Extend the main batches with the group's sequences and targets\n        all_batches.extend(group_sequences)\n        all_targets.extend(group_targets)\n\n    df_batches = np.array(all_batches)\n    df_targets = np.array(all_targets)\n\n\n    return df_batches, df_targets\n\n","metadata":{"cellView":"form","id":"kl7j4Jbr78om","execution":{"iopub.status.busy":"2023-12-18T01:17:01.380105Z","iopub.execute_input":"2023-12-18T01:17:01.380758Z","iopub.status.idle":"2023-12-18T01:17:01.408667Z","shell.execute_reply.started":"2023-12-18T01:17:01.380708Z","shell.execute_reply":"2023-12-18T01:17:01.407287Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"#@title RNN model\n\ndef apply_conv_layers(input_layer, kernel_sizes, filters=16, do_ratio=0.5):\n    conv_outputs = []\n\n    for kernel_size in kernel_sizes:\n        conv_layer = Conv1D(filters=filters, kernel_size=kernel_size, activation='relu', padding='same')(input_layer)\n        conv_layer = BatchNormalization()(conv_layer)\n        conv_layer = Dropout(do_ratio)(conv_layer)\n\n        shortcut = conv_layer\n\n        conv_layer = Conv1D(filters=filters, kernel_size=kernel_size, padding='same')(conv_layer)\n        conv_layer = BatchNormalization()(conv_layer)\n        conv_layer = Activation('relu')(conv_layer)\n\n        # Add the output of the first Conv1D layer\n        conv_layer = Add()([conv_layer, shortcut])\n        conv_outputs.append(conv_layer)\n\n\n    concatenated_conv = Concatenate(axis=-1)(conv_outputs)\n    flattened_conv_output = Flatten()(concatenated_conv)\n\n    return flattened_conv_output\n\ndef create_rnn_model_with_residual(window_size, numerical_features, initial_learning_rate=0.001):\n\n    categorical_features = 'seconds_in_bucket'\n    categorical_uniques  = { 'seconds_in_bucket' : 55}\n    embedding_dim        = {'seconds_in_bucket' : 10}\n\n    input_layer = Input(shape=(window_size, len(numerical_features) + 1), name=\"combined_input\")\n\n    # Split the input into numerical and categorical parts\n    numerical_input = Lambda(lambda x: x[:, :, :-1], name=\"numerical_part\")(input_layer)\n    categorical_input = Lambda(lambda x: x[:, :, -1:], name=\"categorical_part\")(input_layer)\n\n    first_numerical = Lambda(lambda x: x[:, 0])(numerical_input)\n\n\n    # diffrentiate layers\n    def create_difference_layer(lag):\n        return Lambda(lambda x: x[:, lag:, :] - x[:, :-lag, :], name=f\"difference_layer_lag{lag}\")\n\n    difference_layers = []\n    for lag in range(1, window_size):\n        diff_layer = create_difference_layer(lag)(numerical_input)\n        padding = ZeroPadding1D(padding=(lag, 0))(diff_layer)  # Add padding to the beginning of the sequence\n        difference_layers.append(padding)\n    combined_diff_layer = Concatenate(name=\"combined_difference_layer\")(difference_layers)\n\n\n    # Embedding for categorical part\n    vocab_size, embedding_dim = categorical_uniques[categorical_features], embedding_dim[categorical_features]\n    embedding = Embedding(vocab_size, embedding_dim, input_length=window_size)(categorical_input)\n    embedding = Reshape((window_size, -1))(embedding)\n\n    first_embedding = Lambda(lambda x: x[:, 0])(embedding)\n\n    # Concatenate numerical input and embedding\n    # conv_input = concatenate([enhanced_numerical_input, embedding], axis=-1)\n\n    kernel_sizes = [2,3]\n    do_ratio = 0.4\n\n    flattened_conv_output = apply_conv_layers(numerical_input, kernel_sizes, do_ratio=do_ratio)\n    flattened_conv_output_cat = apply_conv_layers(embedding, kernel_sizes, do_ratio=do_ratio)\n    flattened_conv_output_diff = apply_conv_layers(combined_diff_layer, kernel_sizes, do_ratio=do_ratio)\n\n\n    dense_output = Concatenate(axis=-1)([flattened_conv_output,flattened_conv_output_cat,flattened_conv_output_diff, Reshape((-1,))(combined_diff_layer),first_numerical,first_embedding])\n\n    dense_sizes = [512, 256, 128, 64, 32]\n    do_ratio = 0.5\n    for size in dense_sizes:\n        dense_output = Dense(size, activation='swish')(dense_output)\n        dense_output = BatchNormalization()(dense_output)\n        dense_output = Dropout(do_ratio)(dense_output)\n\n    # Output layer\n    output = Dense(1, name='output_layer')(dense_output)\n\n    # Learning rate schedule\n    lr_schedule = ExponentialDecay(\n        initial_learning_rate=initial_learning_rate,\n        decay_steps=10000,\n        decay_rate=0.7,\n        staircase=True)\n\n    # Create and compile the model\n    model = Model(inputs=input_layer, outputs=output)\n    optimizer = Adam(learning_rate=lr_schedule)\n\n    model.compile(optimizer=optimizer, loss=\"mean_absolute_error\")\n\n    return model","metadata":{"id":"AVwvP9HX78on","execution":{"iopub.status.busy":"2023-12-18T01:17:01.410838Z","iopub.execute_input":"2023-12-18T01:17:01.411233Z","iopub.status.idle":"2023-12-18T01:17:01.430613Z","shell.execute_reply.started":"2023-12-18T01:17:01.411197Z","shell.execute_reply":"2023-12-18T01:17:01.429664Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"#@title runnig pipeline\n\nexcluded_columns = ['row_id', 'date_id', 'time_id', 'target','stock_return']\n\nif run_pipeline:\n\n    train_eng = feature_pipeline(train)\n\n    if is_rnn:\n        excluded_columns = excluded_columns + ['stock_id']\n\n        features = [col for col in train_eng.columns if col not in excluded_columns]\n        rnn_categorical_features =  ['seconds_in_bucket']\n        rnn_numerical_features = [feat for feat in features if feat not in rnn_categorical_features]\n        print(\"we have {} numerical and {} categorical\".format(len(rnn_numerical_features),len(rnn_categorical_features)))\n\n\n        rnn_scaler = StandardScaler()\n        rnn_medians = train_eng.median()\n\n        train_eng.fillna(rnn_medians, inplace=True)\n        train_eng[rnn_numerical_features] = rnn_scaler.fit_transform(train_eng[rnn_numerical_features])\n\n\n        rnn_all_data = {\n            \"rnn_scaler\": rnn_scaler,\n            \"rnn_medians\": rnn_medians,\n            \"rnn_categorical_features\": rnn_categorical_features,\n            \"rnn_numerical_features\": rnn_numerical_features\n        }\n\n        save_pickle(rnn_all_data, f'{models_path}rnn_all_data.pkl')\n        print(\"Pipline Done!\")\n\n\n    train_data       = split_by_date(train_eng, dates_train)\n    test_data        = split_by_date(train_eng, dates_test)\n    print(\"number of dates in train = {} , number of dates in test {}\".format (train_data['date_id'].nunique(),test_data['date_id'].nunique()))\n\n    cleaning = False\n    if cleaning:\n        import gc\n        #del train\n        del train_eng\n        gc.collect()\n\n\n","metadata":{"papermill":{"duration":0.404747,"end_time":"2023-10-30T00:45:36.928334","exception":false,"start_time":"2023-10-30T00:45:36.523587","status":"completed"},"tags":[],"cellView":"form","id":"ff1mJMFj78on","outputId":"0f525c2d-f6c8-47c9-9e5e-3745ae8831d1","execution":{"iopub.status.busy":"2023-12-18T01:17:01.431965Z","iopub.execute_input":"2023-12-18T01:17:01.433007Z","iopub.status.idle":"2023-12-18T01:17:18.726754Z","shell.execute_reply.started":"2023-12-18T01:17:01.432949Z","shell.execute_reply":"2023-12-18T01:17:18.725296Z"},"trusted":true},"execution_count":9,"outputs":[{"name":"stdout","text":"lagging target column for 3 lags.\nDone...\nwe have 35 numerical and 1 categorical\nData saved to vals-258/rnn_all_data.pkl\nPipline Done!\nnumber of dates in train = 6 , number of dates in test 6\n","output_type":"stream"}]},{"cell_type":"code","source":"#@title TPU\ntry:\n    # Create a TPUClusterResolver\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    # Connect to the TPU cluster\n    tf.config.experimental_connect_to_cluster(tpu)\n    # Initialize the TPU system\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    # Create a TPUStrategy for distributed training\n    tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept ValueError:\n    tpu_strategy = None  # No TPU found\n","metadata":{"papermill":{"duration":0.01349,"end_time":"2023-10-30T00:45:36.967612","exception":false,"start_time":"2023-10-30T00:45:36.954122","status":"completed"},"tags":[],"cellView":"form","id":"HYfNqtG078oo","execution":{"iopub.status.busy":"2023-12-18T01:17:18.728779Z","iopub.execute_input":"2023-12-18T01:17:18.729359Z","iopub.status.idle":"2023-12-18T01:17:18.738578Z","shell.execute_reply.started":"2023-12-18T01:17:18.729313Z","shell.execute_reply":"2023-12-18T01:17:18.737300Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"#@title prepare train models\nif train_models:\n\n    if test_data.empty:\n        print(\"--------------test data is empty so adjusting the last date for test-----------------\")\n        test_data = train_data.query(\"date_id == 480\").copy()\n\n    if is_rnn:\n\n        train_batches, train_targets = second_pass_for_rnn(train_data, rnn_numerical_features, rnn_categorical_features, window_size)\n        test_batches, test_targets  = second_pass_for_rnn(test_data, rnn_numerical_features, rnn_categorical_features, window_size)\n        print(f\"train batches shape:{train_batches.shape}\")\n","metadata":{"cellView":"form","id":"rMVaGjaM78oo","outputId":"c926813f-17c2-4941-d9fc-46821fecb205","execution":{"iopub.status.busy":"2023-12-18T01:17:18.740419Z","iopub.execute_input":"2023-12-18T01:17:18.741211Z","iopub.status.idle":"2023-12-18T01:17:38.310597Z","shell.execute_reply.started":"2023-12-18T01:17:18.741175Z","shell.execute_reply":"2023-12-18T01:17:38.309265Z"},"trusted":true},"execution_count":11,"outputs":[{"name":"stderr","text":"Processing groups: 100%|██████████| 1200/1200 [00:09<00:00, 126.69it/s]\nProcessing groups: 100%|██████████| 1200/1200 [00:09<00:00, 127.26it/s]\n","output_type":"stream"},{"name":"stdout","text":"train batches shape:(66000, 3, 36)\n","output_type":"stream"}]},{"cell_type":"code","source":"#@title training models\n# train_data = flatten_outliers(train_data, 'target', lower_quantile=0.01, upper_quantile=0.99)\nif train_models:\n\n        directory = os.path.dirname(models_path)\n        if not os.path.exists(directory):\n            os.makedirs(directory)\n\n\n        if is_rnn:\n\n            callbacks = [BestScoresCallback()]  # Always include BestScoresCallback\n            if dates_train[1] != 480:\n                early_stopping = EarlyStopping(monitor='val_loss', patience=20, restore_best_weights=False)\n                callbacks.append(early_stopping)\n\n            if True or tpu_strategy:\n                # with tpu_strategy.scope():\n\n                    rnn_models = []\n                    for i in range(num_models['rnn']):\n\n                        print(f\"Training rnn model {i+1} out of {num_models['rnn']} with seed {42+i}\")\n                        print(\"---------------------------------------\")\n                        set_all_seeds(42+i)\n\n                        rnn_model = create_rnn_model_with_residual(window_size, rnn_numerical_features, initial_learning_rate=rnn_lr)\n                        history = rnn_model.fit(train_batches, train_targets, validation_data=(test_batches, test_targets), epochs=rnn_ep, batch_size=rnn_bs, callbacks=callbacks)\n                        print(\"---------------------------------------\")\n                        rnn_model.save(f'{models_path}rnn_model_seed_{i}.h5')\n                        rnn_models.append(rnn_model)\n\n                    predictions =  make_predictions(rnn_models, test_batches,model = 'nn')\n                    print(f\"Ensemble Mean Absolute Error: {mean_absolute_error(test_targets, predictions):.4f}\")\n","metadata":{"papermill":{"duration":0.017103,"end_time":"2023-10-30T00:45:36.990986","exception":false,"start_time":"2023-10-30T00:45:36.973883","status":"completed"},"tags":[],"cellView":"form","id":"quC3VRDB78oo","outputId":"12637a44-1198-4d6e-db69-92cfe1e9c086","execution":{"iopub.status.busy":"2023-12-18T01:17:38.312154Z","iopub.execute_input":"2023-12-18T01:17:38.312641Z","iopub.status.idle":"2023-12-18T01:18:27.156130Z","shell.execute_reply.started":"2023-12-18T01:17:38.312606Z","shell.execute_reply":"2023-12-18T01:18:27.155090Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stdout","text":"Training rnn model 1 out of 2 with seed 42\n---------------------------------------\n17/17 [==============================] - 17s 332ms/step - loss: 6.1802 - val_loss: 6.0470\nBest training loss: 6.180210113525391, Best validation loss: 6.046993732452393\n---------------------------------------\n","output_type":"stream"},{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/keras/src/engine/training.py:3000: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n  saving_api.save_model(\n","output_type":"stream"},{"name":"stdout","text":"Training rnn model 2 out of 2 with seed 43\n---------------------------------------\n17/17 [==============================] - 16s 353ms/step - loss: 6.1880 - val_loss: 6.0473\nBest training loss: 6.180210113525391, Best validation loss: 6.046993732452393\n---------------------------------------\n5/5 [==============================] - 2s 162ms/step\n5/5 [==============================] - 2s 159ms/step\nEnsemble Mean Absolute Error: 6.0466\n","output_type":"stream"}]},{"cell_type":"code","source":"#@title timeseries functions\n\ndef clean_format(df):\n    df['target'] = df['target'].astype('float64')\n    for feat in ['stock_id','date_id','seconds_in_bucket']:\n        df[feat] = df[feat].astype('int64')\n    return df\n\ndef clean_up(df):\n    df = df[['stock_id','revealed_date_id','seconds_in_bucket','revealed_target']].rename(columns={'revealed_date_id': 'date_id', 'revealed_target': 'target'})\n    df['target'].fillna(-0.06020069, inplace=True)\n    df = clean_format(df)\n    return df\n\ndef manage_buffer(buffer, df, max_lag):\n\n    # df['iteration'] = df['iteration'].max()  # Ensure the iteration number is consistent in the new df chunk\n    if buffer.empty:\n        return df.copy()\n    elif buffer['iteration'].nunique() < max_lag:\n        return pd.concat([buffer, df], ignore_index=True)\n    else:\n        oldest_iteration = buffer['iteration'].min()\n        buffer = buffer[buffer['iteration'] > oldest_iteration]\n        return pd.concat([buffer, df], ignore_index=True)\n\n\ndef append_target_lags(df, target_buffer, lags):\n\n    column_to_lag = 'target'\n    df_lagged = df.copy()\n\n    for lag in lags:\n\n        temp_df = target_buffer.copy()\n        temp_df['date_id'] = temp_df['date_id'] + lag\n\n        temp_df.rename(columns={column_to_lag: f'lag{lag}_{column_to_lag}'}, inplace=True)\n\n        df_lagged = pd.merge(df_lagged, temp_df[['stock_id', 'date_id', 'seconds_in_bucket', f'lag{lag}_{column_to_lag}']],\n                             on=['stock_id', 'date_id', 'seconds_in_bucket'],\n                             how='left')\n\n    return df_lagged\n\n\n","metadata":{"papermill":{"duration":0.022966,"end_time":"2023-10-30T00:45:37.057562","exception":false,"start_time":"2023-10-30T00:45:37.034596","status":"completed"},"tags":[],"cellView":"form","id":"LlYf_VO_78op","execution":{"iopub.status.busy":"2023-12-18T01:18:27.157410Z","iopub.execute_input":"2023-12-18T01:18:27.158471Z","iopub.status.idle":"2023-12-18T01:18:27.170324Z","shell.execute_reply.started":"2023-12-18T01:18:27.158429Z","shell.execute_reply":"2023-12-18T01:18:27.168461Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"if kaggle:\n    import gc\n    # remove train_small and train and clear the memory\n    del train\n    gc.collect()","metadata":{"papermill":{"duration":0.229017,"end_time":"2023-10-30T00:45:37.292673","exception":false,"start_time":"2023-10-30T00:45:37.063656","status":"completed"},"tags":[],"id":"bKPY9kxN78oq","execution":{"iopub.status.busy":"2023-12-18T01:18:27.174688Z","iopub.execute_input":"2023-12-18T01:18:27.175476Z","iopub.status.idle":"2023-12-18T01:18:27.674211Z","shell.execute_reply.started":"2023-12-18T01:18:27.175432Z","shell.execute_reply":"2023-12-18T01:18:27.673016Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"#@title is inference\nif is_inference:\n\n    if online_learning:\n\n\n        #rnn\n        online_rnn_lr_rate    = 2e-4 #2e-4\n        online_rnn_batch_size = 2**13 #2**12 reduced for memory\n        online_rnn_epochs     = 4      #2\n\n\n\n    if kaggle:\n        import optiver2023\n        env = optiver2023.make_env()\n        iter_test = env.iter_test()\n    else:\n        env = MockApi()\n        iter_test = env.iter_test()\n\n    if load_models:\n\n\n\n        if is_rnn:\n\n            print(\"loading rnn models...\")\n            loaded_data          = load_pickle(f'{models_path}rnn_all_data.pkl')\n            rnn_scaler               = loaded_data[\"rnn_scaler\"]\n            rnn_medians              = loaded_data[\"rnn_medians\"]\n            rnn_categorical_features = loaded_data[\"rnn_categorical_features\"]\n            rnn_numerical_features   = loaded_data[\"rnn_numerical_features\"]\n\n            rnn_models = []\n            for i in range(num_models['rnn']):\n                loaded_model = load_model(f\"{models_path}rnn_model_seed_{i}.h5\")\n                rnn_models.append(loaded_model)\n            print(\"Done!\")\n\n\n\n    #---------------------------buffer management ---------------------------\n    max_target_lag  = num_of_target_lags\n    i               = 0\n    target_counter  = 0\n    target_buffer   = pd.DataFrame()\n    total_test      = pd.DataFrame()\n    daily_online    = pd.DataFrame()\n    #---------------------------buffer management ---------------------------\n\n\n    for (test, revealed_targets, sample_prediction) in iter_test:\n\n\n        test = convert_price_cols_float32(test)\n\n\n        if test.seconds_in_bucket.iloc[0] == 0:\n\n            #----------------------------- storing previous targets -------------------------\n            revealed_targets = clean_up(revealed_targets)\n            revealed_targets['iteration'] = target_counter\n            target_buffer = manage_buffer(target_buffer, revealed_targets, max_target_lag)\n            target_counter += 1\n\n\n\n            if not daily_online.empty and online_learning:\n\n                daily_online.drop(columns='target', inplace=True)\n                daily_online = pd.merge(daily_online, revealed_targets[['stock_id', 'date_id', 'seconds_in_bucket', 'target']],\n                                            on=['stock_id', 'date_id', 'seconds_in_bucket'],\n                                            how='inner')\n\n\n                if is_rnn:\n\n                    online_batches, online_targets = online_pass_for_rnn(daily_online, rnn_numerical_features, rnn_categorical_features, window_size)\n\n                    for rnn_model in rnn_models:\n                        lr_schedule = ExponentialDecay( initial_learning_rate=online_rnn_lr_rate, decay_steps=10000, decay_rate=0.8,staircase=True)\n                        optimizer = Adam(learning_rate=lr_schedule)\n                        rnn_model.compile(optimizer=optimizer, loss = \"mean_absolute_error\")\n                        rnn_model.fit(online_batches, online_targets, epochs=online_rnn_epochs, batch_size=online_rnn_batch_size)\n                        K.clear_session()\n                        gc.collect()\n\n            daily_online = pd.DataFrame()\n            #----------------------------- storing previous targets -------------------------\n\n\n        test['target'] = 0\n\n\n \n        test = compute_imbalances(test, columns_sizes,prefix='_sz_')\n        test = compute_imbalances(test, columns_prices,prefix = '_pr_')\n\n        test = append_target_lags(test, target_buffer, target_lags)\n\n        if not daily_online.empty:\n            full_data = pd.concat([daily_online[test.columns], test], ignore_index=True)\n        else:\n            full_data = test\n\n        # getting back to test size\n        test = full_data.iloc[-test.shape[0]:].copy()\n\n\n        if test.currently_scored.iloc[0]:\n\n            if is_rnn:\n                X_test, _           = second_pass_for_rnn(full_data, rnn_numerical_features, rnn_categorical_features, window_size, is_inference=True)\n                test['target'] += make_predictions(rnn_models, X_test,model='nn').flatten()\n\n\n\n        if not kaggle:\n            total_test = pd.concat([total_test, test], ignore_index=True)\n\n\n        daily_online = pd.concat([daily_online, test], ignore_index=True)\n\n\n        sample_prediction = pd.merge(sample_prediction.drop(columns='target'), test[['row_id', 'target']], on=['row_id'], how='left')\n        sample_prediction['target'].fillna(0, inplace=True)\n\n        env.predict(sample_prediction)\n        i += 1","metadata":{"papermill":{"duration":128.751281,"end_time":"2023-10-30T00:47:46.050532","exception":false,"start_time":"2023-10-30T00:45:37.299251","status":"completed"},"tags":[],"id":"DVpNJANE78oq","execution":{"iopub.status.busy":"2023-12-18T01:18:27.676418Z","iopub.execute_input":"2023-12-18T01:18:27.676873Z","iopub.status.idle":"2023-12-18T01:19:38.494364Z","shell.execute_reply.started":"2023-12-18T01:18:27.676837Z","shell.execute_reply":"2023-12-18T01:19:38.493067Z"},"trusted":true},"execution_count":15,"outputs":[{"name":"stdout","text":"loading rnn models...\nDone!\nThis version of the API is not optimized and should not be used to estimate the runtime of your code on the hidden test set.\n","output_type":"stream"},{"name":"stderr","text":"Processing groups: 100%|██████████| 200/200 [00:01<00:00, 111.26it/s]\n","output_type":"stream"},{"name":"stdout","text":"Epoch 1/4\n2/2 [==============================] - 11s 168ms/step - loss: 5.3183\nEpoch 2/4\n2/2 [==============================] - 1s 154ms/step - loss: 5.3245\nEpoch 3/4\n2/2 [==============================] - 1s 163ms/step - loss: 5.3382\nEpoch 4/4\n2/2 [==============================] - 1s 165ms/step - loss: 5.3144\nEpoch 1/4\n2/2 [==============================] - 11s 172ms/step - loss: 5.3426\nEpoch 2/4\n2/2 [==============================] - 1s 154ms/step - loss: 5.2941\nEpoch 3/4\n2/2 [==============================] - 1s 159ms/step - loss: 5.3137\nEpoch 4/4\n2/2 [==============================] - 1s 153ms/step - loss: 5.3159\n","output_type":"stream"},{"name":"stderr","text":"Processing groups: 100%|██████████| 200/200 [00:01<00:00, 112.68it/s]\n","output_type":"stream"},{"name":"stdout","text":"Epoch 1/4\n2/2 [==============================] - 11s 169ms/step - loss: 5.8347\nEpoch 2/4\n2/2 [==============================] - 1s 161ms/step - loss: 5.8590\nEpoch 3/4\n2/2 [==============================] - 1s 167ms/step - loss: 5.8248\nEpoch 4/4\n2/2 [==============================] - 1s 178ms/step - loss: 5.8296\nEpoch 1/4\n2/2 [==============================] - 11s 170ms/step - loss: 5.8536\nEpoch 2/4\n2/2 [==============================] - 1s 162ms/step - loss: 5.8410\nEpoch 3/4\n2/2 [==============================] - 1s 171ms/step - loss: 5.8234\nEpoch 4/4\n2/2 [==============================] - 1s 162ms/step - loss: 5.8415\n","output_type":"stream"}]},{"cell_type":"code","source":"","metadata":{"id":"8ai-6TrA78oq"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"WShhzgK478or"},"execution_count":null,"outputs":[]}]}