4.2. Supervised learning_GB.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d62f25c9",
   "metadata": {},
   "source": [
    "# Supervised Learning: Gradient Boosting Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "sophisticated-dover",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f4bda236",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from supervised_learning.cross_validation import PanelDataSplit\n",
    "from supervised_learning.cross_validation import search_best_model\n",
    "from sample_panel.merge_datasets import merge_bank_macro_datasets\n",
    "\n",
    "from supervised_learning.estimate_errors import estimate_median_relative_error\n",
    "from supervised_learning.estimate_errors import estimate_errors"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "667106bd",
   "metadata": {},
   "source": [
    "## Preparing a Data Sample"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86bd8396",
   "metadata": {},
   "source": [
    "### Loading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "stylish-giant",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load bank panel data\n",
    "bank_data = pd.read_csv('df_response_vars.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "received-spank",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load macroeconomic data\n",
    "macro_data = pd.read_csv('macro_features.csv')\n",
    "macro_columns = macro_data.columns\n",
    "\n",
    "# Factors with lags are not used in the model. Remove factors with lags\n",
    "new_macro_columns = [col for col in macro_columns if '_lag' not in col]\n",
    "macro_data = macro_data[new_macro_columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b6c6430b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load PCA components\n",
    "pca_data = pd.read_csv('macro_pca_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "231c8250",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load additional macro variables\n",
    "macro_data1 = pd.read_csv('macro_most_inf_df.csv')\n",
    "# Clean column names\n",
    "macro_data1.columns = [col.replace('\\n', ' ') for col in macro_data1.columns]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b0edfc7",
   "metadata": {},
   "source": [
    "### Merging bank panel and macroeconomic time series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "stuck-buffalo",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Merge the bank panel and macroeconomic indicators\n",
    "result_df = merge_bank_macro_datasets(bank_data, macro_data, pca_data, macro_data1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2495b62c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Delete Nans values due to the lag of the response variable\n",
    "result_df.dropna(subset=['Provision_Lag1'], inplace=True)\n",
    "result_df.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "055ba87c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Report Date</th>\n",
       "      <th>IDRSSD</th>\n",
       "      <th>Financial Institution Name</th>\n",
       "      <th>Provision for Loan Lease Losses as % of Aver. Assets</th>\n",
       "      <th>Real GDP growth</th>\n",
       "      <th>Nominal GDP growth</th>\n",
       "      <th>Unemployment rate</th>\n",
       "      <th>Unemployment rate change</th>\n",
       "      <th>3-month Treasury rate change</th>\n",
       "      <th>BBB corporate yield</th>\n",
       "      <th>...</th>\n",
       "      <th>Nominal disposable income growth</th>\n",
       "      <th>CPI inflation rate</th>\n",
       "      <th>3-month Treasury rate</th>\n",
       "      <th>5-year Treasury yield</th>\n",
       "      <th>10-year Treasury yield</th>\n",
       "      <th>Households_Net_Worth</th>\n",
       "      <th>Developing Asia real GDP growth</th>\n",
       "      <th>Euro area inflation</th>\n",
       "      <th>Developing Asia inflation</th>\n",
       "      <th>Provision_Lag1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2003-03-31</td>\n",
       "      <td>12311</td>\n",
       "      <td>HUNTINGTON NATIONAL BANK, THE</td>\n",
       "      <td>0.54</td>\n",
       "      <td>2.1</td>\n",
       "      <td>4.1</td>\n",
       "      <td>5.9</td>\n",
       "      <td>0.2</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>6.2</td>\n",
       "      <td>...</td>\n",
       "      <td>2.9</td>\n",
       "      <td>4.2</td>\n",
       "      <td>1.2</td>\n",
       "      <td>2.9</td>\n",
       "      <td>4.2</td>\n",
       "      <td>45531292.0</td>\n",
       "      <td>6.6</td>\n",
       "      <td>3.3</td>\n",
       "      <td>3.6</td>\n",
       "      <td>0.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2003-03-31</td>\n",
       "      <td>14409</td>\n",
       "      <td>CITIZENS BANK OF MASSACHUSETTS</td>\n",
       "      <td>0.32</td>\n",
       "      <td>2.1</td>\n",
       "      <td>4.1</td>\n",
       "      <td>5.9</td>\n",
       "      <td>0.2</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>6.2</td>\n",
       "      <td>...</td>\n",
       "      <td>2.9</td>\n",
       "      <td>4.2</td>\n",
       "      <td>1.2</td>\n",
       "      <td>2.9</td>\n",
       "      <td>4.2</td>\n",
       "      <td>45531292.0</td>\n",
       "      <td>6.6</td>\n",
       "      <td>3.3</td>\n",
       "      <td>3.6</td>\n",
       "      <td>0.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2003-03-31</td>\n",
       "      <td>17147</td>\n",
       "      <td>FIRST MERCHANTS BANK, NATIONAL ASSOCIATION</td>\n",
       "      <td>1.77</td>\n",
       "      <td>2.1</td>\n",
       "      <td>4.1</td>\n",
       "      <td>5.9</td>\n",
       "      <td>0.2</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>6.2</td>\n",
       "      <td>...</td>\n",
       "      <td>2.9</td>\n",
       "      <td>4.2</td>\n",
       "      <td>1.2</td>\n",
       "      <td>2.9</td>\n",
       "      <td>4.2</td>\n",
       "      <td>45531292.0</td>\n",
       "      <td>6.6</td>\n",
       "      <td>3.3</td>\n",
       "      <td>3.6</td>\n",
       "      <td>0.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2003-03-31</td>\n",
       "      <td>23504</td>\n",
       "      <td>BRIDGEHAMPTON NATIONAL BANK, THE</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.1</td>\n",
       "      <td>4.1</td>\n",
       "      <td>5.9</td>\n",
       "      <td>0.2</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>6.2</td>\n",
       "      <td>...</td>\n",
       "      <td>2.9</td>\n",
       "      <td>4.2</td>\n",
       "      <td>1.2</td>\n",
       "      <td>2.9</td>\n",
       "      <td>4.2</td>\n",
       "      <td>45531292.0</td>\n",
       "      <td>6.6</td>\n",
       "      <td>3.3</td>\n",
       "      <td>3.6</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2003-03-31</td>\n",
       "      <td>30810</td>\n",
       "      <td>DISCOVER BANK</td>\n",
       "      <td>5.93</td>\n",
       "      <td>2.1</td>\n",
       "      <td>4.1</td>\n",
       "      <td>5.9</td>\n",
       "      <td>0.2</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>6.2</td>\n",
       "      <td>...</td>\n",
       "      <td>2.9</td>\n",
       "      <td>4.2</td>\n",
       "      <td>1.2</td>\n",
       "      <td>2.9</td>\n",
       "      <td>4.2</td>\n",
       "      <td>45531292.0</td>\n",
       "      <td>6.6</td>\n",
       "      <td>3.3</td>\n",
       "      <td>3.6</td>\n",
       "      <td>6.20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 48 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Report Date  IDRSSD                  Financial Institution Name  \\\n",
       "0  2003-03-31   12311               HUNTINGTON NATIONAL BANK, THE   \n",
       "1  2003-03-31   14409              CITIZENS BANK OF MASSACHUSETTS   \n",
       "2  2003-03-31   17147  FIRST MERCHANTS BANK, NATIONAL ASSOCIATION   \n",
       "3  2003-03-31   23504            BRIDGEHAMPTON NATIONAL BANK, THE   \n",
       "4  2003-03-31   30810                               DISCOVER BANK   \n",
       "\n",
       "   Provision for Loan Lease Losses as % of Aver. Assets  Real GDP growth  \\\n",
       "0                                               0.54                 2.1   \n",
       "1                                               0.32                 2.1   \n",
       "2                                               1.77                 2.1   \n",
       "3                                               0.00                 2.1   \n",
       "4                                               5.93                 2.1   \n",
       "\n",
       "   Nominal GDP growth  Unemployment rate  Unemployment rate change  \\\n",
       "0                 4.1                5.9                       0.2   \n",
       "1                 4.1                5.9                       0.2   \n",
       "2                 4.1                5.9                       0.2   \n",
       "3                 4.1                5.9                       0.2   \n",
       "4                 4.1                5.9                       0.2   \n",
       "\n",
       "   3-month Treasury rate change  BBB corporate yield  ...  \\\n",
       "0                          -0.5                  6.2  ...   \n",
       "1                          -0.5                  6.2  ...   \n",
       "2                          -0.5                  6.2  ...   \n",
       "3                          -0.5                  6.2  ...   \n",
       "4                          -0.5                  6.2  ...   \n",
       "\n",
       "   Nominal disposable income growth  CPI inflation rate  \\\n",
       "0                               2.9                 4.2   \n",
       "1                               2.9                 4.2   \n",
       "2                               2.9                 4.2   \n",
       "3                               2.9                 4.2   \n",
       "4                               2.9                 4.2   \n",
       "\n",
       "   3-month Treasury rate  5-year Treasury yield  10-year Treasury yield  \\\n",
       "0                    1.2                    2.9                     4.2   \n",
       "1                    1.2                    2.9                     4.2   \n",
       "2                    1.2                    2.9                     4.2   \n",
       "3                    1.2                    2.9                     4.2   \n",
       "4                    1.2                    2.9                     4.2   \n",
       "\n",
       "   Households_Net_Worth  Developing Asia real GDP growth  Euro area inflation  \\\n",
       "0            45531292.0                              6.6                  3.3   \n",
       "1            45531292.0                              6.6                  3.3   \n",
       "2            45531292.0                              6.6                  3.3   \n",
       "3            45531292.0                              6.6                  3.3   \n",
       "4            45531292.0                              6.6                  3.3   \n",
       "\n",
       "   Developing Asia inflation  Provision_Lag1  \n",
       "0                        3.6            0.74  \n",
       "1                        3.6            0.30  \n",
       "2                        3.6            0.31  \n",
       "3                        3.6            0.05  \n",
       "4                        3.6            6.20  \n",
       "\n",
       "[5 rows x 48 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f35db144",
   "metadata": {},
   "source": [
    "### Additional features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "60c0443c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Adding Fixed Effects\n",
    "# Assuming 'Financial Institution Name' is a categorical variable, so we can one-hot encode it\n",
    "result_df = pd.get_dummies(result_df, columns=['IDRSSD'], drop_first=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fad2739f",
   "metadata": {},
   "source": [
    "## Supervised model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "divine-foster",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Response variable column\n",
    "y_col = 'Provision for Loan Lease Losses as % of Aver. Assets'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "335369aa",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "gb_model = GradientBoostingRegressor(random_state=42)\n",
    "\n",
    "pipeline = Pipeline(steps=[(\"scaler\", scaler), (\"gb_model\", gb_model)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "370c06b0",
   "metadata": {},
   "source": [
    "### Train-test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3716f1ba",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# The last year is for test,remaining data - for train\n",
    "data_set_train = result_df[result_df['Report Date']<='2021-12-31'].copy()\n",
    "data_set_test = result_df[result_df['Report Date']>'2021-12-31'].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ab4d41f",
   "metadata": {},
   "source": [
    "#### Removing outliers from the train set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ab09956b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "lower_limit = np.percentile(data_set_train[y_col], 0.5)\n",
    "upper_limit = np.percentile(data_set_train[y_col], 99)\n",
    "\n",
    "data_set_train = data_set_train[(data_set_train[y_col]<=upper_limit)&(data_set_train[y_col]>=lower_limit)].copy()\n",
    "data_set_train.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93688d2b",
   "metadata": {},
   "source": [
    "## Optimizing Model Selection: Factor Selection and Hyperparameter Tuning\n",
    "The analysis on time series data for the \"average\" bank indicates that the response variable is most influenced by its lag of one quarter. Moreover, the residuals of the autoregressive time series model AR(1) exhibit the strongest correlation with the following factors: 'Real GDP growth_ema3', 'BBB corporate yield', '3-month Treasury rate change', 'Dow Jones Total Stock Market Index change', 'Market Volatility Index', 'Market Volatility Index change'. \n",
    "\n",
    "'Nominal GDP growth_ema1.75' also displays a high correlation with the AR(1) residuals, but it was excluded from the models based on economic reasoning. The decision to exclude it based on the fact that high Nominal GDP values do not always mean a healthy economy; during periods of high inflation, high Nominal GDP values can reflect inflation rather than economic well-being.\n",
    "\n",
    "To select the optimal model, we created a list of model variants that include these influential factors (models1-models11). Addidtionally, we added model12 and model13 based on the results of usupervised learning.\n",
    "\n",
    "Model selection and hyperparameter tuning are performed using cross-validation, allowing us to determine the best model and optimal hyperparameter values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f4aaf405",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Custom cross-validation split for panel data\n",
    "panel_cv = PanelDataSplit(test_size=4, date_axis=data_set_train['Report Date'], n_splits=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ee0f950e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Models\n",
    "models = {'model1': ['Provision_Lag1', 'Real GDP growth_ema3', 'BBB corporate yield'],\n",
    "          'model2': ['Provision_Lag1', 'Real GDP growth_ema3', 'Market Volatility Index'],\n",
    "          'model3': ['Provision_Lag1', 'Real GDP growth_ema3', 'Market Volatility Index change'],\n",
    "          'model4': ['Provision_Lag1', 'Real GDP growth_ema3', 'Dow Jones Total Stock Market Index change'],\n",
    "          'model5': ['Provision_Lag1', 'Real GDP growth_ema3', '3-month Treasury rate change'],\n",
    "          'model6': ['Provision_Lag1', 'Real GDP growth_ema3', 'Market Volatility Index', 'BBB corporate yield'],\n",
    "          'model7': ['Provision_Lag1', 'Real GDP growth_ema3', 'Market Volatility Index', '3-month Treasury rate change'],\n",
    "          'model8': ['Provision_Lag1', 'Real GDP growth_ema3', 'Market Volatility Index change', 'BBB corporate yield'],\n",
    "          'model9': ['Provision_Lag1', 'Real GDP growth_ema3', 'Market Volatility Index change', '3-month Treasury rate change'],\n",
    "          'model10': ['Provision_Lag1', 'Real GDP growth_ema3', 'Market Volatility Index change', '3-month Treasury rate change', 'BBB corporate yield'],\n",
    "          'model11': ['Provision_Lag1', 'Real GDP growth_ema3', 'BBB corporate yield', '3-month Treasury rate change'],\n",
    "          'model12': ['Provision_Lag1'] + list(pca_data.columns[:-1]), #PCA components\n",
    "          'model13': ['Provision_Lag1', 'Japan bilateral dollar exchange rate (yen/USD)', \n",
    "                      'Euro area bilateral dollar exchange rate (USD/euro)',\n",
    "                      'NBER_Recession_Indicator_Peak_through_Trough', 'Commercial_Banks_Treasury_and_Agency_Securities',\n",
    "                      'Real disposable income growth', 'U.K. bilateral dollar exchange rate (USD/pound)', \n",
    "                      'Unemployment rate', 'BBB corporate yield', 'Households_Net_Worth', 'Euro area inflation', \n",
    "                      'Market Volatility Index', 'Developing Asia inflation'],\n",
    "                             \n",
    "         }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "fewer-level",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Grid to search over to be able get better test results and reduce overfitting\n",
    "\n",
    "# Define the parameters\n",
    "param_grid = {\n",
    "    'gb_model__n_estimators': [100, 200, 300, 400, 500],\n",
    "    'gb_model__learning_rate': [0.05, 0.1, 0.2, 0.3],\n",
    "    'gb_model__max_depth': [3, 4, 5]\n",
    "}\n",
    "\n",
    "\n",
    "# Initialize the grid search\n",
    "grid_search = GridSearchCV(estimator=pipeline, param_grid=param_grid, scoring='r2', cv=panel_cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4ed38a25",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "best_model_name, best_score, best_model, models_results, estimators = \\\n",
    "    search_best_model(data_set_train, models, grid_search, \n",
    "                      y_col, [col for col in result_df.columns if col.startswith('IDRSSD_')])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1c02054",
   "metadata": {},
   "source": [
    "### Results of model selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "11b9e0da",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'model4'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "8feda8d7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Provision_Lag1',\n",
       " 'Real GDP growth_ema3',\n",
       " 'Dow Jones Total Stock Market Index change']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Factors of the best model\n",
    "models[best_model_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "39198eef",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;scaler&#x27;, StandardScaler()),\n",
       "                (&#x27;gb_model&#x27;, GradientBoostingRegressor(random_state=42))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;scaler&#x27;, StandardScaler()),\n",
       "                (&#x27;gb_model&#x27;, GradientBoostingRegressor(random_state=42))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GradientBoostingRegressor</label><div class=\"sk-toggleable__content\"><pre>GradientBoostingRegressor(random_state=42)</pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('scaler', StandardScaler()),\n",
       "                ('gb_model', GradientBoostingRegressor(random_state=42))])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pipeline for the best model\n",
    "best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "a609b395",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cross-Validation R^2</th>\n",
       "      <th>Cross-Validation R^2 std</th>\n",
       "      <th>Best Hyperparameters</th>\n",
       "      <th>Cross-Validation R^2 Standard Error of the Mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>model1</th>\n",
       "      <td>0.641872</td>\n",
       "      <td>0.441640</td>\n",
       "      <td>{'gb_model__learning_rate': 0.05, 'gb_model__m...</td>\n",
       "      <td>0.197507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model2</th>\n",
       "      <td>0.652643</td>\n",
       "      <td>0.419762</td>\n",
       "      <td>{'gb_model__learning_rate': 0.05, 'gb_model__m...</td>\n",
       "      <td>0.187723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model3</th>\n",
       "      <td>0.657573</td>\n",
       "      <td>0.417645</td>\n",
       "      <td>{'gb_model__learning_rate': 0.1, 'gb_model__ma...</td>\n",
       "      <td>0.186777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model4</th>\n",
       "      <td>0.679486</td>\n",
       "      <td>0.387952</td>\n",
       "      <td>{'gb_model__learning_rate': 0.1, 'gb_model__ma...</td>\n",
       "      <td>0.173497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model5</th>\n",
       "      <td>0.637102</td>\n",
       "      <td>0.430668</td>\n",
       "      <td>{'gb_model__learning_rate': 0.1, 'gb_model__ma...</td>\n",
       "      <td>0.192601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model6</th>\n",
       "      <td>0.649688</td>\n",
       "      <td>0.429142</td>\n",
       "      <td>{'gb_model__learning_rate': 0.1, 'gb_model__ma...</td>\n",
       "      <td>0.191918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model7</th>\n",
       "      <td>0.643928</td>\n",
       "      <td>0.418659</td>\n",
       "      <td>{'gb_model__learning_rate': 0.1, 'gb_model__ma...</td>\n",
       "      <td>0.187230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model8</th>\n",
       "      <td>0.651683</td>\n",
       "      <td>0.425119</td>\n",
       "      <td>{'gb_model__learning_rate': 0.1, 'gb_model__ma...</td>\n",
       "      <td>0.190119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model9</th>\n",
       "      <td>0.635401</td>\n",
       "      <td>0.439048</td>\n",
       "      <td>{'gb_model__learning_rate': 0.05, 'gb_model__m...</td>\n",
       "      <td>0.196348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model10</th>\n",
       "      <td>0.650965</td>\n",
       "      <td>0.400298</td>\n",
       "      <td>{'gb_model__learning_rate': 0.2, 'gb_model__ma...</td>\n",
       "      <td>0.179019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model11</th>\n",
       "      <td>0.651306</td>\n",
       "      <td>0.414367</td>\n",
       "      <td>{'gb_model__learning_rate': 0.05, 'gb_model__m...</td>\n",
       "      <td>0.185311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model12</th>\n",
       "      <td>0.632300</td>\n",
       "      <td>0.447186</td>\n",
       "      <td>{'gb_model__learning_rate': 0.1, 'gb_model__ma...</td>\n",
       "      <td>0.199987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model13</th>\n",
       "      <td>0.646419</td>\n",
       "      <td>0.436056</td>\n",
       "      <td>{'gb_model__learning_rate': 0.05, 'gb_model__m...</td>\n",
       "      <td>0.195010</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Cross-Validation R^2  Cross-Validation R^2 std  \\\n",
       "model1               0.641872                  0.441640   \n",
       "model2               0.652643                  0.419762   \n",
       "model3               0.657573                  0.417645   \n",
       "model4               0.679486                  0.387952   \n",
       "model5               0.637102                  0.430668   \n",
       "model6               0.649688                  0.429142   \n",
       "model7               0.643928                  0.418659   \n",
       "model8               0.651683                  0.425119   \n",
       "model9               0.635401                  0.439048   \n",
       "model10              0.650965                  0.400298   \n",
       "model11              0.651306                  0.414367   \n",
       "model12              0.632300                  0.447186   \n",
       "model13              0.646419                  0.436056   \n",
       "\n",
       "                                      Best Hyperparameters  \\\n",
       "model1   {'gb_model__learning_rate': 0.05, 'gb_model__m...   \n",
       "model2   {'gb_model__learning_rate': 0.05, 'gb_model__m...   \n",
       "model3   {'gb_model__learning_rate': 0.1, 'gb_model__ma...   \n",
       "model4   {'gb_model__learning_rate': 0.1, 'gb_model__ma...   \n",
       "model5   {'gb_model__learning_rate': 0.1, 'gb_model__ma...   \n",
       "model6   {'gb_model__learning_rate': 0.1, 'gb_model__ma...   \n",
       "model7   {'gb_model__learning_rate': 0.1, 'gb_model__ma...   \n",
       "model8   {'gb_model__learning_rate': 0.1, 'gb_model__ma...   \n",
       "model9   {'gb_model__learning_rate': 0.05, 'gb_model__m...   \n",
       "model10  {'gb_model__learning_rate': 0.2, 'gb_model__ma...   \n",
       "model11  {'gb_model__learning_rate': 0.05, 'gb_model__m...   \n",
       "model12  {'gb_model__learning_rate': 0.1, 'gb_model__ma...   \n",
       "model13  {'gb_model__learning_rate': 0.05, 'gb_model__m...   \n",
       "\n",
       "         Cross-Validation R^2 Standard Error of the Mean  \n",
       "model1                                          0.197507  \n",
       "model2                                          0.187723  \n",
       "model3                                          0.186777  \n",
       "model4                                          0.173497  \n",
       "model5                                          0.192601  \n",
       "model6                                          0.191918  \n",
       "model7                                          0.187230  \n",
       "model8                                          0.190119  \n",
       "model9                                          0.196348  \n",
       "model10                                         0.179019  \n",
       "model11                                         0.185311  \n",
       "model12                                         0.199987  \n",
       "model13                                         0.195010  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models_results['Cross-Validation R^2 Standard Error of the Mean'] = \\\n",
    "    models_results['Cross-Validation R^2 std'] / panel_cv.get_n_splits()**0.5\n",
    "\n",
    "models_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "c9454db5-2c9e-4b9d-96e1-061f4ff032fe",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "models_results.to_csv('GB_models_results.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "106fc6fc",
   "metadata": {},
   "source": [
    "## Final model after hyperparameter tuning: Test sample performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "9345420e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Best model features\n",
    "features_all = models[best_model_name] + [col for col in result_df.columns if col.startswith('IDRSSD_')]\n",
    "\n",
    "X_train = data_set_train[features_all]\n",
    "X_test = data_set_test[features_all]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "43ae0648",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Fit the best model using the whole train set\n",
    "best_model.fit(X_train, y_train)\n",
    "y_pred = best_model.predict(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9924b59d-a7b5-49a3-8ede-f8d228a84afc",
   "metadata": {},
   "source": [
    "### Train sample performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "096172e7-b542-43e6-a630-daa725672304",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>measure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>R squared</th>\n",
       "      <td>0.882779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RMSE</th>\n",
       "      <td>0.297373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>median relative error, %</th>\n",
       "      <td>17.976709</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            measure\n",
       "R squared                  0.882779\n",
       "RMSE                       0.297373\n",
       "median relative error, %  17.976709"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimate_errors(y_train, y_pred, lower_limit, upper_limit)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "394d848b-1fb4-48e6-abc6-d568947c1896",
   "metadata": {},
   "source": [
    "### Test sample performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "42fcc8c1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "y_pred = best_model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "04f03e93",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>measure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>R squared</th>\n",
       "      <td>0.821973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RMSE</th>\n",
       "      <td>0.221627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>median relative error, %</th>\n",
       "      <td>40.290357</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            measure\n",
       "R squared                  0.821973\n",
       "RMSE                       0.221627\n",
       "median relative error, %  40.290357"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# All errors together (excluding outliers)\n",
    "estimate_errors(y_test, y_pred, lower_limit, upper_limit)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}