Tabular/2MAR23_Playground Series 3_8.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5a93b098-43cc-4efe-9fef-1947f6732c4d",
   "metadata": {},
   "source": [
    "#### Install and import neccesary packages to the the environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "L0PWpLd57OkE",
   "metadata": {
    "id": "L0PWpLd57OkE",
    "tags": []
   },
   "outputs": [],
   "source": [
    "! pip install mamba\n",
    "! mamba install optuna textstat category_encoders lightgbm plotly -y\n",
    "! mamba install -c conda-forge pandas-profiling -y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70b65a7e-54aa-414c-974a-02b4aacb0869",
   "metadata": {
    "id": "70b65a7e-54aa-414c-974a-02b4aacb0869",
    "tags": []
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import os, random, optuna, textstat, multiprocessing, re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas.api.types import CategoricalDtype\n",
    "from pandas_profiling import ProfileReport\n",
    "\n",
    "from scipy.stats import mode\n",
    "from sklearn.model_selection import cross_validate, KFold, RepeatedKFold, StratifiedKFold, RepeatedStratifiedKFold, train_test_split\n",
    "from sklearn.preprocessing import FunctionTransformer, StandardScaler, RobustScaler\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error, balanced_accuracy_score, silhouette_score\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import IterativeImputer, SimpleImputer, KNNImputer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.neighbors import NearestNeighbors, LocalOutlierFactor\n",
    "from sklearn.metrics import accuracy_score, mean_squared_error\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.cluster import KMeans, OPTICS, MiniBatchKMeans\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.feature_selection import mutual_info_regression\n",
    "from sklearn.linear_model import LinearRegression, LogisticRegression\n",
    "from sklearn.base import TransformerMixin, BaseEstimator\n",
    "\n",
    "import category_encoders as ce\n",
    "\n",
    "import lightgbm as lgb\n",
    "from lightgbm import LGBMClassifier, LGBMRegressor, early_stopping, Dataset\n",
    "\n",
    "import seaborn as sns\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "plt.style.use(\"seaborn-whitegrid\")\n",
    "plt.rc(\"figure\", autolayout=True)\n",
    "plt.rc(\n",
    "    \"axes\",\n",
    "    labelweight=\"bold\",\n",
    "    labelsize=\"large\",\n",
    "    titleweight=\"bold\",\n",
    "    titlesize=14,\n",
    "    titlepad=10,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a98585f8-fe02-4730-911d-4f71a2d04f1c",
   "metadata": {},
   "source": [
    "#### Import the Data\n",
    "\n",
    "__note__: depending on the data source, this may require additional packages being installed and/or imported"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83565d2f-3b23-43e6-b641-ce8d14e0b1af",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "! mamba install -c conda-forge kaggle -y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5148253d-b501-4b8e-bfe1-7e9c8f1528e8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "! kaggle competitions download -c playground-series-s3e8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8037873b-607e-4583-a62c-9f6262bce7ea",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "! unzip playground-series-s3e8.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "944b55af-350e-4d09-84ec-65b6c9b90c5e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(\"train.csv\")\n",
    "test_df = pd.read_csv(\"test.csv\")\n",
    "df= pd.concat([train_df, test_df])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef781d9b-5ae3-479e-82a1-466ef7749457",
   "metadata": {
    "id": "ef781d9b-5ae3-479e-82a1-466ef7749457"
   },
   "source": [
    "# 1. Exploratory Data Analysis\n",
    "- look at the dataset basics (size of the data, data types, look at a few examples etc.). \n",
    "    - You should also note elements of the data collection, which might be important for developing models on the data. An example would be if all of the data samples come from a handful of sensors. In such a case you may want to consider using a `GroupKFold` split with the senors being the croups for cross-validation.\n",
    "- look for any missing data\n",
    "- look at the target value\n",
    "- look for any outliers in the data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72605f49-bc1f-4971-96ae-3d32f6ca1ab1",
   "metadata": {
    "id": "72605f49-bc1f-4971-96ae-3d32f6ca1ab1"
   },
   "source": [
    "## 1(a) Profile the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fa302a1-7ccc-48ce-8f3f-0257f0d1474c",
   "metadata": {
    "id": "5fa302a1-7ccc-48ce-8f3f-0257f0d1474c",
    "tags": []
   },
   "outputs": [],
   "source": [
    "display(df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e18458b4-9d3a-4a13-9316-62988203b35b",
   "metadata": {
    "id": "e18458b4-9d3a-4a13-9316-62988203b35b",
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(\"df shape:\", df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86a57a89-8495-47e9-b4ce-3069ac37ca1c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(\"train shape {}, test_shape: {}\".format(train_df.shape, test_df.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "xedfRci88-9O",
   "metadata": {
    "id": "xedfRci88-9O",
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f96c5dbc-e0b7-4cd2-beb0-19dbd97c000a",
   "metadata": {
    "id": "f96c5dbc-e0b7-4cd2-beb0-19dbd97c000a",
    "tags": []
   },
   "outputs": [],
   "source": [
    "ProfileReport(train_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ebbaa82f-aa21-4efe-9141-33a03d7a4be9",
   "metadata": {
    "id": "ebbaa82f-aa21-4efe-9141-33a03d7a4be9"
   },
   "source": [
    "From the initial dataset profiling:\n",
    "\n",
    "- We have no missing data that we need to account for\n",
    "- This is a regression problem with a skewed target variable\n",
    "- There don't appear to be any interactions\n",
    "- the target variable of price strongly correlates with the x,y,z and carat, which correlate with each other. These are size elements of the gemstones\n",
    "- we have two categorical features of `color` and `clarity`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8938819a-3feb-45b3-8909-75b70f1c7490",
   "metadata": {
    "id": "8938819a-3feb-45b3-8909-75b70f1c7490"
   },
   "source": [
    "## 1(b) look at the missing values\n",
    "- look for any patterns in missing data\n",
    "- look at some examples of missing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1ddb092-f251-4d4e-88fa-70512384f001",
   "metadata": {
    "id": "d1ddb092-f251-4d4e-88fa-70512384f001",
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1fdbc8fa-537c-4923-85b8-a93427697c84",
   "metadata": {
    "id": "1fdbc8fa-537c-4923-85b8-a93427697c84"
   },
   "source": [
    "## 1(c) Look at the target variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ac499ab-5242-4716-9a3f-39933a3c2282",
   "metadata": {
    "id": "1ac499ab-5242-4716-9a3f-39933a3c2282",
    "tags": []
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,8))\n",
    "sns.histplot(df['price'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f8f6605-a255-471c-a897-774f867054cb",
   "metadata": {
    "id": "6f8f6605-a255-471c-a897-774f867054cb"
   },
   "source": [
    "## 1(d) Look for Outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "Sdy6bX2q_oHp",
   "metadata": {
    "id": "Sdy6bX2q_oHp",
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "sJN9lcuUA3qL",
   "metadata": {
    "id": "sJN9lcuUA3qL",
    "tags": []
   },
   "outputs": [],
   "source": [
    "variable ='carat'\n",
    "\n",
    "outlier_df = df[(df[variable] > df[variable].mean()+3*df[variable].std()) | (df[variable] < df[variable].mean()-3*df[variable].std())]\n",
    "outlier_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3390fe3b-6466-4296-ac9e-28858d7555e5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "outlier_df['cut'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8339136c-42ae-45fc-bd16-cae097d8c49d",
   "metadata": {
    "id": "8339136c-42ae-45fc-bd16-cae097d8c49d",
    "tags": []
   },
   "source": [
    "# 2. Import and Preprocess Data\n",
    "- Import some helper functions to do imputation and deal with data outliers\n",
    "- Use a function to do the data import and cleaning, so that it can be done as a repeatable process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "870635a6-ee2c-4d40-918d-b756048676af",
   "metadata": {
    "id": "870635a6-ee2c-4d40-918d-b756048676af",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def simple_impute(df):\n",
    "    '''\n",
    "    Impute missing values in a DataFrame. Impute the numerical columns by the median value for each column and\n",
    "    impute the categorical columns by the most frequent, or mode, for each column\n",
    "    Note: one can easily switch in different imputers for each of the data types to something like kNN or iterative\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df : pandas DataFrame\n",
    "        Input DataFrame containing missing values.\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    imp_df : pandas DataFrame\n",
    "        DataFrame with imputed missing values.\n",
    "    '''\n",
    "    \n",
    "    df= df.copy()\n",
    "    # Impute missing values for numerical data\n",
    "    # imp_num = IterativeImputer(estimator=ExtraTreesRegressor(), initial_strategy='median', max_iter=20)\n",
    "    imp_num = SimpleImputer(strategy='median')\n",
    "    numerical_df = df.select_dtypes(\"number\")\n",
    "    numerical_df = pd.DataFrame(data=imp_num.fit_transform(numerical_df), index=df.index, columns =numerical_df.columns)\n",
    "    \n",
    "    if df.select_dtypes(\"category\").shape[1] >0:\n",
    "        # Imput missing values for categorical data\n",
    "        # imp_cat = IterativeImputer(estimator=ExtraTreesClassifier(), initial_strategy='most_frequent', max_iter=20)\n",
    "        imp_cat = SimpleImputer(strategy='most_frequent')\n",
    "        categorical_df = df.select_dtypes(\"category\")\n",
    "        enc = OrdinalEncoder()\n",
    "        categorical_df = pd.DataFrame(data=enc.fit_transform(categorical_df), columns=categorical_df.columns)\n",
    "        categorical_imputations = enc.inverse_transform(imp_cat.fit_transform(categorical_df))\n",
    "        categorical_df = pd.DataFrame(data=categorical_imputations, index=df.index, columns =categorical_df.columns, dtype=\"category\")\n",
    "        return categorical_df.join(numerical_df).reindex(columns= df.columns)\n",
    "    else:\n",
    "        return numerical_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8547868-cf34-4b06-9523-63bfec5b8364",
   "metadata": {
    "id": "e8547868-cf34-4b06-9523-63bfec5b8364",
    "tags": []
   },
   "outputs": [],
   "source": [
    "class ML_Impute(TransformerMixin):\n",
    "    '''\n",
    "    Impute missing values by treating the imputational as a machine learning problem. For numerical\n",
    "    columns, we can treat the problem as a regression problem, and for categorical, a classification problem.\n",
    "    For this method, we'll iterate through all of the columns with one column being the target variable\n",
    "    and the others as being predictor variables\n",
    "    '''\n",
    "\n",
    "    def __init__(self, params={}):\n",
    "        self.params = {}\n",
    "        self.models = {}\n",
    "\n",
    "    def fit(self, df):\n",
    "        df = df.copy()\n",
    "        # label encode categorical variables\n",
    "        columns = df.columns.to_list()\n",
    "        cat_cols = df.select_dtypes(\"category\").columns.to_list()\n",
    "        self.enc = OrdinalEncoder()\n",
    "        df[cat_cols] = self.enc.fit_transform(df[cat_cols])\n",
    "\n",
    "        # Randomized column selection\n",
    "        for i in random.sample(range(len(df.columns)), len(df.columns)):\n",
    "\n",
    "        # Starting with most null values to least\n",
    "        # for i in np.argsort(-df.isnull().sum().values):\n",
    "            column = columns[i]\n",
    "            # Check to make sure there are null values that need to be imputed\n",
    "            if not df[column].isnull().any():\n",
    "                continue\n",
    "\n",
    "            print(\"Creating Imputation Model for Column: {}\".format(column))\n",
    "\n",
    "            # Create train, test, and validation data using the null values of the column of interest\n",
    "            X= df.loc[df[column].notnull()]\n",
    "            y = X.pop(column)\n",
    "            X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1)\n",
    "\n",
    "            X_test = df.loc[df[column].isnull()]\n",
    "            _ = X_test.pop(column)\n",
    "\n",
    "            # If we have more data, we use more estimators for the imputation model\n",
    "            n_estimators = min(100, int(len(X_train) / 10))\n",
    "            if column in cat_cols:\n",
    "                model = LGBMClassifier(**self.params, verbose=-1, n_estimators=n_estimators)\n",
    "            else:\n",
    "                model = LGBMRegressor(**self.params, verbose=-1, n_estimators=n_estimators)\n",
    "\n",
    "            model.fit(X_train, y_train, eval_set=[(X_valid, y_valid)],\n",
    "                      callbacks = [early_stopping(min(20, int(len(X_train) / 10)))])\n",
    "            print(\"Score of Column {} is {}\".format(column, model.score(X,y)))\n",
    "            self.models[column] = model \n",
    "\n",
    "    def transform(self, df):\n",
    "        df = df.copy()\n",
    "\n",
    "        # label encode categorical variables\n",
    "        columns = df.columns.to_list()\n",
    "        cat_cols = df.select_dtypes(\"category\").columns.to_list()\n",
    "        df[cat_cols] = self.enc.transform(df[cat_cols])\n",
    "\n",
    "        for column in self.models.keys():\n",
    "            X_test = df.loc[df[column].isnull()]\n",
    "            _ = X_test.pop(column)\n",
    "            model = self.models[column]\n",
    "            preds =model.predict(X_test)\n",
    "            m = df[column].isna()\n",
    "            df.loc[m, column]  = preds.flatten()\n",
    "\n",
    "        if len(cat_cols) >0:\n",
    "            df[cat_cols] = self.enc.inverse_transform(df[cat_cols])\n",
    "        return df\n",
    "\n",
    "    def fit_transform(self, df):\n",
    "        self.fit(df)\n",
    "        df = self.transform(df)\n",
    "        return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6JZWFbEFB9Lo",
   "metadata": {
    "id": "6JZWFbEFB9Lo",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def cap_outliers(df, variables=[None]):\n",
    "    \"\"\"\n",
    "    Caps outlier values in a dataframe using the mean and standard deviation of each specified variable.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df : pandas DataFrame\n",
    "        The dataframe containing the data to be capped.\n",
    "    variables : list or None, optional (default=[None])\n",
    "        The list of column names to cap outlier values. If None, all columns with numerical data will be capped.\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    pandas DataFrame\n",
    "        The dataframe with capped outlier values.\n",
    "\n",
    "    Example:\n",
    "    --------\n",
    "    >>> import pandas as pd\n",
    "    >>> import numpy as np\n",
    "    >>> data = {'A': [1, 2, 3, 1000], 'B': [4, 5, 6, 2000]}\n",
    "    >>> df = pd.DataFrame(data)\n",
    "    >>> df\n",
    "          A     B\n",
    "    0     1     4\n",
    "    1     2     5\n",
    "    2     3     6\n",
    "    3  1000  2000\n",
    "    >>> cap_outliers(df, variables=['A'])\n",
    "         A     B\n",
    "    0    1     4\n",
    "    1    2     5\n",
    "    2    3     6\n",
    "    3  100    2000\n",
    "    \"\"\"\n",
    "    df = df.copy()\n",
    "    for variable in variables:\n",
    "        upper_limit = df[variable].mean() + 3*df[variable].std()\n",
    "        lower_limit = df[variable].mean() - 3*df[variable].std()\n",
    "        df[variable] = np.where(df[variable]> upper_limit, upper_limit, np.where(\n",
    "            df[variable]<lower_limit, lower_limit, df[variable]\n",
    "        ))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75ef91dd-cda0-4298-b493-0156e091ceb3",
   "metadata": {
    "id": "75ef91dd-cda0-4298-b493-0156e091ceb3",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Wrapper function to read in, encode and impute missing values for the data\n",
    "\n",
    "def preprocess_data(train_df, test_df=None, cat_features=[None], outlier_features=[None]):\n",
    "    \n",
    "    if test_df is not None:\n",
    "        df = pd.concat([train_df, test_df])\n",
    "    \n",
    "    # Specify categorical variables\n",
    "    for name in cat_features:\n",
    "        df[name] = df[name].astype(\"category\")\n",
    "        # Add a None category for missing values\n",
    "        if \"None\" not in df[name].cat.categories:\n",
    "            df[name].cat.add_categories(\"None\", inplace=True)\n",
    "            \n",
    "    '''\n",
    "    # specify data corruptions (i.e. physical quantiites being 0.0)\n",
    "    df[['x']] = df[['x']].where(df['x'] != 0, pd.NA)\n",
    "    df[['y']] = df[['y']].where(df['y'] != 0, pd.NA)\n",
    "    df[['z']] = df[['z']].where(df['z'] != 0, pd.NA)\n",
    "    '''\n",
    "            \n",
    "    # Reform splits or create test data set, if applicable\n",
    "    if test_df is not None:\n",
    "        test = df.loc[test_df.index, :]\n",
    "        train = df.drop(test_df.index)\n",
    "    else:\n",
    "        train, test = train_test_split(df, test_size=0.2)\n",
    "\n",
    "    # Preprocessing for missing data and outliers\n",
    "    imputer = ML_Impute()\n",
    "    train = imputer.fit_transform(train)\n",
    "    train = cap_outliers(train, outlier_features)\n",
    "\n",
    "    test = imputer.transform(test)\n",
    "    test = cap_outliers(test, outlier_features)\n",
    "\n",
    "\n",
    "    return train, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2befbe86-0b38-4884-8dd5-281d092df517",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Now, load in and preprocess the data\n",
    "\n",
    "train_df = pd.read_csv(\"train.csv\", index_col='id')\n",
    "test_df = pd.read_csv(\"test.csv\", index_col='id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be2ee427-5728-4bfe-95ca-7e977fb891be",
   "metadata": {
    "id": "be2ee427-5728-4bfe-95ca-7e977fb891be",
    "tags": []
   },
   "outputs": [],
   "source": [
    "target_var = \"price\"\n",
    "\n",
    "train, test = preprocess_data(train_df, test_df=test_df, cat_features = ['color', 'clarity', 'cut'], outlier_features=[])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2dc8ec75-2575-441f-bf28-5ff9664fc39b",
   "metadata": {
    "id": "2dc8ec75-2575-441f-bf28-5ff9664fc39b"
   },
   "source": [
    "# 3. Create a Baseline\n",
    "- specify a baseline scoring function\n",
    "    - __note__: the cross-validation for the scoring should be consistent with observations from the EDA\n",
    "- create a baseline model. \n",
    "  - For regression use `LGBMRegressor` and `scoring='neg_root_mean_squared_error'` and multiply the result by `-1`\n",
    "  - For classification use `LGBMClassifier` and `scoring='balanced_accuracy'`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65ecf5d3-860b-456a-b86a-678de670630e",
   "metadata": {
    "id": "65ecf5d3-860b-456a-b86a-678de670630e",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def score_dataset(X, y, \n",
    "                  model=LGBMRegressor(n_estimators=1000, verbose=-1, random_state=42)\n",
    "                 ):\n",
    "    \n",
    "    \n",
    "    scores = cross_validate(\n",
    "        model, X, y, cv=StratifiedKFold(n_splits=5).split(X, X['cut'].astype(str)+\"_\"+X[\"color\"].astype(str)+\"_\"+X['clarity'].astype(str)), \n",
    "        n_jobs=-1, scoring='neg_root_mean_squared_error', return_train_score=True\n",
    "    )\n",
    "    \n",
    "    return {\"Training\":-1*np.mean(scores[\"train_score\"]), \"Validation\":-1*np.mean(scores[\"test_score\"])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdb0b75e-9d33-4927-9ea8-eb3118f04d34",
   "metadata": {
    "id": "fdb0b75e-9d33-4927-9ea8-eb3118f04d34",
    "tags": []
   },
   "outputs": [],
   "source": [
    "X = train.copy()\n",
    "y = X.pop(target_var)\n",
    "X = ce.OrdinalEncoder().fit_transform(X)\n",
    "\n",
    "score_dataset(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e295babf-8d18-443b-abb7-fea6222be667",
   "metadata": {
    "id": "e295babf-8d18-443b-abb7-fea6222be667",
    "tags": []
   },
   "source": [
    "# 4. Featurize the Data\n",
    "\n",
    "__Note:__ For featurizing data, really be creative. Most especially, use your subject matter expertise or domain knowledge to create features \n",
    "\n",
    "#### Tips on Creating Features\n",
    "It's good to keep in mind your model's own strengths and weaknesses when creating features. Here are some guidelines:\n",
    "\n",
    "- Linear models learn sums and differences naturally, but can't learn anything more complex.\n",
    "- Ratios seem to be difficult for most models to learn. Ratio combinations often lead to some easy performance gains.\n",
    "- Linear models and neural nets generally do better with normalized features. Neural nets especially need features scaled to values not too far from 0. Tree-based models (like random forests and XGBoost) can sometimes benefit from normalization, but usually much less so.\n",
    "- Tree models can learn to approximate almost any combination of features, but when a combination is especially important they can still benefit from having it explicitly created, especially when data is limited.\n",
    "- Counts are especially helpful for tree models, since these models don't have a natural way of aggregating information across many features at once.\n",
    "\n",
    "Example feature engineering steps:\n",
    "- Remove uniformative features\n",
    "- Create interactions\n",
    "- Binning\n",
    "- Indicate Outliers\n",
    "- Try different encodings for categorical variables (https://contrib.scikit-learn.org/category_encoders/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37e9fb69-5bf6-4a1a-aed2-d6208c584516",
   "metadata": {
    "id": "37e9fb69-5bf6-4a1a-aed2-d6208c584516"
   },
   "outputs": [],
   "source": [
    "def make_mi_scores(X, y):\n",
    "    X = X.copy()\n",
    "    for colname in X.select_dtypes([\"object\", \"category\"]):\n",
    "        X[colname], _ = X[colname].factorize()\n",
    "    # All discrete features should now have integer dtypes\n",
    "    discrete_features = [pd.api.types.is_integer_dtype(t) for t in X.dtypes]\n",
    "    mi_scores = mutual_info_regression(X, y, discrete_features=discrete_features, random_state=0)\n",
    "    mi_scores = pd.Series(mi_scores, name=\"MI Scores\", index=X.columns)\n",
    "    mi_scores = mi_scores.sort_values(ascending=False)\n",
    "    return mi_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "_bErwNVFH_a5",
   "metadata": {
    "id": "_bErwNVFH_a5"
   },
   "outputs": [],
   "source": [
    "X = train.copy()\n",
    "y = X.pop(target_var)\n",
    "mi_scores = make_mi_scores(X, y)\n",
    "mi_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9af24a6b-0233-42b1-953e-e142b5cf8a6c",
   "metadata": {
    "id": "9af24a6b-0233-42b1-953e-e142b5cf8a6c"
   },
   "outputs": [],
   "source": [
    "# Try removing some of the uninformative features to see if that improves scores\n",
    "uninformative_features = [\n",
    "'cut', 'depth'\n",
    "]\n",
    "\n",
    "X = train.copy()\n",
    "y = X.pop(target_var)\n",
    "X = X.loc[:,~X.columns.isin(uninformative_features)]\n",
    "X = ce.OrdinalEncoder().fit_transform(X)\n",
    "\n",
    "score_dataset(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "EPCRuCtzJ7uS",
   "metadata": {
    "id": "EPCRuCtzJ7uS"
   },
   "source": [
    "For mathematical transforms, try things like log transforms:\n",
    "\n",
    "`X['feature'] = np.log1p(X['feature'])`\n",
    "\n",
    "For interations try multiplying or dividing features, especially between levels\n",
    "of a categorical feature and a continuous feature. Use subject matter expertise here. Often, interactions between levels of a categorical variable and values of numerical variable make for good interactions\n",
    "\n",
    "`X = X.join(ce.OneHotEncoder().fit_transform(df['A']).multiply(df['B'], axis='index'), rsuffix='A_B_interation')`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "QaC0WBgFJ5Nf",
   "metadata": {
    "id": "QaC0WBgFJ5Nf",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def domain_transforms(df):\n",
    "    X = pd.DataFrame(index=df.index)\n",
    "    \n",
    "    X['volume'] = df['x'] * df['y'] * df['z']\n",
    "    \n",
    "    X['clarity_letter'] = df['clarity'].str.replace('\\d+', '')\n",
    "    \n",
    "    X['clarity_number'] = df['clarity'].str.extract('(\\d+)', expand=False)\n",
    "    \n",
    "    X['density'] = df['carat'].divide(X['volume'] + 1e-6, axis=0)\n",
    "    \n",
    "    X['carat_depth_ratio'] = df['carat'].divide(df['depth'], axis=0)\n",
    "    \n",
    "    X['carat_table_ratio'] = df['carat'].divide(df['table'], axis=0)\n",
    "    \n",
    "    X['depth_per_volume'] = df['depth'].divide(X['volume'], axis=0)\n",
    "    \n",
    "    X['depth_per_volume'] = df['depth'].divide(X['density'], axis=0)\n",
    "    \n",
    "    X['depth_table_ratio'] = df['depth'].divide(df['table'], axis=0)\n",
    "    \n",
    "    X['symmetry'] = (abs(df['x'] - df['z']) + abs(df['y'] - df['z'])) / (df['x'] + df['y'] + df['z']+ 1e-6)\n",
    "    \n",
    "    X['surface_area'] = 2 * ((df['x'] * df['y']) + (df['x'] * df['z']) + (df['y'] * df['z']))\n",
    "    \n",
    "    grouped_var = df[['cut', 'clarity', 'color', 'carat']].groupby(['cut', 'clarity', 'color'], as_index=False).agg(carat_mean=('carat', np.mean), carat_std=('carat', np.std))\n",
    "    \n",
    "    X[['carat_mean', 'carat_std']] = df.merge(grouped_var, on=['cut', 'clarity', 'color'], how='left')[['carat_mean', 'carat_std']]\n",
    "\n",
    "    return X\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8f87534-ff38-4f56-80ea-9f6c2a7bea11",
   "metadata": {
    "id": "vl6bFWROKs-3",
    "tags": []
   },
   "outputs": [],
   "source": [
    "X = train.copy()\n",
    "y = X.pop(target_var)\n",
    "X = X.join(domain_transforms(X))\n",
    "X = ce.OrdinalEncoder().fit_transform(X)\n",
    "\n",
    "score_dataset(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3923b19-f7c6-4b2a-906c-133e17a050db",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def interaction_transforms(df):\n",
    "    X = pd.DataFrame(index=df.index)\n",
    "    \n",
    "    interaction = ce.OneHotEncoder().fit_transform(df['cut']).multiply(df['carat'], axis=0)\n",
    "    interaction.columns = [i+'_cut_carat_interation' for i in interaction.columns]\n",
    "    X = X.join(interaction)\n",
    "\n",
    "    # Note, the following requires running domain_transform first\n",
    "    interaction = ce.OneHotEncoder().fit_transform(df['cut']).multiply(df['volume'], axis=0)\n",
    "    interaction.columns = [i+'_cut_volume_interation' for i in interaction.columns]\n",
    "    X = X.join(interaction)\n",
    "    \n",
    "    interaction = ce.OneHotEncoder().fit_transform(df['clarity_letter']).multiply(df['volume'], axis=0)\n",
    "    interaction.columns = [i+'_clarity_letter_volume_interation' for i in interaction.columns]\n",
    "    X = X.join(interaction)\n",
    "\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "414b5924-9de1-4723-9ef5-fbdefea5c7cd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "X = train.copy()\n",
    "y = X.pop(target_var)\n",
    "X = X.join(domain_transforms(X))\n",
    "X = X.join(interaction_transforms(X))\n",
    "X = ce.OrdinalEncoder().fit_transform(X)\n",
    "\n",
    "score_dataset(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "Ipv16eN3K5j0",
   "metadata": {
    "id": "Ipv16eN3K5j0"
   },
   "source": [
    "Notes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "C_S9gZfYIiZ4",
   "metadata": {
    "id": "C_S9gZfYIiZ4",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def cluster_labels(df, features, n_clusters=10):\n",
    "    X = df.copy()\n",
    "    X_scaled = X.loc[:, features]\n",
    "    X_scaled = ce.OneHotEncoder().fit_transform(X_scaled)\n",
    "    scaler = RobustScaler()\n",
    "    X_scaled = scaler.fit_transform(X_scaled.to_numpy())\n",
    "    kmeans = MiniBatchKMeans(n_clusters=n_clusters, batch_size=4096, n_init='auto')\n",
    "    X_new = pd.DataFrame(index=X.index)\n",
    "    X_new[\"Cluster\"] = kmeans.fit_predict(X_scaled)\n",
    "    X_new[\"Cluster\"] = X_new[\"Cluster\"].astype(\"category\")\n",
    "    #print(\"Silhouette Score: {}\".format(silhouette_score(X_scaled, X_new[\"Cluster\"])))\n",
    "    return X_new[\"Cluster\"]\n",
    "\n",
    "\n",
    "def cluster_distance(df, features, n_clusters=10):\n",
    "    X = df.copy()\n",
    "    X_scaled = X.loc[:, features]\n",
    "    X_scaled = ce.OneHotEncoder().fit_transform(X_scaled)\n",
    "    scaler = RobustScaler()\n",
    "    X_scaled = scaler.fit_transform(X_scaled.to_numpy())\n",
    "    kmeans = MiniBatchKMeans(n_clusters=n_clusters, batch_size=4096, n_init='auto')\n",
    "    X_cd = kmeans.fit_transform(X_scaled)\n",
    "    #print(\"Silhouette Score: {}\".format(silhouette_score(X_scaled, kmeans.predict(X_scaled))))\n",
    "    # Label features and join to dataset\n",
    "    X_cd = pd.DataFrame(\n",
    "        X_cd, columns=[f\"Centroid_{i}\" for i in range(X_cd.shape[1])], index=X.index\n",
    "    )\n",
    "    return X_cd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0aef44b9-c6b2-45e4-8b22-af16dc1564ff",
   "metadata": {},
   "source": [
    "Try adjusting what features you cluster together, `cluster_features`, as well as the number of clusters `n_clusters`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1xBut4HcIqCM",
   "metadata": {
    "id": "1xBut4HcIqCM",
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster_features = [\n",
    "    'carat', 'volume', 'x', 'y', 'z', 'table', 'depth'\n",
    "]\n",
    "\n",
    "n_clusters = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "jTiUmSfUJKvB",
   "metadata": {
    "id": "jTiUmSfUJKvB",
    "tags": []
   },
   "outputs": [],
   "source": [
    "X = train.copy()\n",
    "y = X.pop(target_var)\n",
    "X = X.join(domain_transforms(X))\n",
    "X = X.join(cluster_labels(X, cluster_features, n_clusters=n_clusters))\n",
    "X = ce.OrdinalEncoder().fit_transform(X)\n",
    "\n",
    "score_dataset(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "wRTQdmVcIqjG",
   "metadata": {
    "id": "wRTQdmVcIqjG",
    "tags": []
   },
   "outputs": [],
   "source": [
    "X = train.copy()\n",
    "y = X.pop(target_var)\n",
    "X = X.join(domain_transforms(X))\n",
    "X = X.join(cluster_distance(X, cluster_features, n_clusters=n_clusters))\n",
    "X = ce.OrdinalEncoder().fit_transform(X)\n",
    "\n",
    "score_dataset(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "l6jPeNQpLIPO",
   "metadata": {
    "id": "l6jPeNQpLIPO"
   },
   "source": [
    "Notes:\n",
    "- 'cut', 'color', 'clarity', 'carat' (n=10)\n",
    "    - labels: SS: 0.19963, T: 469.95, V: 591.745\n",
    "    - distance: SS: 0.2676, T: 456.2513, V: 592.71927\n",
    "- 'cut', 'color', 'clarity', 'carat' (n=5)\n",
    "    - labels: SS: 0.0998, T: 469.706, V: 592.104\n",
    "    - distance: SS: 0.16608, T: 458.3802, V: 592.2948\n",
    "- 'cut', 'color', 'clarity' (n=12):\n",
    "    - labels: SS: 0.2381, T: 468.095, V: 591.112\n",
    "    - distance: SS: 0.22733, T: 457.3325, V: 592.075\n",
    "- 'cut', 'color', 'clarity', 'clarity_letter', 'clarity_number' (n=10):\n",
    "    - labels: SS: 0.267 , T: 441.258, V: 591.8199\n",
    "    - distance: SS: 0.256, T: 431.6164, V: 592.4309 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "uJPD5PVzLJpX",
   "metadata": {
    "id": "uJPD5PVzLJpX",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def flag_outliers(df):\n",
    "    df = df.copy()\n",
    "    df = ce.OneHotEncoder().fit_transform(df)\n",
    "    clf = LocalOutlierFactor(n_neighbors = int(np.floor(np.log2(len(X)))))\n",
    "    df[\"Outlier\"] = clf.fit_predict(df)\n",
    "    return df[\"Outlier\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "yorht94RM1X9",
   "metadata": {
    "id": "yorht94RM1X9",
    "tags": []
   },
   "outputs": [],
   "source": [
    "X = train.copy()\n",
    "y = X.pop(target_var)\n",
    "X = X.join(domain_transforms(X))\n",
    "X = X.join(flag_outliers(X))\n",
    "X = ce.OrdinalEncoder().fit_transform(X)\n",
    "\n",
    "score_dataset(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "JdR7Quw_NRF9",
   "metadata": {
    "id": "JdR7Quw_NRF9"
   },
   "source": [
    "Notes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "279fc029-6fa9-43f1-b3a2-1b846e5987fe",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def add_count_features(df, cols=[]):\n",
    "    count_encoder = ce.count.CountEncoder(cols=cols)\n",
    "    count_df = count_encoder.fit_transform(df)\n",
    "    count_df = count_df[cols]\n",
    "    count_df.columns = [i+\"_counts\" for i in count_df.columns]\n",
    "    return count_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bcd1d02-1f62-448c-8865-91b20f72686d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "X = train.copy()\n",
    "y = X.pop(target_var)\n",
    "X = X.join(domain_transforms(X))\n",
    "X = X.join(add_count_features(X, cols=['color', 'clarity', 'cut', 'clarity_letter']))\n",
    "X = ce.OrdinalEncoder().fit_transform(X)\n",
    "\n",
    "score_dataset(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df013cef-dabe-4cba-8ce2-f221ae20050e",
   "metadata": {},
   "source": [
    "Try out different catgeorical encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33d1bc81-018c-410f-8681-2665d60ecf9a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "X = train.copy()\n",
    "y = X.pop(target_var)\n",
    "X = X.join(domain_transforms(X))\n",
    "X = X.join(add_count_features(X, cols=['color', 'clarity', 'cut', 'clarity_letter']))\n",
    "cat_encoder_1 = ce.m_estimate.MEstimateEncoder(cols=['cut', 'color', 'clarity', 'clarity_letter'])\n",
    "#cat_encoder_1 = ce.glmm.GLMMEncoder(cols=['cut', 'color', 'clarity', 'clarity_letter'])\n",
    "cat_encoder_2 = ce.OrdinalEncoder()\n",
    "X = cat_encoder_1.fit_transform(X, y)\n",
    "X = cat_encoder_2.fit_transform(X)\n",
    "\n",
    "score_dataset(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78600898-c660-4b0f-99a9-39b7a773ad5e",
   "metadata": {
    "id": "78600898-c660-4b0f-99a9-39b7a773ad5e"
   },
   "source": [
    "# 5. Finalize Features for Final Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6adc8836-a8ea-4cfa-96f1-0ea73e59328d",
   "metadata": {
    "id": "6adc8836-a8ea-4cfa-96f1-0ea73e59328d",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_features(df, target_var, test_df=None):\n",
    "    X = df.copy()\n",
    "    y = X.pop(target_var)\n",
    "    \n",
    "    if test_df is not None:\n",
    "        X_test = test_df.copy()\n",
    "        if target_var in X_test.columns:\n",
    "            y_test = X_test.pop(target_var)\n",
    "        else:\n",
    "            y_test = None\n",
    "        X = pd.concat([X, X_test])\n",
    "        \n",
    "    # Add in engineered features\n",
    "    cluster_features = [\n",
    "        'carat', 'volume', 'x', 'y', 'z', 'table', 'depth'\n",
    "    ]\n",
    "\n",
    "    n_clusters = 10\n",
    "    \n",
    "    X = X.join(domain_transforms(X))\n",
    "    X = X.join(add_count_features(X, cols=['color', 'clarity', 'cut', 'clarity_letter']))\n",
    "    X = X.join(cluster_labels(X, cluster_features, n_clusters=n_clusters))\n",
    "\n",
    "    # Reform splits\n",
    "    if test_df is not None:\n",
    "        X_test = X.loc[test_df.index, :]\n",
    "        X.drop(test_df.index, inplace=True)\n",
    "        \n",
    "    # Do final encoding of categorical features on Train\n",
    "    # cat_encoder_1 = ce.glmm.GLMMEncoder(cols=['cut', 'color', 'clarity', 'clarity_letter'])\n",
    "    cat_encoder_2 = ce.OrdinalEncoder()\n",
    "    # X = cat_encoder_1.fit_transform(X, y)\n",
    "    X = cat_encoder_2.fit_transform(X)\n",
    "    \n",
    "    if test_df is not None:\n",
    "        # Do final encoding of categorical features on Test\n",
    "        #X_test = cat_encoder_1.transform(X_test)\n",
    "        X_test = cat_encoder_2.transform(X_test)\n",
    "        return X, y, X_test, y_test\n",
    "    else:\n",
    "        return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "575e5ae4-2ba8-4a7f-bd39-e580e5a1df76",
   "metadata": {
    "id": "575e5ae4-2ba8-4a7f-bd39-e580e5a1df76",
    "tags": []
   },
   "outputs": [],
   "source": [
    "X, y, X_test, y_test = create_features(train, target_var, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50074849-2979-45f0-9ace-660280aa8307",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48e923da-06ff-4996-ace2-e2d034ba2cef",
   "metadata": {
    "id": "48e923da-06ff-4996-ace2-e2d034ba2cef",
    "tags": []
   },
   "outputs": [],
   "source": [
    "score_dataset(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f418a52e-5c10-48a9-a658-e17f1d586255",
   "metadata": {
    "id": "f418a52e-5c10-48a9-a658-e17f1d586255"
   },
   "source": [
    "# 6. Hyperparameter Tuning\n",
    "Use Optuna to find the beset set of hyperparameters for the final model\n",
    "- you need to specify the objective (i.e. `'regression'`, `'multiclass'` with '`num_class`', etc.) \n",
    "- make sure the `folds` matches the fold evaluation scheme \n",
    "- you can set `early_stopping_rounds` and `num_boost_round` to custom values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d85c9cfc-ad6e-452e-87a5-630404590170",
   "metadata": {
    "id": "d85c9cfc-ad6e-452e-87a5-630404590170",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def objective(trial, X, y):\n",
    "    # Specify a search space using distributions across plausible values of hyperparameters.\n",
    "    param = {\n",
    "        \"objective\": \"regression\",\n",
    "        \"verbosity\": -1,              \n",
    "        'lambda_l1': trial.suggest_float('lambda_l1', 1e-8, 10.0),\n",
    "        'lambda_l2': trial.suggest_float('lambda_l2', 1e-8, 10.0),\n",
    "        'num_leaves': trial.suggest_int('num_leaves', 2, 512),\n",
    "        'feature_fraction': trial.suggest_float('feature_fraction', 0.1, 1.0),\n",
    "        'bagging_fraction': trial.suggest_float('bagging_fraction', 0.1, 1.0),\n",
    "        'bagging_freq': trial.suggest_int('bagging_freq', 0, 15),\n",
    "        'min_child_samples': trial.suggest_int('min_child_samples', 1, 100),\n",
    "    }\n",
    "    \n",
    "    # Run LightGBM for the hyperparameter values\n",
    "    lgbcv = lgb.cv(param,\n",
    "                   lgb.Dataset(X, label=y),\n",
    "                   folds= StratifiedKFold(n_splits=5).split(X, X['cut'].astype(str)+\"_\"+X[\"color\"].astype(str)+\"_\"+X['clarity'].astype(str)),\n",
    "                   callbacks=[lgb.early_stopping(75), lgb.log_evaluation(0)],                 \n",
    "                   num_boost_round=1500\n",
    "                  )\n",
    "    \n",
    "    cv_score = lgbcv['l2-mean'][-1]\n",
    "    \n",
    "    # Return metric of interest\n",
    "    return cv_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2cfb70ba-c8b4-4a7f-97fe-2f8d6c3989c3",
   "metadata": {
    "id": "2cfb70ba-c8b4-4a7f-97fe-2f8d6c3989c3",
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "optuna.logging.set_verbosity(optuna.logging.WARNING) \n",
    "study = optuna.create_study(direction='minimize')\n",
    "\n",
    "# timeout is the time in seconds until the program stops the optimization, even if its not done\n",
    "study.optimize(lambda trial: objective(trial, X, y), timeout=600, n_trials=50) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "865f4d54-39d8-4cbe-8b83-90600413a2e3",
   "metadata": {
    "id": "865f4d54-39d8-4cbe-8b83-90600413a2e3",
    "tags": []
   },
   "outputs": [],
   "source": [
    "optuna.visualization.plot_optimization_history(study)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ba1849e-46ba-4666-9cad-a76fb239622d",
   "metadata": {
    "id": "9ba1849e-46ba-4666-9cad-a76fb239622d",
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(study.best_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3de91670-0e83-4b19-af9f-8f9d9c5ae6f3",
   "metadata": {
    "id": "3de91670-0e83-4b19-af9f-8f9d9c5ae6f3",
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(study.best_value**2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f1512f9-1d87-450e-a379-f8c6abe291f4",
   "metadata": {
    "id": "9f1512f9-1d87-450e-a379-f8c6abe291f4"
   },
   "source": [
    "# 7. Fit final model and check predictions\n",
    "- Fit an ensemble of models, using different data splits to create the ensemble members, and simple averaging or stacking for the final model\n",
    "- you need to specify the objective (i.e. `'regression'`, `'multiclass'` with '`num_class`', etc.) \n",
    "- Lastly, plot the results and look at text accuracy to get a sense of how good the model is"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b749c88-f108-4feb-b26d-4f8775fd46c9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "class PredStacking(BaseEstimator):\n",
    "    \"\"\"\n",
    "    A stacking ensemble method for machine learning. This class takes in a list of predictions done by\n",
    "    base machine learning models and trains a final estimator on top of them to make the final predictions.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, final_estimator=LinearRegression()):\n",
    "        self.final_estimator = final_estimator\n",
    "\n",
    "    def fit(self, train_preds, y):\n",
    "        \"\"\"\n",
    "        Fit the final estimator on the stacked data.\n",
    "\n",
    "        Parameters:\n",
    "        -----------\n",
    "        train_preds: list\n",
    "            A list of predictions done by base machine learning models on the training data.\n",
    "\n",
    "        y: array-like, shape (n_samples,)\n",
    "            The target values.\n",
    "\n",
    "        Returns:\n",
    "        --------\n",
    "        self: object\n",
    "            Returns self.\n",
    "        \"\"\"\n",
    "        self.final_estimator.fit(np.array(train_preds).transpose(), y)\n",
    "        return self\n",
    "\n",
    "    def predict(self, test_preds):\n",
    "        \"\"\"\n",
    "        Make predictions using the final estimator on the stacked data.\n",
    "\n",
    "        Parameters:\n",
    "        -----------\n",
    "        test_preds: list\n",
    "            A list of predictions done by base machine learning models on the test data.\n",
    "\n",
    "        Returns:\n",
    "        --------\n",
    "        final_preds: array, shape (n_samples,)\n",
    "            The predicted target values.\n",
    "        \"\"\"\n",
    "        final_preds = self.final_estimator.predict(np.array(test_preds).transpose())\n",
    "        return final_preds\n",
    "\n",
    "    def fit_predict(self, preds, y):\n",
    "        \"\"\"\n",
    "        Fit the final estimator on the stacked data and make predictions using the final estimator\n",
    "        on the test data.\n",
    "\n",
    "        Parameters:\n",
    "        -----------\n",
    "        train_preds: list\n",
    "            A list of predictions done by base machine learning models on the training data.\n",
    "\n",
    "        test_preds: list\n",
    "            A list of predictions done by base machine learning models on the test data.\n",
    "\n",
    "        y: array-like, shape (n_samples,)\n",
    "            The target values.\n",
    "\n",
    "        Returns:\n",
    "        --------\n",
    "        final_preds: array, shape (n_samples,)\n",
    "            The predicted target values.\n",
    "        \"\"\"\n",
    "        self.fit(preds, y)\n",
    "        final_preds = self.predict(preds)\n",
    "        return final_preds\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85bbf6bf-e9ea-47c6-bfd5-6fdd34438a15",
   "metadata": {},
   "source": [
    "Below, we are using different folds of the data as the method to generate ensemble members. Some hyperparameters that could be adjust is the number of folds (i.e. `n_splits` and `n_repeats`), the number of trees (i.e. `n_estimators`), wether to have early stopping to prevent overfitting and how many non-productive trees to allow before stopping (i.e. `early_stopping()`).\n",
    "\n",
    "__note:__ make sure the error square matches the task. E.g., `mean_squared_error` for regression and `balanced_accuracy_score` for classifcation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f8a30b2-de9d-44e0-baee-7154b28ee6e5",
   "metadata": {
    "id": "9f8a30b2-de9d-44e0-baee-7154b28ee6e5",
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_preds = []\n",
    "train_preds = []\n",
    "skf = RepeatedStratifiedKFold(n_splits=3, n_repeats=3)\n",
    "\n",
    "for fold_idx, (train_index, valid_index) in enumerate(skf.split(X, X['cut'].astype(str)+\"_\"+X[\"color\"].astype(str)+\"_\"+X['clarity'].astype(str))):\n",
    "    X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\n",
    "    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n",
    "\n",
    "    lgb_train = lgb.Dataset(X_train, y_train)\n",
    "    lgb_eval = lgb.Dataset(X_valid, y_valid, reference=lgb_train)\n",
    "    lgb_params = {\n",
    "        'objective': \"regression\",\n",
    "        'verbose': -1,\n",
    "        'n_estimators': 1500,\n",
    "        **study.best_params\n",
    "    }\n",
    "    model = lgb.train(lgb_params, lgb_train, valid_sets=lgb_eval, callbacks=[lgb.early_stopping(75)])\n",
    "\n",
    "    y_pred = model.predict(X_valid)\n",
    "    score = mean_squared_error(y_valid, y_pred, squared=False)\n",
    "    print(\"Fold {} Score: {}\".format(fold_idx, score))\n",
    "    print(\"----------------------\")\n",
    "    test_preds.append( model.predict(X_test))\n",
    "    train_preds.append( model.predict(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09bbcd7d-340f-45d0-8478-b220f65e97f1",
   "metadata": {
    "id": "09bbcd7d-340f-45d0-8478-b220f65e97f1",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Use average for ensembling of the labels\n",
    "# final_test_preds = np.mean(test_preds, axis=0)\n",
    "\n",
    "# Use stacking for the Labels\n",
    "ensembler = PredStacking(final_estimator=LinearRegression())\n",
    "final_train_preds = ensembler.fit_predict(train_preds, y)\n",
    "final_test_preds = ensembler.predict(test_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30a7dbd3-c6d8-45d4-a943-f3125b57212e",
   "metadata": {
    "id": "30a7dbd3-c6d8-45d4-a943-f3125b57212e",
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(2, 2, sharey=True, sharex=True, figsize=(16,8))\n",
    "sns.histplot(y, ax=axs[0,0])\n",
    "axs[0,0].set_title(\"Distribution of Training Target Variable\")\n",
    "sns.histplot(final_train_preds , ax=axs[0,1])\n",
    "axs[0,1].set_title(\"Distribution of Predicted Target Variable on Train\")\n",
    "sns.histplot(final_test_preds , ax=axs[1,1])\n",
    "axs[1,1].set_title(\"Distribution of Predicted Target Variable on Test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83431446-bb61-44ca-8483-6b486500d075",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "res_df = pd.DataFrame(data = y.values, columns=['y'])\n",
    "res_df['y_pred'] = final_train_preds\n",
    "fig = plt.figure(figsize=(12, 8), dpi=100)\n",
    "sns.regplot(data=res_df, x='y_pred', y='y', color='red',\n",
    "            scatter=True, line_kws={\"color\": \"black\"}, scatter_kws={'s': 3})\n",
    "plt.title('Target versus Predicted Target');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5570550-b919-4fe8-8ea3-71fb2d19545b",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# look at those examples that had the highest error\n",
    "\n",
    "error = abs(y.values - final_train_preds)\n",
    "error_df = train.copy()\n",
    "error_df['pred_price'] = final_train_preds\n",
    "error_df['error'] = error_df['pred_price'] - error_df['price']\n",
    "error_df['abs_error'] = np.abs(error_df['error'])\n",
    "error_df.sort_values('abs_error', ascending=False).head(n=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66c9e33d-1eca-4c9a-b6c0-42a79a5a8ccf",
   "metadata": {
    "id": "66c9e33d-1eca-4c9a-b6c0-42a79a5a8ccf",
    "tags": []
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame(index=test.index)\n",
    "submission['price'] = final_test_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "yjDNx8pIQyJf",
   "metadata": {
    "id": "yjDNx8pIQyJf",
    "tags": []
   },
   "outputs": [],
   "source": [
    "submission.to_csv('submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73518db1-da83-4705-85f1-74cba401391b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}