ML_Comp_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 294,
   "id": "9086c2ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.feature_selection import mutual_info_classif\n",
    "from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV,RandomizedSearchCV\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import roc_auc_score,confusion_matrix,accuracy_score,SCORERS, f1_score\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder, OrdinalEncoder\n",
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "id": "56687bdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x = pd.read_csv(\"training_set_values.csv\")\n",
    "train_y = pd.read_csv(\"training_set_labels.csv\")\n",
    "test_x=pd.read_csv(\"test_set_values.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22c75e7c",
   "metadata": {},
   "source": [
    "# Dropping Features\n",
    "- Dropping features with too many missing values - scheme_name\n",
    "- Dropping unusable/unnecessary features - num_private, recorded_by\n",
    "-  Dropping features with very high correlation and leaving only one of each"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "id": "b4d76e0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dropping features with too many missing values, unusable features and features with very high correlation\n",
    "drop_features = [\"scheme_name\", \"num_private\", \"recorded_by\"]\n",
    "repeated = [\"payment\", \"quality_group\", \"quantity_group\", \"source\", \"source_class\",\n",
    "            \"region\", \"extraction_type\", \"extraction_type_group\", \"waterpoint_type_group\"]\n",
    "\n",
    "X = train_x.copy()\n",
    "X=X.drop(drop_features, axis=1)\n",
    "X=X.drop(repeated, axis=1)\n",
    "\n",
    "test_x = test_x.drop(drop_features, axis=1)\n",
    "test_x = test_x.drop(repeated, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "id": "afc736d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# X.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "id": "d3f405db",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pd.set_option('display.max_rows', 3000)\n",
    "# train_x['date_recorded'].value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "id": "e646892f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pd.reset_option('display.max_rows')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3c15789",
   "metadata": {},
   "source": [
    "# Filling in null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "id": "c0a8daef",
   "metadata": {},
   "outputs": [],
   "source": [
    "missing_null=[\"funder\", \"installer\", \"subvillage\", \"public_meeting\", \"scheme_management\",  \"permit\",]\n",
    "\n",
    "for col in missing_null:\n",
    "    X[col].fillna(X[col].mode()[0], inplace=True)\n",
    "    test_x[col].fillna(X[col].mode()[0], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b931b135",
   "metadata": {},
   "source": [
    "# Filling in other types of missing values such as 0, None, unkown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "id": "28998038",
   "metadata": {},
   "outputs": [],
   "source": [
    "# boolean=[\"public_meeting\", \"permit\"]\n",
    "missing_None_mode=[\"scheme_management\"]\n",
    "for col in missing_None_mode:\n",
    "    mode = X[col].mode()[0]\n",
    "    X[col].replace(\"None\", mode, inplace=True)\n",
    "    test_x[col].replace(\"None\", mode, inplace=True)\n",
    "    \n",
    "missing_unknown_values_mode = [\"payment_type\", \"water_quality\", \"quantity\", \"management\", \"management_group\"]\n",
    "for col in missing_unknown_values_mode:\n",
    "    mode = X[col].mode()[0]\n",
    "    X[col].replace(\"unknown\", mode, inplace=True)\n",
    "    \n",
    "missing_zero_mean_int = [\"gps_height\", \"population\", \"construction_year\"]\n",
    "for col in missing_zero_mean_int:\n",
    "    X[col].replace(0, np.NaN, inplace=True)\n",
    "    mean = int(X[col].mean(skipna=True))\n",
    "    X[col].fillna(mean, inplace=True)\n",
    "    test_x[col].replace(0, mean, inplace=True)\n",
    "    \n",
    "missing_none_drop_temp = [\"wpt_name\"]\n",
    "# for col in missing_unknown_values_mode:\n",
    "#     X[col].replace(\"none\", X[col].mode()[0], inplace=True)\n",
    "\n",
    "X[\"construction_year\"] = X.construction_year.apply(lambda x: pd.to_datetime(x,format='%Y'))\n",
    "test_x[\"construction_year\"] = test_x.construction_year.apply(lambda x: pd.to_datetime(x,format='%Y'))\n",
    "\n",
    "X = X.drop(missing_none_drop_temp, axis=1)\n",
    "test_x = test_x.drop(missing_none_drop_temp, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "id": "cb26846a",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', 3000)\n",
    "# combined_dataset['construction_year'].value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "id": "ac8d07da-d00c-4c34-b385-6d2dab9a4c56",
   "metadata": {},
   "outputs": [],
   "source": [
    "# X.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "id": "41b75a4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get list of categorical variables\n",
    "# s = (X.dtypes == 'object')\n",
    "# object_cols = list(s[s].index)\n",
    "# object_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "id": "6a993459",
   "metadata": {},
   "outputs": [],
   "source": [
    "X[\"test\"] = 0\n",
    "test_x[\"test\"] = 1\n",
    "frames = [X, test_x]\n",
    "combined_dataset = pd.concat(frames)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9509d725",
   "metadata": {},
   "source": [
    "## Feature Engineering\n",
    "### Added new active_time feature\n",
    "- Represents the time the pump has been active\n",
    "- Created by difference between construction_year and date_recorded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "id": "272330ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_dataset['active_time']=(combined_dataset.date_recorded.apply(pd.to_datetime)-combined_dataset.construction_year).dt.days\n",
    "# combined_dataset['active_time']=combined_dataset.active_time.apply(lambda x: float(x)/ (365.25*24*60*60*1e9))\n",
    "combined_dataset.loc[combined_dataset['active_time'] < 0,combined_dataset.columns=='active_time'] = 5000\n",
    "combined_dataset['construction_year'] = pd.DatetimeIndex(combined_dataset['construction_year']).year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "id": "bbc9897f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# combined_dataset = combined_dataset.drop(['active_time'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "id": "e51b5655",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', 3000)\n",
    "# combined_dataset['active_time'].value_counts(bins=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "id": "db7c23b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Onehot and Ordinal Encoders chosen based on whether features are ordinal or not and the number of categories.\n",
    "# Ordinal used for non ordinal features when number of categories are very high.\n",
    "one_hot=[\"basin\", \"scheme_management\", \"extraction_type_class\",\n",
    "         \"management_group\", \"quantity\", \"source_type\", \"waterpoint_type\"]\n",
    "ordinal=[\"funder\",\"installer\", \"subvillage\", \"lga\", \"ward\", \"public_meeting\",\n",
    "         \"permit\", \"management\", \"payment_type\", \"water_quality\", 'date_recorded']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0eaf9dd4",
   "metadata": {},
   "source": [
    "# Ordinal Encoding categorical features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "id": "e1c80544-39c5-4471-addd-85c49aec4dd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "ordEnc=OrdinalEncoder()\n",
    "combined_dataset[ordinal] = ordEnc.fit_transform(combined_dataset[ordinal])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "id": "80ea2d40",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_x = combined_dataset.loc[combined_dataset['test'] == 1]\n",
    "test_x = test_x.drop('test', axis=1)\n",
    "X = combined_dataset.loc[combined_dataset['test'] == 0]\n",
    "X = X.drop('test', axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3888c600",
   "metadata": {},
   "source": [
    "# OneHot Encoding categorical features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "id": "b7d3c41c-a132-4237-9982-5ade939afaf4",
   "metadata": {},
   "outputs": [],
   "source": [
    "ohe = OneHotEncoder(handle_unknown='ignore')\n",
    "for col in one_hot:\n",
    "    ohot_encoded = ohe.fit_transform(X[col].values.reshape(len(X[col].values),1)).toarray()\n",
    "    ohot_encoded2 = ohot_encoded[:,:].astype(int)\n",
    "    df_ohot = pd.DataFrame(ohot_encoded2, columns=ohe.get_feature_names())\n",
    "    X = pd.concat([X,df_ohot], axis=1)\n",
    "    X.drop(col, axis=1, inplace=True)\n",
    "    ohot_encoded_test = ohe.transform(test_x[col].values.reshape(len(test_x[col].values),1)).toarray()\n",
    "    ohot_encoded2_test = ohot_encoded_test[:,:].astype(int)\n",
    "    df_ohot_test = pd.DataFrame(ohot_encoded2_test, columns=ohe.get_feature_names())\n",
    "    test_x = pd.concat([test_x,df_ohot_test], axis=1)\n",
    "    test_x.drop(col, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "id": "e41e218f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_y.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25cd0930",
   "metadata": {},
   "source": [
    "# Label Encoding categorical labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "id": "02e08225",
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "train_y[[\"status_group\"]]=train_y[[\"status_group\"]].apply(lambda col : le.fit_transform(col))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8c559b0",
   "metadata": {},
   "source": [
    "# Training Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "id": "8a15ea74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>amount_tsh</th>\n",
       "      <th>date_recorded</th>\n",
       "      <th>funder</th>\n",
       "      <th>gps_height</th>\n",
       "      <th>installer</th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "      <th>subvillage</th>\n",
       "      <th>region_code</th>\n",
       "      <th>...</th>\n",
       "      <th>x0_river/lake</th>\n",
       "      <th>x0_shallow well</th>\n",
       "      <th>x0_spring</th>\n",
       "      <th>x0_cattle trough</th>\n",
       "      <th>x0_communal standpipe</th>\n",
       "      <th>x0_communal standpipe multiple</th>\n",
       "      <th>x0_dam</th>\n",
       "      <th>x0_hand pump</th>\n",
       "      <th>x0_improved spring</th>\n",
       "      <th>x0_other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>69572</td>\n",
       "      <td>6000.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>1548.0</td>\n",
       "      <td>1390.0</td>\n",
       "      <td>1706.0</td>\n",
       "      <td>34.938093</td>\n",
       "      <td>-9.856322</td>\n",
       "      <td>13116.0</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8776</td>\n",
       "      <td>0.0</td>\n",
       "      <td>322.0</td>\n",
       "      <td>522.0</td>\n",
       "      <td>1399.0</td>\n",
       "      <td>610.0</td>\n",
       "      <td>34.698766</td>\n",
       "      <td>-2.147466</td>\n",
       "      <td>17596.0</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34310</td>\n",
       "      <td>25.0</td>\n",
       "      <td>313.0</td>\n",
       "      <td>924.0</td>\n",
       "      <td>686.0</td>\n",
       "      <td>2296.0</td>\n",
       "      <td>37.460664</td>\n",
       "      <td>-3.821329</td>\n",
       "      <td>10096.0</td>\n",
       "      <td>21</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>67743</td>\n",
       "      <td>0.0</td>\n",
       "      <td>285.0</td>\n",
       "      <td>1961.0</td>\n",
       "      <td>263.0</td>\n",
       "      <td>2078.0</td>\n",
       "      <td>38.486161</td>\n",
       "      <td>-11.155298</td>\n",
       "      <td>9998.0</td>\n",
       "      <td>90</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>19728</td>\n",
       "      <td>0.0</td>\n",
       "      <td>111.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>1018.0</td>\n",
       "      <td>133.0</td>\n",
       "      <td>31.130847</td>\n",
       "      <td>-1.825359</td>\n",
       "      <td>8583.0</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59395</th>\n",
       "      <td>60739</td>\n",
       "      <td>10.0</td>\n",
       "      <td>351.0</td>\n",
       "      <td>486.0</td>\n",
       "      <td>1210.0</td>\n",
       "      <td>229.0</td>\n",
       "      <td>37.169807</td>\n",
       "      <td>-3.253847</td>\n",
       "      <td>6315.0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59396</th>\n",
       "      <td>27263</td>\n",
       "      <td>4700.0</td>\n",
       "      <td>97.0</td>\n",
       "      <td>196.0</td>\n",
       "      <td>1212.0</td>\n",
       "      <td>299.0</td>\n",
       "      <td>35.249991</td>\n",
       "      <td>-9.070629</td>\n",
       "      <td>3323.0</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59397</th>\n",
       "      <td>37057</td>\n",
       "      <td>0.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>507.0</td>\n",
       "      <td>1018.0</td>\n",
       "      <td>442.0</td>\n",
       "      <td>34.017087</td>\n",
       "      <td>-8.750434</td>\n",
       "      <td>9784.0</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59398</th>\n",
       "      <td>31282</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>992.0</td>\n",
       "      <td>1018.0</td>\n",
       "      <td>1360.0</td>\n",
       "      <td>35.861315</td>\n",
       "      <td>-6.378573</td>\n",
       "      <td>15553.0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59399</th>\n",
       "      <td>26348</td>\n",
       "      <td>0.0</td>\n",
       "      <td>63.0</td>\n",
       "      <td>2106.0</td>\n",
       "      <td>191.0</td>\n",
       "      <td>2288.0</td>\n",
       "      <td>38.104048</td>\n",
       "      <td>-6.747464</td>\n",
       "      <td>6556.0</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>59400 rows × 70 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  amount_tsh  date_recorded  funder  gps_height  installer  \\\n",
       "0      69572      6000.0           54.0  1548.0      1390.0     1706.0   \n",
       "1       8776         0.0          322.0   522.0      1399.0      610.0   \n",
       "2      34310        25.0          313.0   924.0       686.0     2296.0   \n",
       "3      67743         0.0          285.0  1961.0       263.0     2078.0   \n",
       "4      19728         0.0          111.0    20.0      1018.0      133.0   \n",
       "...      ...         ...            ...     ...         ...        ...   \n",
       "59395  60739        10.0          351.0   486.0      1210.0      229.0   \n",
       "59396  27263      4700.0           97.0   196.0      1212.0      299.0   \n",
       "59397  37057         0.0           82.0   507.0      1018.0      442.0   \n",
       "59398  31282         0.0           48.0   992.0      1018.0     1360.0   \n",
       "59399  26348         0.0           63.0  2106.0       191.0     2288.0   \n",
       "\n",
       "       longitude   latitude  subvillage  region_code  ...  x0_river/lake  \\\n",
       "0      34.938093  -9.856322     13116.0           11  ...              0   \n",
       "1      34.698766  -2.147466     17596.0           20  ...              0   \n",
       "2      37.460664  -3.821329     10096.0           21  ...              0   \n",
       "3      38.486161 -11.155298      9998.0           90  ...              0   \n",
       "4      31.130847  -1.825359      8583.0           18  ...              0   \n",
       "...          ...        ...         ...          ...  ...            ...   \n",
       "59395  37.169807  -3.253847      6315.0            3  ...              0   \n",
       "59396  35.249991  -9.070629      3323.0           11  ...              1   \n",
       "59397  34.017087  -8.750434      9784.0           12  ...              0   \n",
       "59398  35.861315  -6.378573     15553.0            1  ...              0   \n",
       "59399  38.104048  -6.747464      6556.0            5  ...              0   \n",
       "\n",
       "       x0_shallow well  x0_spring  x0_cattle trough  x0_communal standpipe  \\\n",
       "0                    0          1                 0                      1   \n",
       "1                    0          0                 0                      1   \n",
       "2                    0          0                 0                      0   \n",
       "3                    0          0                 0                      0   \n",
       "4                    0          0                 0                      1   \n",
       "...                ...        ...               ...                    ...   \n",
       "59395                0          1                 0                      1   \n",
       "59396                0          0                 0                      1   \n",
       "59397                0          0                 0                      0   \n",
       "59398                1          0                 0                      0   \n",
       "59399                1          0                 0                      0   \n",
       "\n",
       "       x0_communal standpipe multiple  x0_dam  x0_hand pump  \\\n",
       "0                                   0       0             0   \n",
       "1                                   0       0             0   \n",
       "2                                   1       0             0   \n",
       "3                                   1       0             0   \n",
       "4                                   0       0             0   \n",
       "...                               ...     ...           ...   \n",
       "59395                               0       0             0   \n",
       "59396                               0       0             0   \n",
       "59397                               0       0             1   \n",
       "59398                               0       0             1   \n",
       "59399                               0       0             1   \n",
       "\n",
       "       x0_improved spring  x0_other  \n",
       "0                       0         0  \n",
       "1                       0         0  \n",
       "2                       0         0  \n",
       "3                       0         0  \n",
       "4                       0         0  \n",
       "...                   ...       ...  \n",
       "59395                   0         0  \n",
       "59396                   0         0  \n",
       "59397                   0         0  \n",
       "59398                   0         0  \n",
       "59399                   0         0  \n",
       "\n",
       "[59400 rows x 70 columns]"
      ]
     },
     "execution_count": 315,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "id": "d9876c8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = X.iloc[:,:70].values\n",
    "y = train_y.iloc[:,1].values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "733539c6",
   "metadata": {},
   "source": [
    "## Training and testing using train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "id": "add84a3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5890,   87,  467],\n",
       "       [ 491,  253,  123],\n",
       "       [ 999,   41, 3529]], dtype=int64)"
      ]
     },
     "execution_count": 317,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.2, random_state = 0)\n",
    "# model=RandomForestClassifier(n_estimators=600, max_depth=40, min_samples_split=10)\n",
    "# model.fit(x_train,y_train)\n",
    "# predictions=model.predict(x_test)\n",
    "# confusion_matrix(y_test,predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "id": "60601190",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8141414141414142"
      ]
     },
     "execution_count": 318,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy_score(y_test,predictions)\n",
    "# f1_score(y_test, predictions, average='weighted')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de301f05",
   "metadata": {},
   "source": [
    "## Training full dataset with train data and predicting for test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "id": "bdda162b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test Data Set\n",
    "model=RandomForestClassifier(n_estimators=600,max_depth=40, min_samples_split=10)\n",
    "model.fit(x,y)\n",
    "predictions=model.predict(test_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "id": "da05d91a",
   "metadata": {},
   "outputs": [],
   "source": [
    "out_preds=le.inverse_transform(predictions)\n",
    "indexes=pd.read_csv(\"test_set_values.csv\").iloc[:,0].values\n",
    "out_data=pd.DataFrame({\"id\":indexes,\"status_group\":out_preds})\n",
    "out_data.to_csv(\"output.csv\",encoding='utf-8',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9fe8f08b",
   "metadata": {},
   "source": [
    "## Hyperparameter optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "id": "8c21ff73",
   "metadata": {},
   "outputs": [],
   "source": [
    "# params={\"n_estimators\":[100,400,600,800],\n",
    "#         \"max_depth\":[10,20,40,80]}\n",
    "# model=RandomForestClassifier()\n",
    "# cv=KFold(n_splits=10,shuffle=True)\n",
    "# gsearch = GridSearchCV(model, params,cv=cv, verbose=2, n_jobs=-1)\n",
    "# results = gsearch.fit(x_train, y_train)\n",
    "# results.best_params_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}