Automatic Model Hyper Parameter Search (SAAK).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score, KFold\n",
    "import xgboost\n",
    "import lightgbm\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.decomposition import PCA, SparsePCA, MiniBatchSparsePCA, KernelPCA\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import log_loss, accuracy_score\n",
    "import copy\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import ExtraTreesClassifier, GradientBoostingClassifier, RandomForestClassifier, AdaBoostClassifier, BaggingClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "kfold = KFold(5, shuffle=True, random_state=999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_train_df(top_percent=.9):\n",
    "    train_X = pd.read_csv('Data/saak_transformed_train_energy_90.csv')\n",
    "    train_y = np.array(train_X['is_iceberg']).reshape((train_X.shape[0],))\n",
    "    \n",
    "    columns = f_test_sort(train_X, top_percent=top_percent)\n",
    "    train_X = train_X[columns]\n",
    "    #train_X.drop(columns=['is_iceberg'], inplace=True)\n",
    "    \n",
    "    print(\"Train data X: {}, y: {}\".format(train_X.shape, train_y.shape))\n",
    "    return train_X, train_y, columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import operator\n",
    "    \n",
    "def f_test_sort(train_X, top_percent=.9):\n",
    "    features = train_X.columns.tolist()\n",
    "    features.remove('is_iceberg')\n",
    "    overall_mean = train_X[features].mean(axis=0)\n",
    "    \n",
    "    class_train = [train_X[train_X.is_iceberg == label] for label in [0,1]]\n",
    "    print(class_train[0].shape, class_train[1].shape)\n",
    "    \n",
    "    f_dict = {}\n",
    "    for i,f in enumerate(features):\n",
    "        bgv = 0.0\n",
    "        wgv = 0.0\n",
    "        for c in [0,1]:\n",
    "            bgv += class_train[c].shape[0]*(class_train[c][f].mean() - overall_mean[i])**2\n",
    "            wgv += ((class_train[c][f] - class_train[c][f].mean())**2).sum()\n",
    "        wgv /= train_X.shape[0] - 2\n",
    "        f_dict[f] = bgv/wgv\n",
    "    \n",
    "    f_dict_sorted = sorted(f_dict.items(), key=operator.itemgetter(1), reverse=True)\n",
    "    \n",
    "    select_num = int(top_percent*len(f_dict_sorted))\n",
    "    select_cols = [col for (col, val) in f_dict_sorted[:select_num]]\n",
    "    return select_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_test_df(cols=None):\n",
    "    if cols is None:\n",
    "        test_df = pd.read_csv('Data/saak_transformed_test_energy_90.csv')\n",
    "    else:\n",
    "        test_df = pd.read_csv('Data/saak_transformed_test_energy_90.csv', usecols=cols)\n",
    "        \n",
    "    print(\"Test data X: {}\".format(test_df.shape))\n",
    "    return test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_feature_num(pca, th):\n",
    "    counts = pd.Series(pca.explained_variance_ >= th).value_counts(sort=False)\n",
    "    return counts.values[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_transformed_df(pca, df):\n",
    "    new_data = np.array(pca.transform(df))\n",
    "    columns = ['f_{}'.format(i) for i in range(new_data.shape[1])]\n",
    "    df = pd.DataFrame(data=new_data,    # values\n",
    "                  index=df.index,\n",
    "                  columns=columns)  # 1st row as the column names\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_log_loss(fn, df, y):\n",
    "    pca = PCA(whiten=False, svd_solver='randomized', random_state=0, n_components=fn)\n",
    "    pca.fit(df)\n",
    "    \n",
    "    X = get_transformed_df(pca, df)\n",
    "    \n",
    "    clf = lightgbm.LGBMClassifier(n_jobs=4, objective='binary', random_state=0, n_estimators=100, boosting_type='dart')\n",
    "    log_loss = -1*cross_val_score(clf, X=X, y=y, scoring='neg_log_loss', cv=kfold)\n",
    "    avg_log_loss = sum(log_loss) / float(len(log_loss))\n",
    "    \n",
    "    return avg_log_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def feature_selection(train_X, train_y):\n",
    "    pca = PCA(whiten=False, svd_solver='randomized', random_state=0)\n",
    "    pca.fit(train_X)\n",
    "    \n",
    "    th_list = [1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8]\n",
    "    f_num = [get_feature_num(pca, th) for th in th_list]\n",
    "    \n",
    "    print(\"# of features to test: \",f_num)\n",
    "    cv_scores = [get_log_loss(fn, train_X, train_y) for fn in f_num]\n",
    "    \n",
    "    index_min = np.argmin(cv_scores)\n",
    "    print(\"# of features by light gbm: {}, cv score: {:.3f}\".format(f_num[index_min], cv_scores[index_min]))\n",
    "    \n",
    "    pca = PCA(whiten=False, svd_solver='randomized', random_state=0, n_components=f_num[index_min])\n",
    "    pca.fit(train_X)\n",
    "    train_X = get_transformed_df(pca, train_X)\n",
    "    \n",
    "    print(\"New train data X: {}, y: {}\".format(train_X.shape, train_y.shape))\n",
    "    return train_X, train_y, pca"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class base_tuner():\n",
    "    def __init__(self, X, y):\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.default_params = {}\n",
    "        \n",
    "    def fit_and_update_params(self, params, update=True):\n",
    "        clf = self.get_clf()\n",
    "\n",
    "        gs = GridSearchCV(clf, params, scoring='neg_log_loss', cv=kfold)\n",
    "        gs.fit(self.X, self.y)\n",
    "        \n",
    "        cv_df = pd.DataFrame().from_dict(gs.cv_results_)\n",
    "        cv_df = cv_df[['mean_train_score', 'mean_test_score', 'std_test_score', 'params', 'rank_test_score']]\n",
    "        cv_df = cv_df.sort_values(by=['rank_test_score', 'std_test_score']).reset_index(drop=True)\n",
    "        best_params = cv_df.loc[0, 'params']\n",
    "        \n",
    "        if update is True:\n",
    "            self.default_params.update(best_params)\n",
    "        \n",
    "        print('Selected hyper-params:', best_params)\n",
    "        print('==============================> cv score: {:.4f}'.format(cv_df.loc[0, 'mean_test_score']))\n",
    "        return best_params\n",
    "    \n",
    "    def tune(self):\n",
    "        pass\n",
    "    \n",
    "    def get_clf(self):\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class lgbm_tuner(base_tuner):\n",
    "    def __init__(self, X, y):\n",
    "        super(lgbm_tuner, self).__init__(X, y)\n",
    "        self.default_params = {\n",
    "            'n_jobs': 4,\n",
    "            'objective': 'binary',\n",
    "            'random_state': 0,\n",
    "            'boosting_type': 'dart'\n",
    "        }\n",
    "    \n",
    "    def tune_est_num_and_lr(self):\n",
    "        params = {\n",
    "            'n_estimators': [100, 200, 400, 800],\n",
    "            'learning_rate': [0.1, 0.05, 0.01, 0.005]\n",
    "        }\n",
    "\n",
    "        self.fit_and_update_params(params)\n",
    "   \n",
    "    def tune_leaves_num_and_gamma(self):\n",
    "        params = {\n",
    "            'num_leaves': [2, 3, 7, 15, 31, 63],\n",
    "            'min_split_gain': [.0, .1, .2]\n",
    "        }\n",
    "        self.fit_and_update_params(params)\n",
    "     \n",
    "    def tune_sampling(self):\n",
    "        params = {\n",
    "            'subsample': [1., .8, .6, .4, .2],\n",
    "            'colsample_bytree': [1., .8, .6, .4, .2]\n",
    "        }\n",
    "        best_parmas = self.fit_and_update_params(params, update=False)\n",
    "        \n",
    "        next_params = {}\n",
    "        for k,v in best_parmas.items():\n",
    "            if v == 1.:\n",
    "                next_params[k] = [1., .95, .9, .85]\n",
    "            else:\n",
    "                next_params[k] = [v+.15, v+.1, v+.05, v, v-.05, v-.1, v-.15]\n",
    " \n",
    "        self.fit_and_update_params(next_params)\n",
    "        \n",
    "    def tune_regularization(self):\n",
    "        params = {\n",
    "            'reg_alpha': [1., .8, .6, .4, .2, .0],\n",
    "            'reg_lambda': [1., .8, .6, .4, .2, .0]\n",
    "        }\n",
    "        \n",
    "        best_parmas = self.fit_and_update_params(params, update=False)\n",
    "        \n",
    "        next_params = {}\n",
    "        for k,v in best_parmas.items():\n",
    "            if v == 1.:\n",
    "                next_params[k] = [1., .95, .9, .85]\n",
    "            elif v == 0.:\n",
    "                next_params[k] = [.0, .05, .1, .15]\n",
    "            else:\n",
    "                next_params[k] = [v+.15, v+.1, v+.05, v, v-.05, v-.1, v-.15]\n",
    " \n",
    "        self.fit_and_update_params(next_params)\n",
    "    \n",
    "    def tune(self):\n",
    "        print('lgb tuner start tuning')\n",
    "        self.tune_est_num_and_lr()\n",
    "        self.tune_leaves_num_and_gamma()\n",
    "        self.tune_sampling()\n",
    "        self.tune_regularization()\n",
    "        \n",
    "        return self.get_clf()\n",
    "    \n",
    "    def get_clf(self):\n",
    "        return lightgbm.LGBMClassifier(**self.default_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class xgb_tuner(base_tuner):\n",
    "    def __init__(self, X, y):\n",
    "        super(xgb_tuner, self).__init__(X, y)\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.default_params = {\n",
    "            'n_jobs': 4,\n",
    "            'objective': 'binary:logistic',\n",
    "            'seed': 0,\n",
    "            'eval_metric': 'logloss'\n",
    "        }\n",
    "    \n",
    "    def tune_booster(self):\n",
    "        params = {\n",
    "            'booster': ['dart', 'gbtree']\n",
    "        }\n",
    "        self.fit_and_update_params(params)\n",
    "        \n",
    "    def tune_est_num_and_lr(self):\n",
    "        params = {\n",
    "            'n_estimators': [100, 200, 400, 800],\n",
    "            'learning_rate': [0.1, 0.05, 0.01, 0.005]\n",
    "        }\n",
    "\n",
    "        self.fit_and_update_params(params)\n",
    "   \n",
    "    def tune_max_depth(self):\n",
    "        params = {\n",
    "            'max_depth': [1, 3, 5, 7, 9]\n",
    "        }\n",
    "        self.fit_and_update_params(params)\n",
    "        \n",
    "    def tune_child_w_and_gamma(self):\n",
    "        params = {\n",
    "            'min_child_weight': [1, 2, 4, 6, 8, 10],\n",
    "            'gamma': [0, 0.1, 0.2]\n",
    "        }\n",
    "        self.fit_and_update_params(params)\n",
    "     \n",
    "    def tune_sampling(self):\n",
    "        params = {\n",
    "            'subsample': [1., .8, .6, .4, .2],\n",
    "            'colsample_bytree': [1., .8, .6, .4, .2]\n",
    "        }\n",
    "        best_parmas = self.fit_and_update_params(params, update=False)\n",
    "        \n",
    "        next_params = {}\n",
    "        for k,v in best_parmas.items():\n",
    "            if v == 1.:\n",
    "                next_params[k] = [1., .95, .9, .85]\n",
    "            else:\n",
    "                next_params[k] = [v+.15, v+.1, v+.05, v, v-.05, v-.1, v-.15]\n",
    " \n",
    "        self.fit_and_update_params(next_params)\n",
    "        \n",
    "    def tune_regularization(self):\n",
    "        params = {\n",
    "            'reg_alpha': [1., .8, .6, .4, .2, .0],\n",
    "            'reg_lambda': [1., .8, .6, .4, .2, .0]\n",
    "        }\n",
    "        \n",
    "        best_parmas = self.fit_and_update_params(params, update=False)\n",
    "        \n",
    "        next_params = {}\n",
    "        for k,v in best_parmas.items():\n",
    "            if v == 1.:\n",
    "                next_params[k] = [1., .95, .9, .85]\n",
    "            elif v == 0.:\n",
    "                next_params[k] = [.0, .05, .1, .15]\n",
    "            else:\n",
    "                next_params[k] = [v+.15, v+.1, v+.05, v, v-.05, v-.1, v-.15]\n",
    " \n",
    "        self.fit_and_update_params(next_params)\n",
    "    \n",
    "    def tune(self):\n",
    "        print('xgb tuner start tuning')\n",
    "        self.tune_booster()\n",
    "        self.tune_est_num_and_lr()\n",
    "        self.tune_max_depth()\n",
    "        self.tune_child_w_and_gamma()\n",
    "        self.tune_sampling()\n",
    "        self.tune_regularization()\n",
    "        \n",
    "        return self.get_clf()\n",
    "    \n",
    "    def get_clf(self):\n",
    "        return xgboost.XGBClassifier(**self.default_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class lr_tuner(base_tuner):\n",
    "    def __init__(self, X, y):\n",
    "        super(lr_tuner, self).__init__(X, y)\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.default_params = {\n",
    "            'penalty': 'l1',\n",
    "            'max_iter': 10000\n",
    "        }\n",
    "\n",
    "    def tune(self):\n",
    "        print('logistic regression tuner start tuning')\n",
    "        params = {\n",
    "            'solver': ['liblinear', 'saga']\n",
    "        }\n",
    "        \n",
    "        self.fit_and_update_params(params)\n",
    "        \n",
    "        return self.get_clf()\n",
    "    \n",
    "    def get_clf(self):\n",
    "        return LogisticRegression(**self.default_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class mlp_tuner(base_tuner):\n",
    "    def __init__(self, X, y):\n",
    "        super(mlp_tuner, self).__init__(X, y)\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.default_params = {\n",
    "            'learning_rate': 'adaptive',\n",
    "            'learning_rate_init': 0.005,\n",
    "            'max_iter': 2000,\n",
    "            'random_state':0\n",
    "        }\n",
    "\n",
    "    def tune(self):\n",
    "        print('mlp tuner start tuning')\n",
    "        params = {\n",
    "            'solver':['lbfgs', 'sgd', 'adam'],\n",
    "            'hidden_layer_sizes': [(100,), (150,), (100, 100,)],\n",
    "        }\n",
    "        \n",
    "        self.fit_and_update_params(params)\n",
    "        \n",
    "        params = {\n",
    "            'alpha': [10., 5., 2., 1., .8, .5, .2, .1],\n",
    "        }\n",
    "        \n",
    "        self.fit_and_update_params(params)\n",
    "        \n",
    "        return self.get_clf()\n",
    "    \n",
    "    def get_clf(self):\n",
    "        return MLPClassifier(**self.default_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class adb_tuner(base_tuner):\n",
    "    def __init__(self, X, y):\n",
    "        super(adb_tuner, self).__init__(X, y)\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.default_params = {\n",
    "            'algorithm': 'SAMME.R',\n",
    "            'random_state':0\n",
    "        }\n",
    "\n",
    "    def tune(self):\n",
    "        print('adaboost tuner start tuning')\n",
    "        params = {\n",
    "            'n_estimators': [100, 200, 400, 800],\n",
    "            'learning_rate': [0.1, 0.05, 0.01, 0.005]\n",
    "        }\n",
    "        \n",
    "        self.fit_and_update_params(params)\n",
    "\n",
    "        return self.get_clf()\n",
    "    \n",
    "    def get_clf(self):\n",
    "        return AdaBoostClassifier(**self.default_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class bg_tuner(base_tuner):\n",
    "    def __init__(self, X, y):\n",
    "        super(bg_tuner, self).__init__(X, y)\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.default_params = {\n",
    "            'random_state':0,\n",
    "            'bootstrap_features':True\n",
    "        }\n",
    "        \n",
    "    def tune_est_num(self):\n",
    "        params = {\n",
    "            'n_estimators': [100, 200, 400, 800]\n",
    "        }\n",
    "\n",
    "        self.fit_and_update_params(params)\n",
    "     \n",
    "    def tune_sampling(self):\n",
    "        params = {\n",
    "            'max_samples': [1., .8, .6, .4, .2],\n",
    "            'max_features': [1., .8, .6, .4, .2]\n",
    "        }\n",
    "        best_parmas = self.fit_and_update_params(params, update=False)\n",
    "        \n",
    "        next_params = {}\n",
    "        for k,v in best_parmas.items():\n",
    "            if v == 1.:\n",
    "                next_params[k] = [1., .95, .9, .85]\n",
    "            else:\n",
    "                next_params[k] = [v+.15, v+.1, v+.05, v, v-.05, v-.1, v-.15]\n",
    " \n",
    "        self.fit_and_update_params(next_params)\n",
    "    \n",
    "    def tune(self):\n",
    "        print('bagging tuner start tuning')\n",
    "        self.tune_est_num()\n",
    "        self.tune_sampling()\n",
    "\n",
    "        return self.get_clf()\n",
    "    \n",
    "    def get_clf(self):\n",
    "        return BaggingClassifier(**self.default_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class gb_tuner(base_tuner):\n",
    "    def __init__(self, X, y):\n",
    "        super(gb_tuner, self).__init__(X, y)\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.default_params = {\n",
    "            'random_state':0\n",
    "        }\n",
    "        \n",
    "    def tune_loss_criterion(self):\n",
    "        params = {\n",
    "            'loss': ['deviance', 'exponential'],\n",
    "            'criterion': ['friedman_mse', 'mse']\n",
    "        }\n",
    "\n",
    "        self.fit_and_update_params(params)\n",
    "        \n",
    "    def tune_est_num_and_lr(self):\n",
    "        params = {\n",
    "            'n_estimators': [100, 200, 400, 800],\n",
    "            'learning_rate': [0.1, 0.05, 0.01, 0.005]\n",
    "        }\n",
    "\n",
    "        self.fit_and_update_params(params)\n",
    "   \n",
    "    def tune_max_depth(self):\n",
    "        params = {\n",
    "            'max_depth': [1, 3, 5, 7, 9]\n",
    "        }\n",
    "        self.fit_and_update_params(params)\n",
    "        \n",
    "    def tune_child(self):\n",
    "        params = {\n",
    "            'min_samples_split': [2, 3, 7, 15,31],\n",
    "            'min_impurity_decrease': [0, 0.1, 0.2]\n",
    "        }\n",
    "        self.fit_and_update_params(params)\n",
    "     \n",
    "    def tune_sampling(self):\n",
    "        params = {\n",
    "            'subsample': [1., .8, .6, .4, .2],\n",
    "            'max_features': [1., .8, .6, .4, .2]\n",
    "        }\n",
    "        best_parmas = self.fit_and_update_params(params, update=False)\n",
    "        \n",
    "        next_params = {}\n",
    "        for k,v in best_parmas.items():\n",
    "            if v == 1.:\n",
    "                next_params[k] = [1., .95, .9, .85]\n",
    "            else:\n",
    "                next_params[k] = [v+.15, v+.1, v+.05, v, v-.05, v-.1, v-.15]\n",
    " \n",
    "        self.fit_and_update_params(next_params)\n",
    "      \n",
    "    def tune(self):\n",
    "        print('gradient boosting tuner start tuning')\n",
    "        self.tune_loss_criterion()\n",
    "        self.tune_est_num_and_lr()\n",
    "        self.tune_max_depth()\n",
    "        self.tune_child()\n",
    "        self.tune_sampling()\n",
    "        \n",
    "        return self.get_clf()\n",
    "    \n",
    "    def get_clf(self):\n",
    "        return GradientBoostingClassifier(**self.default_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class rf_tuner(base_tuner):\n",
    "    def __init__(self, X, y):\n",
    "        super(rf_tuner, self).__init__(X, y)\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.default_params = {\n",
    "            'random_state':0,\n",
    "            'n_jobs': 4\n",
    "        }\n",
    "        \n",
    "    def tune_loss_criterion(self):\n",
    "        params = {\n",
    "            'class_weight': [None, 'balanced'],\n",
    "            'criterion': ['gini', 'entropy']\n",
    "        }\n",
    "\n",
    "        self.fit_and_update_params(params)\n",
    "        \n",
    "    def tune_est_num(self):\n",
    "        params = {\n",
    "            'n_estimators': [100, 200, 400, 800]\n",
    "        }\n",
    "\n",
    "        self.fit_and_update_params(params)\n",
    "   \n",
    "    def tune_max_depth(self):\n",
    "        params = {\n",
    "            'max_depth': [1, 3, 5, 7, 9]\n",
    "        }\n",
    "        self.fit_and_update_params(params)\n",
    "        \n",
    "    def tune_child(self):\n",
    "        params = {\n",
    "            'min_samples_split': [2, 3, 7, 15,31],\n",
    "            'min_impurity_decrease': [0, 0.1, 0.2]\n",
    "        }\n",
    "        self.fit_and_update_params(params)\n",
    "     \n",
    "    def tune_sampling(self):\n",
    "        params = {\n",
    "            'max_features': [1., .8, .6, .4, .2]\n",
    "        }\n",
    "        best_parmas = self.fit_and_update_params(params, update=False)\n",
    "        \n",
    "        next_params = {}\n",
    "        for k,v in best_parmas.items():\n",
    "            if v == 1.:\n",
    "                next_params[k] = [1., .95, .9, .85]\n",
    "            else:\n",
    "                next_params[k] = [v+.15, v+.1, v+.05, v, v-.05, v-.1, v-.15]\n",
    " \n",
    "        self.fit_and_update_params(next_params)\n",
    "      \n",
    "    def tune(self):\n",
    "        print('random forest tuner start tuning')\n",
    "        self.tune_loss_criterion()\n",
    "        self.tune_est_num()\n",
    "        self.tune_max_depth()\n",
    "        self.tune_child()\n",
    "        self.tune_sampling()\n",
    "        \n",
    "        return self.get_clf()\n",
    "    \n",
    "    def get_clf(self):\n",
    "        return RandomForestClassifier(**self.default_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class et_tuner(rf_tuner):\n",
    "    def __init__(self, X, y):\n",
    "        super(et_tuner, self).__init__(X, y)\n",
    "\n",
    "    def tune(self):\n",
    "        print('extra tree tuner start tuning')\n",
    "        self.tune_loss_criterion()\n",
    "        self.tune_est_num()\n",
    "        self.tune_max_depth()\n",
    "        self.tune_child()\n",
    "        self.tune_sampling()\n",
    "        \n",
    "        return self.get_clf()\n",
    "    \n",
    "    def get_clf(self):\n",
    "        return ExtraTreesClassifier(**self.default_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from stacking_models_api import StackingAveragedModels\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test data shape: (8424, 4)\n",
      "(851, 31601) (753, 31601)\n",
      "Train data X: (1604, 18960), y: (1604,)\n",
      "[0 0 1 0 0 1 1 0 0 0]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-20-35db7cf628f0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[0mtrain_X\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_y\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mselect_cols\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_train_df\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtop_percent\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m.6\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_y\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m \u001b[0mtrain_X\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_y\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpca\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfeature_selection\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_X\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_y\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     11\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-8-57fe5fe951ee>\u001b[0m in \u001b[0;36mfeature_selection\u001b[1;34m(train_X, train_y)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mfeature_selection\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_X\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_y\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m     \u001b[0mpca\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mPCA\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwhiten\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msvd_solver\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'randomized'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m     \u001b[0mpca\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_X\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mth_list\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;36m1e-1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1e-2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1e-3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1e-4\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1e-5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1e-6\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1e-7\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1e-8\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\decomposition\\pca.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m    327\u001b[0m             \u001b[0mReturns\u001b[0m \u001b[0mthe\u001b[0m \u001b[0minstance\u001b[0m \u001b[0mitself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    328\u001b[0m         \"\"\"\n\u001b[1;32m--> 329\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    330\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    331\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\decomposition\\pca.py\u001b[0m in \u001b[0;36m_fit\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    392\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit_full\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_components\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    393\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0msvd_solver\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'arpack'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'randomized'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 394\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit_truncated\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_components\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msvd_solver\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    395\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    396\u001b[0m             raise ValueError(\"Unrecognized svd_solver='{0}'\"\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\decomposition\\pca.py\u001b[0m in \u001b[0;36m_fit_truncated\u001b[1;34m(self, X, n_components, svd_solver)\u001b[0m\n\u001b[0;32m    493\u001b[0m                                      \u001b[0mn_iter\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0miterated_power\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    494\u001b[0m                                      \u001b[0mflip_sign\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 495\u001b[1;33m                                      random_state=random_state)\n\u001b[0m\u001b[0;32m    496\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    497\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mn_samples_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mn_features_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mn_samples\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_features\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\extmath.py\u001b[0m in \u001b[0;36mrandomized_svd\u001b[1;34m(M, n_components, n_oversamples, n_iter, power_iteration_normalizer, transpose, flip_sign, random_state)\u001b[0m\n\u001b[0;32m    324\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    325\u001b[0m     Q = randomized_range_finder(M, n_random, n_iter,\n\u001b[1;32m--> 326\u001b[1;33m                                 power_iteration_normalizer, random_state)\n\u001b[0m\u001b[0;32m    327\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    328\u001b[0m     \u001b[1;31m# project M to the (k + p) dimensional space using the basis vectors\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\extmath.py\u001b[0m in \u001b[0;36mrandomized_range_finder\u001b[1;34m(A, size, n_iter, power_iteration_normalizer, random_state)\u001b[0m\n\u001b[0;32m    214\u001b[0m             \u001b[0mQ\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msafe_sparse_dot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mA\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mT\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    215\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mpower_iteration_normalizer\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'LU'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 216\u001b[1;33m             \u001b[0mQ\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlinalg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlu\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msafe_sparse_dot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mA\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpermute_l\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    217\u001b[0m             \u001b[0mQ\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlinalg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlu\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msafe_sparse_dot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mA\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mT\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpermute_l\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    218\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mpower_iteration_normalizer\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'QR'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\extmath.py\u001b[0m in \u001b[0;36msafe_sparse_dot\u001b[1;34m(a, b, dense_output)\u001b[0m\n\u001b[0;32m    138\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mret\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    139\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 140\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    141\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    142\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "test_df = pd.read_json('Data/test.json')\n",
    "print(\"Test data shape: {}\".format(test_df.shape))\n",
    "test_ids = test_df['id']\n",
    "del test_df\n",
    "gc.enable()\n",
    "gc.collect()\n",
    "\n",
    "train_X, train_y, select_cols = get_train_df(top_percent=.6)\n",
    "print(train_y[:10])\n",
    "train_X, train_y, pca = feature_selection(train_X, train_y)\n",
    "\n",
    "\n",
    "test_X = get_test_df(select_cols)\n",
    "test_X = get_transformed_df(pca, test_X)    \n",
    "\n",
    "tuners = [lr_tuner(train_X, train_y),\n",
    "          lgbm_tuner(train_X, train_y), \n",
    "          xgb_tuner(train_X, train_y)]\n",
    "clfs = []\n",
    "for tuner in tuners:\n",
    "    tuner.tune()\n",
    "    clfs.append(tuner.get_clf())\n",
    "\n",
    "for i, clf in enumerate(clfs):\n",
    "    if isinstance(tuners[i], lgbm_tuner) or isinstance(tuners[i], xgb_tuner):\n",
    "        if isinstance(tuners[i], lgbm_tuner):\n",
    "            name = 'lgb'\n",
    "        else:\n",
    "            name = 'xgb'\n",
    "\n",
    "        clf.fit(train_X, train_y)\n",
    "        predictions = clf.predict_proba(test_X)[:,1]\n",
    "        print(\"{} tuner predictions\\n\".format(name), predictions)\n",
    "\n",
    "        submission = pd.DataFrame()\n",
    "        submission['id'] = test_ids\n",
    "        submission['is_iceberg'] = predictions\n",
    "        submission.to_csv('Submissions/submission_{}_auto_fine_tune_saak.csv'.format(name), \n",
    "                          float_format=\"%.15f\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                              band_1  \\\n",
      "0  [-27.878360999999998, -27.15416, -28.668615, -...   \n",
      "1  [-12.242375, -14.920304999999999, -14.920363, ...   \n",
      "2  [-24.603676, -24.603714, -24.871029, -23.15277...   \n",
      "3  [-22.454607, -23.082819, -23.998013, -23.99805...   \n",
      "4  [-26.006956, -23.164886, -23.164886, -26.89116...   \n",
      "5  [-20.769371, -20.769434, -25.906025, -25.90602...   \n",
      "6  [-26.673811, -23.666162, -27.622442, -28.31768...   \n",
      "7  [-24.989119, -27.755224, -25.817074, -24.98927...   \n",
      "8  [-17.146641, -17.146572, -17.994583, -19.44553...   \n",
      "9  [-24.020853, -23.551275, -27.18819, -29.126434...   \n",
      "\n",
      "                                              band_2        id inc_angle  \\\n",
      "0  [-27.154118, -29.537888, -31.0306, -32.190483,...  dfd5f913   43.9239   \n",
      "1  [-31.506321, -27.984554, -26.645678, -23.76760...  e25388fd   38.1562   \n",
      "2  [-24.870956, -24.092632, -20.653963, -19.41104...  58b2aaa0   45.2859   \n",
      "3  [-27.889421, -27.519794, -27.165262, -29.10350...  4cfc3a18   43.8306   \n",
      "4  [-27.206915, -30.259186, -30.259186, -23.16495...  271f93f4   35.6256   \n",
      "5  [-29.288746, -29.712593, -28.884804, -28.88480...  b51d18b5   36.9034   \n",
      "6  [-24.557735, -26.97868, -27.622442, -29.073456...  31da1a04   34.4751   \n",
      "7  [-27.755173, -26.732174, -28.124943, -31.83772...  56929c16   41.1769   \n",
      "8  [-25.733608, -24.472507, -24.710424, -22.77215...  525ab75c   35.7829   \n",
      "9  [-28.702518, -33.563324, -29.571918, -29.12643...  192f56eb   43.3007   \n",
      "\n",
      "   is_iceberg  \n",
      "0           0  \n",
      "1           0  \n",
      "2           1  \n",
      "3           0  \n",
      "4           0  \n",
      "5           1  \n",
      "6           1  \n",
      "7           0  \n",
      "8           0  \n",
      "9           0  \n"
     ]
    }
   ],
   "source": [
    "train = pd.read_json('Data/train.json')\n",
    "print(train.head(10))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}