Final model 1 - Best LB.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_uuid": "70cb45c46674b9c0f0bd2217e45b063e94110bcd"
   },
   "outputs": [],
   "source": [
    "SUBMISSION = True\n",
    "USE_RESCUER_CV = not SUBMISSION\n",
    "LB = not SUBMISSION\n",
    "SEED = 209321206\n",
    "VERBOSE = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "e8b7cd0a019b89ad9fb0b0ee6c638badb87f6321"
   },
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_kg_hide-input": false,
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "START_TIME = pd.datetime.now()\n",
    "\n",
    "import gc\n",
    "import glob\n",
    "import os\n",
    "import re\n",
    "import sys\n",
    "import ujson as json\n",
    "\n",
    "from copy import deepcopy\n",
    "from functools import partial\n",
    "\n",
    "import cv2\n",
    "import keras.backend as K\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "from catboost import CatBoostRegressor\n",
    "\n",
    "from IPython.core.display import display\n",
    "from joblib import Parallel,delayed\n",
    "\n",
    "from keras.models import Model\n",
    "from keras.layers import GlobalAveragePooling2D, GlobalMaxPooling2D, Input, Dense\n",
    "from keras.callbacks import EarlyStopping\n",
    "\n",
    "from lightgbm import LGBMRegressor,LGBMClassifier,plot_importance\n",
    "\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer,SnowballStemmer\n",
    "\n",
    "from xgboost import XGBRegressor,XGBClassifier # Not used\n",
    "\n",
    "from scipy.stats import mode\n",
    "from scipy import sparse\n",
    "\n",
    "from sklearn.base import BaseEstimator,RegressorMixin,TransformerMixin\n",
    "from sklearn.decomposition import PCA,TruncatedSVD\n",
    "from sklearn.ensemble import BaggingRegressor,ExtraTreesRegressor\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.linear_model import LinearRegression,Lasso,Ridge,ElasticNet,BayesianRidge,LassoCV,RidgeCV\n",
    "from sklearn.metrics import confusion_matrix,classification_report\n",
    "from sklearn.model_selection import StratifiedKFold,train_test_split,GroupKFold\n",
    "from sklearn.pipeline import FeatureUnion,make_union,make_pipeline,_fit_transform_one,_transform_one\n",
    "from sklearn.preprocessing import MinMaxScaler,LabelEncoder\n",
    "from sklearn.utils.multiclass import unique_labels\n",
    "\n",
    "from tqdm import tqdm_notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_kg_hide-input": true,
    "_uuid": "0754839f340512aa81c64bcfc89305a3dcd741ce"
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "from logging import StreamHandler\n",
    "\n",
    "class KaggleHandler(StreamHandler):\n",
    "    \n",
    "    def __init__(self):\n",
    "        StreamHandler.__init__(self)\n",
    "        \n",
    "    def emit(self, record):\n",
    "        msg_fmt = re.sub(r'[^0-9A-z\\.\\,\\-\\:\\=]', ' ', str(self.format(record)))\n",
    "        secs_passed = (pd.datetime.now() - START_TIME).seconds\n",
    "        msg = '[%ss] %s' % (secs_passed, msg_fmt)\n",
    "        os.system(f'echo {msg}')\n",
    "        sys.stdout.write(msg + '\\n')\n",
    "        \n",
    "logger = logging.getLogger(__name__)\n",
    "logger.handlers = []\n",
    "logger.setLevel(logging.DEBUG)\n",
    "fmt = logging.Formatter('[%(asctime)s]: %(message)s')\n",
    "kh = KaggleHandler()\n",
    "kh.setFormatter(fmt)\n",
    "logger.addHandler(kh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_uuid": "be4b3ddf2eb4aa745b554c9f9bff970fb8db2d71"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5s] [2019-03-26 17:34:57,428]: Seed set to 209321206\n"
     ]
    }
   ],
   "source": [
    "logger.info('Seed set to %d' % SEED)\n",
    "np.random.seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_uuid": "4d2f12d44bb7994add6381cfedf5e05ef9b36e7a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5s] [2019-03-26 17:34:57,447]: [ malaysian-state-data ,  model-weights ,  petfinder-adoption-prediction ]\n"
     ]
    }
   ],
   "source": [
    "logger.info(os.listdir(\"../input\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_uuid": "4195ac0d4ffef179ebc196be714fc020fdafb3e9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5s] [2019-03-26 17:34:57,466]: SUBMISSION=True, USE_RESCUER_CV=False, LB=False, VERBOSE=1\n"
     ]
    }
   ],
   "source": [
    "logger.info(f'SUBMISSION={SUBMISSION}, USE_RESCUER_CV={USE_RESCUER_CV}, LB={LB}, VERBOSE={VERBOSE}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "b6e522416e6d1d1b59c7bcab67c066db7fb4647b"
   },
   "source": [
    "# Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": false,
    "_uuid": "21c8506e1868f71e2b811f02edba4b92e093083a"
   },
   "outputs": [],
   "source": [
    "def qwk(predicted_ratings, actual_ratings, nb_ratings=5):\n",
    "    \"\"\"Calculate quadratic weighted kappa.\"\"\"\n",
    "    assert len(actual_ratings) == len(predicted_ratings) #\"Both the vectors should be of same length\n",
    "    actual_ratings = actual_ratings.astype(np.int64)\n",
    "    predicted_ratings = predicted_ratings.astype(np.int64)\n",
    "\n",
    "    # 1. Get the confusion matrix \n",
    "    conf_mtx = confusion_matrix(actual_ratings, predicted_ratings).astype(np.float64)\n",
    "\n",
    "    # 2. Create a weight matrix\n",
    "    weight_mtx = np.zeros((nb_ratings, nb_ratings))\n",
    "    for i in range(nb_ratings):\n",
    "        for j in range(nb_ratings):\n",
    "            weight_mtx[i][j] = (i-j)**2 / ((nb_ratings-1)**2)\n",
    "\n",
    "    # 3.Get the histograms for both the raters\n",
    "    actual_ratings_hist = np.bincount(actual_ratings,minlength=nb_ratings)\n",
    "    predicted_ratings_hist = np.bincount(predicted_ratings, minlength=nb_ratings)\n",
    "\n",
    "    # 4. Perform an outer product of the histograms\n",
    "    out_prod = np.outer(actual_ratings_hist, predicted_ratings_hist).astype(np.float64)\n",
    "\n",
    "    # 5. Normalize both- the confusion matrix and the outer product matrix\n",
    "    conf_mtx /= conf_mtx.sum()\n",
    "    out_prod /= out_prod.sum()\n",
    "\n",
    "    # Calculate the weighted kappa\n",
    "    numerator = (conf_mtx * weight_mtx).sum()\n",
    "    denominator = (out_prod * weight_mtx).sum()\n",
    "    score = (1 -(numerator/denominator))\n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_kg_hide-input": true,
    "_uuid": "cf36afbc063d45b53accf99a8c88b2edfe3072b1"
   },
   "outputs": [],
   "source": [
    "def get_rescuer_ids(df):\n",
    "    \"\"\"Return RescuerID indices and count.\"\"\"\n",
    "    rescuer_ids = (df.reset_index(drop=True).reset_index().groupby(['RescuerID']).agg({\n",
    "        'index': {'unique': lambda x: x.unique().tolist(), 'count': lambda x: x.nunique()},\n",
    "        'AdoptionSpeed': {'mean': lambda x: x.mean()}\n",
    "    }))\n",
    "    rescuer_ids.columns = rescuer_ids.columns.droplevel(0)\n",
    "    rescuer_ids['bucket'] = (\n",
    "        #pd.cut(rescuer_ids['mean'], bins=[-1,1,1.5,2,2.5,3,5]).cat.codes.astype(str) +\n",
    "        '1' +\n",
    "        pd.cut(rescuer_ids['count'], bins=[-1,2,4,10000]).cat.codes.astype(str)\n",
    "    )\n",
    "    rescuer_ids['count'] = rescuer_ids['bucket']\n",
    "    return rescuer_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_kg_hide-input": true,
    "_uuid": "04a10a0e192114d3c9e2af79e5df8ae3264bf654"
   },
   "outputs": [],
   "source": [
    "def find_nearest(array, value):\n",
    "    \"\"\"Find the nearest value from given array.\"\"\"\n",
    "    array = np.asarray(array)\n",
    "    idx = (np.abs(array - value)).argmin()\n",
    "    return idx\n",
    "\n",
    "def labelize_match_target(y_pred, y_train):\n",
    "    \"\"\"Match target distributions.\"\"\"\n",
    "    pdf_train = np.bincount(y_train)\n",
    "    pdf_train = pdf_train / pdf_train.sum()\n",
    "    cdf_train = np.cumsum(pdf_train)\n",
    "    \n",
    "    counts,bins = np.histogram(y_pred, bins=4000000)\n",
    "    cdf_pred = np.cumsum(counts / counts.sum())\n",
    "    #print(y_train,y_pred)\n",
    "    cutoffs = []\n",
    "    for cutpoint in cdf_train:\n",
    "        cutoff_idx = find_nearest(cdf_pred, cutpoint)\n",
    "        cutoff = bins[cutoff_idx]\n",
    "        cutoffs.append(cutoff)\n",
    "    cutoffs[-1] = 4\n",
    "    if VERBOSE:\n",
    "        logger.info('Target match cutoffs: %s' % cutoffs)\n",
    "    y_pred_rounded = np.zeros(y_pred.shape[0], dtype=int) + 4\n",
    "    min_cutoff = 0.\n",
    "    for label,cutoff in enumerate(cutoffs):\n",
    "        y_pred_rounded[(y_pred >= min_cutoff) & (y_pred <= cutoff)] = label\n",
    "        min_cutoff = cutoff\n",
    "    return y_pred_rounded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(y_true, y_pred,\n",
    "                          normalize=False,\n",
    "                          title=None,\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    classes = np.sort(y_true.unique())[::-1]\n",
    "    if not title:\n",
    "        if normalize:\n",
    "            title = 'Normalized confusion matrix'\n",
    "        else:\n",
    "            title = 'Confusion matrix, without normalization'\n",
    "\n",
    "    # Compute confusion matrix\n",
    "    cm = confusion_matrix(y_true, y_pred)\n",
    "    # Only use the labels that appear in the data\n",
    "    classes = classes[unique_labels(y_true, y_pred)]\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "\n",
    "    fig, ax = plt.subplots()\n",
    "    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    ax.figure.colorbar(im, ax=ax)\n",
    "    # We want to show all ticks...\n",
    "    ax.set(xticks=np.arange(cm.shape[1]),\n",
    "           yticks=np.arange(cm.shape[0]),\n",
    "           # ... and label them with the respective list entries\n",
    "           xticklabels=classes, yticklabels=classes,\n",
    "           title=title,\n",
    "           ylabel='True label',\n",
    "           xlabel='Predicted label')\n",
    "\n",
    "    # Rotate the tick labels and set their alignment.\n",
    "    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\",\n",
    "             rotation_mode=\"anchor\")\n",
    "\n",
    "    # Loop over data dimensions and create text annotations.\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i in range(cm.shape[0]):\n",
    "        for j in range(cm.shape[1]):\n",
    "            ax.text(j, i, format(cm[i, j], fmt),\n",
    "                    ha=\"center\", va=\"center\",\n",
    "                    color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "    fig.tight_layout()\n",
    "    plt.show()\n",
    "    \n",
    "    acc_str = ''\n",
    "    accs = (cm.diagonal() / cm.sum(axis=1)) * 100\n",
    "    for i in range(len(accs)):\n",
    "        logger.info('Class %s accuracy: %.1f %%' % (i,accs[i]))\n",
    "    return accs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "_kg_hide-input": true,
    "_uuid": "d614362223e58413c563a3727f0b6a961781955a"
   },
   "outputs": [],
   "source": [
    "def validate(X, y, rescuer_ids, n_cv=8, n_folds=5):\n",
    "    \"\"\"Validation framework.\"\"\"\n",
    "    df_preds = []\n",
    "    cv_scores = []\n",
    "    cv_accs = []\n",
    "    \n",
    "    global CV,FOLD\n",
    "        \n",
    "    for j in range(n_cv):\n",
    "        CV = j\n",
    "        skf = StratifiedKFold(n_folds, shuffle=True, random_state=SEED+j)\n",
    "        if USE_RESCUER_CV:\n",
    "            rescuer_idx = rescuer_ids['unique'].values\n",
    "            cv_gen = skf.split(rescuer_idx, rescuer_ids['count'].values)\n",
    "        else:\n",
    "            cv_gen = skf.split(X, y)\n",
    "        scores = []\n",
    "        accs = []\n",
    "        for i,(train_idx,test_idx) in enumerate(cv_gen):\n",
    "            FOLD = i\n",
    "            if USE_RESCUER_CV:\n",
    "                train_idx = np.array([idx for l in rescuer_idx[train_idx] for idx in l])\n",
    "                test_idx = np.array([idx for l in rescuer_idx[test_idx] for idx in l])\n",
    "\n",
    "            # Fit and predict\n",
    "            X_train,y_train = X.iloc[train_idx],y.iloc[train_idx]\n",
    "            X_test,y_test = X.iloc[test_idx],y.iloc[test_idx]\n",
    "                        \n",
    "            # All data\n",
    "            y_pred = run_all(X_train, y_train, X_test)         \n",
    "            score = qwk(y_pred, y_test)\n",
    "            scores.append(score)\n",
    "            \n",
    "            acc = [-1, -1, -1, -1, -1]\n",
    "            if VERBOSE:\n",
    "                print(classification_report(y_test, y_pred))\n",
    "                #print(confusion_matrix(y_test, y_pred))\n",
    "                acc = plot_confusion_matrix(y_test, y_pred)\n",
    "            accs.append(acc)\n",
    "            \n",
    "            logger.info('CV %d score: %.3f (running mean %.3f and std %.3f) - Running acc. %s' % \n",
    "                        (i+1, score, np.mean(scores), np.std(scores), np.around(np.array(accs).mean(axis=0), 1)))\n",
    "            \n",
    "            # Save results\n",
    "            df_pred = pd.DataFrame()\n",
    "            df_pred['Prediction'] = y_pred\n",
    "            df_pred['Run'] = j\n",
    "            df_pred['Fold'] = i\n",
    "            df_pred['PetID'] = train['PetID'].values[test_idx]\n",
    "            df_preds.append(df_pred)\n",
    "            \n",
    "        cv_scores.append(scores)\n",
    "        cv_accs.append(accs)\n",
    "        cv_means = np.mean(cv_scores, axis=1)\n",
    "        logger.info('> Running CV mean after %d runs: %.3f [%.3f - %.3f] (std %.3f) - Individual [%.3f - %.3f] - Accs: %s' %\n",
    "                    (j+1,\n",
    "                     np.mean(cv_means), np.min(cv_means), np.max(cv_means), np.std(cv_means), \n",
    "                     np.min(cv_scores), np.max(cv_scores), np.around(np.array(cv_accs).mean(axis=1).mean(axis=0), 1)))\n",
    "        \n",
    "    pd.concat(df_preds, axis=0).to_csv('df_preds.csv', index=False)\n",
    "    all_cv_mean = np.mean(cv_means)\n",
    "    return all_cv_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "_kg_hide-input": true,
    "_uuid": "e22494585793a99c804aaeba621051c679e3d4c3"
   },
   "outputs": [],
   "source": [
    "class PandasTransform(TransformerMixin, BaseEstimator):\n",
    "    \n",
    "    def __init__(self, fn):\n",
    "        self.fn = fn\n",
    "\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X, y=None, copy=None):\n",
    "        return self.fn(X)\n",
    "\n",
    "    \n",
    "class PandasFeatureUnion(FeatureUnion):\n",
    "    \n",
    "    def fit_transform(self, X, y=None, **fit_params):\n",
    "        self._validate_transformers()\n",
    "        result = Parallel(n_jobs=self.n_jobs)(\n",
    "            delayed(_fit_transform_one)(\n",
    "                transformer=trans,\n",
    "                X=X.copy(),\n",
    "                y=y,\n",
    "                weight=weight,\n",
    "                **fit_params)\n",
    "            for name, trans, weight in self._iter())\n",
    "\n",
    "        if not result:\n",
    "            # All transformers are None\n",
    "            return np.zeros((X.shape[0], 0))\n",
    "        Xs, transformers = zip(*result)\n",
    "        self._update_transformer_list(transformers)\n",
    "        if any(sparse.issparse(f) for f in Xs):\n",
    "            Xs = sparse.hstack(Xs).tocsr()\n",
    "        else:\n",
    "            Xs = self.merge_dataframes_by_column(Xs)\n",
    "        return Xs\n",
    "\n",
    "    def merge_dataframes_by_column(self, Xs):\n",
    "        return pd.concat(Xs, axis=\"columns\", copy=False)\n",
    "\n",
    "    def transform(self, X):\n",
    "        Xs = Parallel(n_jobs=self.n_jobs)(\n",
    "            delayed(_transform_one)(\n",
    "                transformer=trans,\n",
    "                X=X.copy(),\n",
    "                y=None,\n",
    "                weight=weight)\n",
    "            for name, trans, weight in self._iter())\n",
    "        if not Xs:\n",
    "            # All transformers are None\n",
    "            return np.zeros((X.shape[0], 0))\n",
    "        if any(sparse.issparse(f) for f in Xs):\n",
    "            Xs = sparse.hstack(Xs).tocsr()\n",
    "        else:\n",
    "            Xs = self.merge_dataframes_by_column(Xs)\n",
    "        return Xs\n",
    "    \n",
    "    \n",
    "class PandasPipeline(TransformerMixin, BaseEstimator):\n",
    "    \n",
    "    def __init__(self, steps=None):\n",
    "        self.steps = steps\n",
    "        \n",
    "        if self.steps is None:\n",
    "            self.steps = []\n",
    "        \n",
    "    def fit_transform(self, X, y=None, **fit_kwargs):\n",
    "        X = X.copy()\n",
    "        for name,trs in self.steps:\n",
    "            X = trs.fit_transform(X)\n",
    "        return X\n",
    "    \n",
    "    def transform(self, X):\n",
    "        X = X.copy()\n",
    "        for name,trs in self.steps:\n",
    "            X = trs.transform(X)\n",
    "        return X\n",
    "    \n",
    "    \n",
    "class FilterCommonValues(TransformerMixin, BaseEstimator):\n",
    "    \n",
    "    def __init__(self, train, test, cols, lim=5, fill_val=-1):\n",
    "        self.train = train\n",
    "        self.test = test\n",
    "        self.cols = cols\n",
    "        self.lim = lim\n",
    "        self.fill_val = fill_val\n",
    "            \n",
    "    def transform(self, X):\n",
    "        for col in self.cols:\n",
    "            if col in X:\n",
    "                X.loc[~X[col].isin(self.keep_vals_[col]), col] = self.fill_val\n",
    "        return X\n",
    "    \n",
    "    def fit_transform(self, X, y=None, **fit_kwargs):\n",
    "        self.keep_vals_ = {}\n",
    "        for col in self.cols:\n",
    "            lim_vals = (self.train[col].value_counts() < self.lim).replace(False, np.nan).dropna().index.tolist()\n",
    "            common_vals = list(set(self.test[col].unique()).intersection(self.train[col].unique()))\n",
    "            self.keep_vals_[col] = list(set(lim_vals + common_vals))\n",
    "        return self.transform(X)\n",
    "    \n",
    "    \n",
    "class SelectColumns(TransformerMixin, BaseEstimator):\n",
    "    \n",
    "    def __init__(self, items=None, like=None, regex=None):\n",
    "        self.items = items\n",
    "        self.like = like\n",
    "        self.regex = regex\n",
    "        \n",
    "    def fit_transform(self, X, y=None, **fit_params):\n",
    "        return self.transform(X)\n",
    "    \n",
    "    def transform(self, X):\n",
    "        X = X.filter(items=self.items, like=self.like,\n",
    "                        regex=self.regex, axis=1)\n",
    "        if X.shape[1] == 0:\n",
    "            logger.info('Cannot find anything with search %s, %s, %s' % (\n",
    "                  self.items,self.like,self.regex))\n",
    "        if VERBOSE > 1:\n",
    "            msg = X.columns.values if X.shape[1] < 50 else np.random.choice(X.columns.values.flatten(), size=50)\n",
    "            logger.info(f'Selected {X.shape[1]} columns (max 50 shown): {msg}')\n",
    "        return X\n",
    "    \n",
    "    \n",
    "class DropColumns(TransformerMixin, BaseEstimator):\n",
    "    \n",
    "    def __init__(self, items=None, like=None, regex=None):\n",
    "        self.items = items\n",
    "        self.like = like\n",
    "        self.regex = regex\n",
    "        \n",
    "    def fit_transform(self, X, y=None, **fit_params):\n",
    "        return self.transform(X)\n",
    "    \n",
    "    def transform(self, X):\n",
    "        drop_cols = X.iloc[0:0].filter(items=self.items, like=self.like,\n",
    "                        regex=self.regex, axis=1).columns.tolist()\n",
    "        X = X.drop(drop_cols, axis=1)\n",
    "        if VERBOSE > 1:\n",
    "            msg = X.columns.values if X.shape[1] < 100 else X.shape[1]\n",
    "            logger.info(f'Dropped to {X.shape[1]} columns: {msg}')\n",
    "        return X\n",
    "    \n",
    "    \n",
    "class ConvertType(TransformerMixin, BaseEstimator):\n",
    "    \n",
    "    def __init__(self, cols, dtype):\n",
    "        self.cols = cols\n",
    "        self.dtype = dtype\n",
    "        \n",
    "    def fit_transform(self, X, y=None, **fit_params):\n",
    "        return self.transform(X)\n",
    "    \n",
    "    def transform(self, X):\n",
    "        X[self.cols] = X[self.cols].astype(self.dtype)\n",
    "        return X\n",
    "    \n",
    "    \n",
    "class SelectType(TransformerMixin, BaseEstimator):\n",
    "    \n",
    "    def __init__(self, include=None, exclude=None):\n",
    "        self.include = include\n",
    "        self.exclude = exclude\n",
    "        \n",
    "    def fit_transform(self, X, y=None, **fit_kwargs):\n",
    "        return self.transform(X)\n",
    "    \n",
    "    def transform(self, X):\n",
    "        return X.select_dtypes(include=self.include, exclude=self.exclude)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "_kg_hide-input": true,
    "_uuid": "44b66eb224cdaf894774f1c83dcc079fabfa1838"
   },
   "outputs": [],
   "source": [
    "class StemmedTfidfVectorizer(TfidfVectorizer):\n",
    "    \n",
    "    def __init__(self, stemmer, *args, **kwargs):\n",
    "        super(StemmedTfidfVectorizer, self).__init__(*args, **kwargs)\n",
    "        self.stemmer = stemmer\n",
    "        \n",
    "    def build_analyzer(self):\n",
    "        analyzer = super(StemmedTfidfVectorizer, self).build_analyzer()\n",
    "        return lambda doc: (self.stemmer.stem(word) for word in analyzer(doc.replace('\\n', ' ')))\n",
    "\n",
    "    \n",
    "def grouped_feature(X, gr_col, agg_cols, agg_funcs):\n",
    "    gr = X.groupby(gr_col)[agg_cols].agg(agg_funcs)\n",
    "    gr.columns = [l1+'_'+'_'.join(gr_col)+'_'+l2 for l1,l2 in zip(gr.columns.get_level_values(0),\n",
    "                                             gr.columns.get_level_values(1))]\n",
    "    gr = gr.reset_index()\n",
    "    X = X.merge(gr, how='left', on=gr_col)\n",
    "    return X[gr.columns.drop(gr_col)]\n",
    "\n",
    "\n",
    "class StratifiedBagger(BaseEstimator,RegressorMixin):\n",
    "    \"\"\"Out-of-fold bagging with stratification.\"\"\"\n",
    "    \n",
    "    def __init__(self, clf=None, n_bags=10, random_state=None, labelize=False, early_stopping_rounds=None):\n",
    "        self.clf = clf\n",
    "        self.n_bags = n_bags\n",
    "        self.random_state = random_state if random_state is not None else SEED\n",
    "        self.labelize = labelize\n",
    "        self.early_stopping_rounds = early_stopping_rounds\n",
    "        \n",
    "    def fit(self, X, y=None, **fit_params):\n",
    "        skf = StratifiedKFold(self.n_bags, shuffle=True, random_state=self.random_state)\n",
    "        folds = skf.split(X, y)\n",
    "        self.clf_ = []\n",
    "        self.y_ = y\n",
    "        for i,(train_idx,valid_idx) in enumerate(folds):\n",
    "            if isinstance(X, pd.DataFrame):\n",
    "                (X_train,y_train,X_valid,y_valid) = (X.iloc[train_idx],y.iloc[train_idx],\n",
    "                                                     X.iloc[valid_idx],y.iloc[valid_idx])\n",
    "            else:\n",
    "                (X_train,y_train,X_valid,y_valid) = (X[train_idx],y.iloc[train_idx],\n",
    "                                                     X[valid_idx],y.iloc[valid_idx])\n",
    "            clf = deepcopy(self.clf)\n",
    "            self.clf_.append(clf)\n",
    "            if self.early_stopping_rounds is None:\n",
    "                self.clf_[i].fit(X_train, y_train, **fit_params)\n",
    "            else:\n",
    "                self.clf_[i].fit(X_train, y_train, eval_set=[(X_valid,y_valid)], \n",
    "                                 early_stopping_rounds=self.early_stopping_rounds, \n",
    "                                 verbose=False, **fit_params)\n",
    "                \n",
    "            if VERBOSE:\n",
    "                if isinstance(self.clf_[i], LGBMRegressor):\n",
    "                    fig,ax = plt.subplots(1, 1, figsize=(12,12))\n",
    "                    plot_importance(self.clf_[i], ax=ax)\n",
    "                    plt.show()\n",
    "                    \n",
    "                if isinstance(self.clf_[i], XGBRegressor):\n",
    "                    fig,ax = plt.subplots(1, 1, figsize=(12,12))\n",
    "                    xgb_plot_importance(self.clf_[i], ax=ax)\n",
    "                    plt.show()\n",
    "                    \n",
    "                if isinstance(self.clf_[i], CatBoostRegressor):\n",
    "                    # # Feature Importance\n",
    "                    fea_imp = pd.DataFrame({'imp': self.clf_[i].feature_importances_, \n",
    "                                            'col': X_train.columns})\n",
    "                    fea_imp = fea_imp.sort_values(['imp', 'col'], ascending=[True, False])\n",
    "                    fea_imp.plot(kind='barh', x='col', y='imp', figsize=(12,12))\n",
    "                    plt.show()\n",
    "        return self\n",
    "    \n",
    "    def predict(self, X, y=None):\n",
    "        y_preds = np.zeros((X.shape[0],self.n_bags))\n",
    "        for i in range(self.n_bags):\n",
    "            y_pred = minmax_scale(self.clf_[i].predict(X), (0,4))\n",
    "            if self.labelize:\n",
    "                y_pred = labelize_match_target(y_pred, self.y_)\n",
    "            y_preds[:,i] = y_pred\n",
    "        \n",
    "        if self.labelize:\n",
    "            y_preds = mode(y_preds.T)[0].T.flatten().astype(np.int32)\n",
    "        else:\n",
    "            y_preds = y_preds.mean(-1)\n",
    "        return y_preds\n",
    "    \n",
    "    \n",
    "class OOFTransformer(TransformerMixin, BaseEstimator):\n",
    "    \"\"\"Make regression out-of-fold predictions.\"\"\"\n",
    "    \n",
    "    def __init__(self, clf, n_folds=5, agg_func='mean', name=None):\n",
    "        self.clf = clf\n",
    "        self.n_folds = n_folds\n",
    "        self.agg_func = agg_func\n",
    "        self.name = name\n",
    "        \n",
    "        if self.name is None:\n",
    "            self.name = str(self.clf.__class__).split('.')[-1][1:-2]\n",
    "        \n",
    "    def fit_transform(self, X, y, **fit_kwargs):\n",
    "        self.name_ = 'oof_' + self.name\n",
    "        self.clf_ = []\n",
    "        skf = StratifiedKFold(self.n_folds, shuffle=True, random_state=SEED+1234)\n",
    "        folds = skf.split(X, y)\n",
    "        X_train = np.zeros((X.shape[0],))\n",
    "        \n",
    "        if not SUBMISSION and not USE_LB:\n",
    "            path = f'{self.name_}_{CV}_{FOLD}_X_train.csv'\n",
    "            if os.path.exists(path) and os.path.exists(path.replace('X_train','X_test')):\n",
    "                if VERBOSE:\n",
    "                    logger.info(f'Read existing for {self.name_} (fit)')\n",
    "                return pd.read_csv(path)\n",
    "        \n",
    "        for i,(fold_idx,oof_idx) in enumerate(folds):\n",
    "            self.clf_.append(deepcopy(self.clf))\n",
    "            if VERBOSE:\n",
    "                logger.info(f'Generating {self.name_} oof {i+1}/{self.n_folds}')\n",
    "            if isinstance(X, pd.DataFrame):\n",
    "                (X_fold,y_fold,X_oof,y_oof) = (X.iloc[fold_idx],y.iloc[fold_idx],\n",
    "                                               X.iloc[oof_idx],y.iloc[oof_idx])\n",
    "            else:\n",
    "                (X_fold,y_fold,X_oof,y_oof) = (X[fold_idx],y.iloc[fold_idx],\n",
    "                                               X[oof_idx],y.iloc[oof_idx])\n",
    "            self.clf_[i].fit(X_fold, y_fold)\n",
    "            X_train[oof_idx] = self.clf_[i].predict(X_oof)\n",
    "            \n",
    "            if VERBOSE:\n",
    "                if isinstance(self.clf_[i], LGBMRegressor):\n",
    "                    fig,ax = plt.subplots(1, 1, figsize=(18,18))\n",
    "                    plot_importance(self.clf_[i], ax=ax)\n",
    "                    plt.show()\n",
    "                    \n",
    "                if isinstance(self.clf_[i], XGBRegressor):\n",
    "                    fig,ax = plt.subplots(1, 1, figsize=(18,18))\n",
    "                    xgb_plot_importance(self.clf_[i], ax=ax)\n",
    "                    plt.show()\n",
    "                    \n",
    "                if isinstance(self.clf_[i], CatBoostRegressor):\n",
    "                    # # Feature Importance\n",
    "                    fea_imp = pd.DataFrame({'imp': self.clf_[i].feature_importances_, \n",
    "                                            'col': X_train.columns})\n",
    "                    fea_imp = fea_imp.sort_values(['imp', 'col'], ascending=[True, False]).iloc[-30:]\n",
    "                    fea_imp.plot(kind='barh', x='col', y='imp', figsize=(18,18))\n",
    "                    plt.show()\n",
    "                    \n",
    "        X_train = pd.DataFrame(X_train, columns=[self.name_])\n",
    "        \n",
    "        if not SUBMISSION and not USE_LB:\n",
    "            X_train.to_csv(path, index=False)\n",
    "        \n",
    "        return X_train\n",
    "    \n",
    "    def transform(self, X):\n",
    "        if not SUBMISSION and not USE_LB:\n",
    "            path = f'{self.name_}_{CV}_{FOLD}_X_test.csv'\n",
    "            if os.path.exists(path):\n",
    "                if VERBOSE:\n",
    "                    print(f'Read existing for {self.name_} (transform)')\n",
    "                return pd.read_csv(path)\n",
    "        \n",
    "        X_test = np.zeros((X.shape[0],self.n_folds))\n",
    "        for i in range(self.n_folds):\n",
    "            X_test[:,i] = self.clf_[i].predict(X)\n",
    "        if self.agg_func == 'mean':\n",
    "            X_test = X_test.mean(-1)\n",
    "        elif self.agg_func == 'median':\n",
    "            X_test = np.median(X_test, axis=-1)\n",
    "        else:\n",
    "            raise ValueError(f'Unknown aggregation {self.agg_func}')\n",
    "        X_test = pd.DataFrame(X_test, columns=[self.name_])\n",
    "        \n",
    "        if not SUBMISSION and not USE_LB:\n",
    "            X_test.to_csv(path, index=False)\n",
    "        \n",
    "        return X_test\n",
    "\n",
    "    \n",
    "class PredictTransformer(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    def __init__(self, clf, labelize=True, name=None):\n",
    "        self.clf = clf\n",
    "        self.labelize = labelize\n",
    "        self.name = name\n",
    "        \n",
    "        if self.name is None:\n",
    "            self.name = str(self.clf.__class__).split('.')[-1][1:-2]\n",
    "        \n",
    "    def fit(self, X, y, **fit_kwargs):\n",
    "        self.name_ = 'pred_' + self.name\n",
    "        \n",
    "        self.clf.fit(X, y, **fit_kwargs)\n",
    "        if VERBOSE:\n",
    "            print(f'Fitting {self.name_}...')\n",
    "            if isinstance(self.clf, LGBMRegressor):\n",
    "                fig,ax = plt.subplots(1, 1, figsize=(18,18))\n",
    "                plot_importance(self.clf, ax=ax)\n",
    "                plt.show()\n",
    "\n",
    "            if isinstance(self.clf, XGBRegressor):\n",
    "                fig,ax = plt.subplots(1, 1, figsize=(18,18))\n",
    "                xgb_plot_importance(self.clf, ax=ax)\n",
    "                plt.show()\n",
    "\n",
    "            if isinstance(self.clf, CatBoostRegressor):\n",
    "                # # Feature Importance\n",
    "                fea_imp = pd.DataFrame({'imp': self.clf.feature_importances_, \n",
    "                                        'col': X_train.columns})\n",
    "                fea_imp = fea_imp.sort_values(['imp', 'col'], ascending=[True, False]).iloc[-30:]\n",
    "                fea_imp.plot(kind='barh', x='col', y='imp', figsize=(18,18))\n",
    "                plt.show()\n",
    "        y_pred = self.clf.predict(X)\n",
    "        \n",
    "        if self.labelize:\n",
    "            self.y_ = y\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        y_pred = self.clf.predict(X)\n",
    "        if self.labelize:\n",
    "            y_pred = labelize_match_target(y_pred, self.y_)\n",
    "        return pd.DataFrame(y_pred, columns=[self.name_])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "c7166b171cd28092842490d32669bee4398b56ef"
   },
   "source": [
    "# Features - Raw data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "_uuid": "1e3d16af7088d7a372207f457c1c548f72028990"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5s] [2019-03-26 17:34:57,598]: Raw data...\n"
     ]
    }
   ],
   "source": [
    "logger.info('Raw data...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "breed = pd.read_csv('../input/petfinder-adoption-prediction/breed_labels.csv')\n",
    "color = pd.read_csv('../input/petfinder-adoption-prediction/color_labels.csv')\n",
    "state = pd.read_csv('../input/petfinder-adoption-prediction/state_labels.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "_uuid": "6b888bc219984c3bcadb0508db46fe38de7b1404"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5s] [2019-03-26 17:34:57,790]: Train shape:  14993, 24 \n",
      "[5s] [2019-03-26 17:34:57,800]: Test shape:  3948, 23 \n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv('../input/petfinder-adoption-prediction/train/train.csv')\n",
    "test = pd.read_csv('../input/petfinder-adoption-prediction/test/test.csv')\n",
    "logger.info('Train shape: %s' % str(train.shape))\n",
    "logger.info('Test shape: %s' % str(test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_uuid": "5336036f15219bbe5dbaf957103a882eb2f20cba"
   },
   "outputs": [],
   "source": [
    "# State information\n",
    "ext_state = pd.read_excel('../input/malaysian-state-data/external_state_data.xlsx', \n",
    "                          decimal=',')[['State','Population2','2016 GDP']]\n",
    "train = train.merge(ext_state, how='left', on='State')\n",
    "test = test.merge(ext_state, how='left', on='State')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "_uuid": "ddddad615587e0cd5ef288b8aded5c55f1ddfad9"
   },
   "outputs": [],
   "source": [
    "train['Description'] = train['Description'].fillna('').astype(str)\n",
    "test['Description'] = test['Description'].fillna('').astype(str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "561eb95f9f2de572c7e9bbf4c25db8bc1c4d15df"
   },
   "source": [
    "# Features - Metadata and sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "_uuid": "b58907e8e99950e7f952e5a0ca51e5709806bddb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6s] [2019-03-26 17:34:58,078]: Metadata and sentiment features...\n"
     ]
    }
   ],
   "source": [
    "logger.info('Metadata and sentiment features...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "_uuid": "334da1ccc42ab7f255c9115b794ab42b254c0ed9"
   },
   "outputs": [],
   "source": [
    "# Add vertex_y and label_score for profile picture (-1.jpg)\n",
    "def create_metadata(pet_id, train=True):\n",
    "    all_res = {}\n",
    "    for pic in range(1,2):\n",
    "        meta_path = '../input/petfinder-adoption-prediction/%s_metadata/%s-%s.json' % ('train' if train else 'test', pet_id,pic)\n",
    "        pic_path = '../input/petfinder-adoption-prediction/%s_images/%s-%s.jpg' % ('train' if train else 'test', pet_id,pic)\n",
    "        res = {\n",
    "            'PetID': pet_id, \n",
    "            f'vertex_y{pic}': 0, \n",
    "            f'label_score{pic}': 0,\n",
    "        }\n",
    "        try:\n",
    "            if os.path.exists(meta_path):\n",
    "                with open(meta_path, 'r') as f:\n",
    "                    d = json.load(f)\n",
    "                res[f'vertex_y{pic}'] = d['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']\n",
    "                if 'labelAnnotations' in d: \n",
    "                    res[f'label_score{pic}'] = d['labelAnnotations'][0]['score'] + 1\n",
    "            all_res.update(res)\n",
    "        except:\n",
    "            logger.warning('Failed loading JSON files!')\n",
    "            pass\n",
    "    return all_res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "_uuid": "8a99b78f8948e665ca6d93d279558bbddbed884a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6s] [2019-03-26 17:34:58,109]: Creating metadata features...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f46a0f1b7ab44e1bb527f62ed48e54c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=14993), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99d0288b0a12466987f75d83e4e3abde",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3948), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[18s] [2019-03-26 17:35:10,346]: Merging metadata features...\n",
      "[18s] [2019-03-26 17:35:10,500]: Metadata features done \n"
     ]
    }
   ],
   "source": [
    "logger.info('Creating metadata features...')\n",
    "train_metadata_feats = Parallel(n_jobs=4)(delayed(create_metadata)(pet_id)\n",
    "                                for pet_id in tqdm_notebook(train['PetID'].unique()))\n",
    "train_metadata_feats = pd.DataFrame(train_metadata_feats)\n",
    "\n",
    "create_metadata_test = partial(create_metadata, train=False)\n",
    "test_metadata_feats = Parallel(n_jobs=4)(delayed(create_metadata_test)(pet_id)\n",
    "                                         for pet_id in tqdm_notebook(test['PetID'].unique()))\n",
    "test_metadata_feats = pd.DataFrame(test_metadata_feats)\n",
    "        \n",
    "logger.info('Merging metadata features...')\n",
    "train = train.merge(train_metadata_feats, how='left', on='PetID')\n",
    "test = test.merge(test_metadata_feats, how='left', on='PetID')\n",
    "\n",
    "del train_metadata_feats,test_metadata_feats\n",
    "gc.collect()\n",
    "logger.info('Metadata features done!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "0c761893f2b7779e4cac3a4cad0b6f21df515f27"
   },
   "source": [
    "# Features - Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "_uuid": "9c1890c93edf2633c30df30802ebdc9701f39afa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[18s] [2019-03-26 17:35:10,570]: Image features...\n"
     ]
    }
   ],
   "source": [
    "logger.info('Image features...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "_uuid": "3d0be164483fd9a82c88ccc969c1bfd9fdbde531"
   },
   "outputs": [],
   "source": [
    "def resize_to_square(im, image_size):\n",
    "    old_size = im.shape[:2] # old_size is in (height, width) format\n",
    "    ratio = float(image_size)/max(old_size)\n",
    "    new_size = tuple([int(x*ratio) for x in old_size])\n",
    "    # new_size should be in (width, height) format\n",
    "    im = cv2.resize(im, (new_size[1], new_size[0]))\n",
    "    delta_w = image_size - new_size[1]\n",
    "    delta_h = image_size - new_size[0]\n",
    "    top, bottom = delta_h//2, delta_h-(delta_h//2)\n",
    "    left, right = delta_w//2, delta_w-(delta_w//2)\n",
    "    color = [0, 0, 0]\n",
    "    new_im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT,value=color)\n",
    "    return new_im\n",
    "\n",
    "def normal(img, target_size):\n",
    "    img = resize_to_square(img, target_size).astype(np.float32)\n",
    "    return img\n",
    "\n",
    "def load_image(path, target_size, backbone_preprocess_func, preprocess_func):\n",
    "    img = cv2.imread(path)\n",
    "    img = preprocess_func(img, target_size)\n",
    "    img = backbone_preprocess_func(img)\n",
    "    return img[None,:,:,:]\n",
    "\n",
    "def image_generator(paths, target_size, backbone_preprocess_func, \n",
    "                    preprocess_func, batch_size):\n",
    "    n_batches = int(np.ceil(len(paths) / batch_size))\n",
    "    yield n_batches\n",
    "    for i in range(n_batches):\n",
    "        batch_paths = paths[i*batch_size:(i+1)*batch_size]\n",
    "        yield np.concatenate([load_image(path, target_size, backbone_preprocess_func, preprocess_func)\n",
    "                              for path in batch_paths], axis=0)\n",
    "    \n",
    "def get_feature_extractor(backbone, target_size, pooling):\n",
    "    inp = Input((target_size,target_size,3))\n",
    "    \n",
    "    if backbone == 'xception':\n",
    "        from keras.applications.xception import Xception\n",
    "        backbone = Xception(weights=None, include_top=False, input_tensor=inp)\n",
    "        backbone.load_weights('../input/model-weights/xception_weights_tf_dim_ordering_tf_kernels_notop.h5')\n",
    "    elif backbone == 'nasnetlarge':\n",
    "        from keras.applications.nasnet import NASNetLarge\n",
    "        backbone = NASNetLarge(weights=None, include_top=False, input_tensor=inp)\n",
    "        backbone.load_weights('../input/model-weights/nasnet_large_no_top.h5')\n",
    "    else:\n",
    "        raise ValueError('Unknown backbone %s!' % backbone)        \n",
    "    x = backbone.output\n",
    "    if pooling == 'max':\n",
    "        x = GlobalMaxPooling2D()(x)\n",
    "    else:\n",
    "        x = GlobalAveragePooling2D()(x)\n",
    "    m = Model(inputs=[inp], outputs=[x])\n",
    "    return m\n",
    "\n",
    "def create_features(preprocess_func, backbone, pooling, batch_size=128):\n",
    "    train_img_paths = glob.glob('../input/petfinder-adoption-prediction/train_images/*-1.jpg')\n",
    "    test_img_paths = glob.glob('../input/petfinder-adoption-prediction/test_images/*-1.jpg')\n",
    "    img_paths = train_img_paths + test_img_paths\n",
    "    \n",
    "    logger.info('Loading backbone %s...' % backbone)\n",
    "    if backbone == 'xception':\n",
    "        from keras.applications.xception import preprocess_input\n",
    "        target_size = 299\n",
    "    elif backbone == 'nasnetlarge':\n",
    "        from keras.applications.nasnet import preprocess_input\n",
    "        target_size = 331\n",
    "    else:\n",
    "        raise ValueError('Unknown backbone %s!' % backbone)\n",
    "    backbone_preprocess_func = preprocess_input\n",
    "    feature_extractor = get_feature_extractor(backbone, target_size, pooling)\n",
    "    \n",
    "    logger.info('Creating features...')\n",
    "    img_gen = image_generator(img_paths, target_size, backbone_preprocess_func,\n",
    "                              preprocess_func, batch_size)\n",
    "    n_batches = next(img_gen)\n",
    "    pet_ids = [e.split('/')[-1].split('-')[0] for e in img_paths]\n",
    "    features = feature_extractor.predict_generator(\n",
    "        img_gen, max_queue_size=10, steps=n_batches, verbose=1).astype(np.float32)\n",
    "    logger.info('Features shape: %s' % str(features.shape))\n",
    "        \n",
    "    logger.info('Saving features...')\n",
    "    n_train = len(train_img_paths)\n",
    "    preprocess_func_name = preprocess_func.__name__\n",
    "    feature_name = f'img_{preprocess_func_name}_{backbone}_{pooling}'\n",
    "    colnames = [f'{feature_name}_{i+1}' for i in range(features.shape[1])]\n",
    "    train_features = (pd.DataFrame(features[:n_train], index=pet_ids[:n_train], columns=colnames)\n",
    "                      .reset_index().rename(columns={'index': 'PetID'}))\n",
    "    test_features = (pd.DataFrame(features[n_train:], index=pet_ids[n_train:], columns=colnames)\n",
    "                     .reset_index().rename(columns={'index': 'PetID'}))\n",
    "    \n",
    "    logger.info('Feature extraction done')\n",
    "    return train_features,test_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "_uuid": "1ec8652ab4aa9d43a600e81de95a0741233e4b9b",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[18s] [2019-03-26 17:35:10,611]: Creating image features...\n",
      "[19s] [2019-03-26 17:35:11,064]: Loading backbone xception...\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "[28s] [2019-03-26 17:35:20,321]: Creating features...\n",
      "145/145 [==============================] - 133s 917ms/step\n",
      "[161s] [2019-03-26 17:37:33,555]: Features shape:  18472, 2048 \n",
      "[161s] [2019-03-26 17:37:33,597]: Saving features...\n",
      "[162s] [2019-03-26 17:37:34,458]: Feature extraction done\n",
      "[162s] [2019-03-26 17:37:34,639]: Loading backbone nasnetlarge...\n",
      "[215s] [2019-03-26 17:38:27,239]: Creating features...\n",
      "145/145 [==============================] - 356s 2s/step\n",
      "[571s] [2019-03-26 17:44:23,620]: Features shape:  18472, 4032 \n",
      "[571s] [2019-03-26 17:44:23,687]: Saving features...\n",
      "[572s] [2019-03-26 17:44:24,875]: Feature extraction done\n",
      "[573s] [2019-03-26 17:44:25,358]: Merging image features...\n",
      "[575s] [2019-03-26 17:44:27,528]: Image features done \n"
     ]
    }
   ],
   "source": [
    "logger.info('Creating image features...')\n",
    "train_img_feats,test_img_feats = create_features(normal, 'xception', 'avg')\n",
    "\n",
    "tmp_train,tmp_test = create_features(normal, 'nasnetlarge', 'avg')\n",
    "train_img_feats = train_img_feats.merge(tmp_train, how='left', on='PetID')\n",
    "test_img_feats = test_img_feats.merge(tmp_test, how='left', on='PetID')\n",
    "        \n",
    "logger.info('Merging image features...')\n",
    "train = train.merge(train_img_feats, how='left', on='PetID')\n",
    "test = test.merge(test_img_feats, how='left', on='PetID')\n",
    "\n",
    "del train_img_feats,test_img_feats\n",
    "gc.collect()\n",
    "logger.info('Image features done!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "09bd7e7bd8f6d7ee98381844fe88ee7020662332"
   },
   "source": [
    "# Features - Custom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "_uuid": "f98db27c06a7c0c209cb1bb938fed7fe460d868c"
   },
   "outputs": [],
   "source": [
    "def row_features(X):\n",
    "    \"\"\"Row level features.\"\"\"\n",
    "    \n",
    "    # Adoption or adopted words found in text\n",
    "    X['adopted'] = 1\n",
    "    X.loc[X['Description'].str.lower().str.contains('adopted'),'adopted'] = 0\n",
    "    X.loc[X['Description'].str.lower().str.contains('adoption'),'adopted'] = 2\n",
    "        \n",
    "    # RescuerID has more than one pet\n",
    "    rescuer_counts = (\n",
    "        X[['RescuerID','PetID']]\n",
    "        .groupby('RescuerID').agg({'PetID': lambda x: (x.count() > 1).astype(np.int8)})\n",
    "        .rename(columns={'PetID': 'MoreThanOneRescuerPet'})\n",
    "        .reset_index()\n",
    "    )\n",
    "    X = X.merge(rescuer_counts, how='left', on='RescuerID')\n",
    "    return X[['adopted','MoreThanOneRescuerPet']]\n",
    "\n",
    "\n",
    "class ExtractLeaky(TransformerMixin, BaseEstimator):\n",
    "    \"\"\"Leaky features that will not be calculated before validation splits.\"\"\"\n",
    "    \n",
    "    def __init__(self, train, test):\n",
    "        self.train = train\n",
    "        self.test = test\n",
    "        \n",
    "    def _ratio(self, gr1, gr2, agg_col='PetID', agg_func='count'):\n",
    "        req_cols = list(set(gr1 + gr2 + [agg_col]))\n",
    "        df = pd.concat([self.train[req_cols], self.test[req_cols]], axis=0)\n",
    "        df1 = df.groupby(gr1)[agg_col].agg(agg_func).to_frame('gr1').reset_index()\n",
    "        df2 = df.groupby(gr2)[agg_col].agg(agg_func).to_frame('gr2').reset_index()\n",
    "        df = df.merge(df1, how='left', on=gr1).merge(df2, how='left', on=gr2)\n",
    "\n",
    "        colname = '%s_%s_%s_%s' % (agg_col,agg_func,'-'.join(gr1),'-'.join(gr2))\n",
    "        df[colname] = (df['gr1'] / df['gr2']).replace([-np.inf,np.inf],np.nan)\n",
    "        df = df.drop(['gr1','gr2'], axis=1)\n",
    "\n",
    "        self.train_feats_ = pd.concat([\n",
    "            self.train_feats_, df[colname].iloc[:self.n_train_]], axis=1)\n",
    "        self.test_feats_ = pd.concat([\n",
    "            self.test_feats_, df[colname].iloc[self.n_train_:].reset_index(drop=True)], axis=1)\n",
    "    \n",
    "    def fit_transform(self, X, y=None, **fit_kwargs):\n",
    "        self.n_train_ = self.train.shape[0]\n",
    "        self.train_feats_ = pd.DataFrame()\n",
    "        self.test_feats_ = pd.DataFrame()\n",
    "        \n",
    "        # Ratio features, (leaky!)\n",
    "        self._ratio(['Breed1'], ['State'], 'PetID', 'count')\n",
    "        self._ratio(['Breed1','RescuerID'], ['PetID'], 'PetID', 'count')\n",
    "        self._ratio(['PetID'], ['Breed1'], 'PhotoAmt', 'mean')\n",
    "        self._ratio(['RescuerID'], ['PetID'], 'Breed1', 'nunique')\n",
    "        self.train_feats_['Breed1_nunique_RescuerID_PetID'] = (\n",
    "            self.train_feats_['Breed1_nunique_RescuerID_PetID'].clip(1, self.test_feats_['Breed1_nunique_RescuerID_PetID'].max()))\n",
    "        self.test_feats_['Breed1_nunique_RescuerID_PetID'] = (\n",
    "            self.test_feats_['Breed1_nunique_RescuerID_PetID'].clip(1, self.test_feats_['Breed1_nunique_RescuerID_PetID'].max()))\n",
    "        return self.train_feats_\n",
    "    \n",
    "    def transform(self, X):\n",
    "        return self.test_feats_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "239de4ec203e1b6d253e165152c9b74f0f229d38"
   },
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "e9219f09b6e459dcc2e526b134da276761b40d60"
   },
   "source": [
    "# Features - Finishing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "_uuid": "394d9f1efdba1f8358eb2327ca2bb5bd92325d1c"
   },
   "outputs": [],
   "source": [
    "target = 'AdoptionSpeed'\n",
    "common_cols = test.columns.intersection(train.columns).values.tolist()\n",
    "train = train[common_cols + [target]]\n",
    "test = test[common_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "_uuid": "20dc202099255b5410e57a2324caef2b7c46dc29",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/pandas/core/groupby/groupby.py:4656: FutureWarning: using a dict with renaming is deprecated and will be removed in a future version\n",
      "  return super(DataFrameGroupBy, self).aggregate(arg, *args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "rescuer_ids = get_rescuer_ids(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "_uuid": "8d4fb486b9c696dbfbc2293b52e5374d38993a4e"
   },
   "outputs": [],
   "source": [
    "# Take \"leaderboard\" out, if needed\n",
    "if LB:\n",
    "    train_idx,test_idx = train_test_split(rescuer_ids['unique'], \n",
    "        stratify=rescuer_ids['count'], test_size=0.2, random_state=209321206+73)\n",
    "    train_idx = [idx for sidx in train_idx for idx in sidx]\n",
    "    test_idx = [idx for sidx in test_idx for idx in sidx]\n",
    "    X_train,y_train,X_test,y_test = (train.iloc[train_idx].drop('AdoptionSpeed', axis=1),\n",
    "                                     train.iloc[train_idx][target],\n",
    "                                     train.iloc[test_idx].drop('AdoptionSpeed', axis=1),\n",
    "                                     train.iloc[test_idx][target])\n",
    "    rescuer_ids = get_rescuer_ids(train.iloc[train_idx]) # Update for these indices\n",
    "else:\n",
    "    X_train,y_train,X_test = train.drop('AdoptionSpeed', axis=1),train[target],test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "f451eea57a8f57487b31818019484d6e80c2acd3"
   },
   "source": [
    "# Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "_uuid": "57ed6c967e1176f744f18a9c01fdb36e1ae3e29f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[578s] [2019-03-26 17:44:30,624]:    Fitting basic pipeline...\n",
      "[582s] [2019-03-26 17:44:34,642]:    Level 0...\n",
      "[646s] [2019-03-26 17:45:38,226]: Generating oof_lvl0_img_nasnet_pca400_lr oof 1 5\n",
      "[671s] [2019-03-26 17:46:03,777]: Generating oof_lvl0_img_nasnet_pca400_lr oof 2 5\n",
      "[695s] [2019-03-26 17:46:27,503]: Generating oof_lvl0_img_nasnet_pca400_lr oof 3 5\n",
      "[719s] [2019-03-26 17:46:51,689]: Generating oof_lvl0_img_nasnet_pca400_lr oof 4 5\n",
      "[744s] [2019-03-26 17:47:16,728]: Generating oof_lvl0_img_nasnet_pca400_lr oof 5 5\n",
      "[770s] [2019-03-26 17:47:42,128]: Generating oof_lvl0_img_xception_svd150_pca400_lr oof 1 5\n",
      "[825s] [2019-03-26 17:48:37,083]: Generating oof_lvl0_img_xception_svd150_pca400_lr oof 2 5\n",
      "[879s] [2019-03-26 17:49:31,313]: Generating oof_lvl0_img_xception_svd150_pca400_lr oof 3 5\n",
      "[934s] [2019-03-26 17:50:26,626]: Generating oof_lvl0_img_xception_svd150_pca400_lr oof 4 5\n",
      "[989s] [2019-03-26 17:51:21,463]: Generating oof_lvl0_img_xception_svd150_pca400_lr oof 5 5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oof_lvl0_img_nasnet_pca400_lr</th>\n",
       "      <th>oof_lvl0_img_xception_svd150_pca400_lr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>oof_lvl0_img_nasnet_pca400_lr</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.650469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oof_lvl0_img_xception_svd150_pca400_lr</th>\n",
       "      <td>0.650469</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        oof_lvl0_img_nasnet_pca400_lr  oof_lvl0_img_xception_svd150_pca400_lr\n",
       "oof_lvl0_img_nasnet_pca400_lr                                1.000000                                0.650469\n",
       "oof_lvl0_img_xception_svd150_pca400_lr                       0.650469                                1.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oof_lvl0_img_nasnet_pca400_lr</th>\n",
       "      <th>oof_lvl0_img_xception_svd150_pca400_lr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>oof_lvl0_img_nasnet_pca400_lr</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.679165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oof_lvl0_img_xception_svd150_pca400_lr</th>\n",
       "      <td>0.679165</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        oof_lvl0_img_nasnet_pca400_lr  oof_lvl0_img_xception_svd150_pca400_lr\n",
       "oof_lvl0_img_nasnet_pca400_lr                                1.000000                                0.679165\n",
       "oof_lvl0_img_xception_svd150_pca400_lr                       0.679165                                1.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1051s] [2019-03-26 17:52:23,661]:    Level 1...\n",
      "[1052s] [2019-03-26 17:52:24,244]: Generating oof_lvl1_lgb_non_img oof 1 5\n",
      "[1052s] [2019-03-26 17:52:24,772]: Generating oof_lvl1_lgb_non_img oof 2 5\n",
      "[1053s] [2019-03-26 17:52:25,352]: Generating oof_lvl1_lgb_non_img oof 3 5\n",
      "[1053s] [2019-03-26 17:52:25,874]: Generating oof_lvl1_lgb_non_img oof 4 5\n",
      "[1054s] [2019-03-26 17:52:26,445]: Generating oof_lvl1_lgb_non_img oof 5 5\n",
      "[1055s] [2019-03-26 17:52:27,224]: Generating oof_lvl1_lgb_img_text oof 1 5\n",
      "[1055s] [2019-03-26 17:52:27,787]: Generating oof_lvl1_lgb_img_text oof 2 5\n",
      "[1056s] [2019-03-26 17:52:28,406]: Generating oof_lvl1_lgb_img_text oof 3 5\n",
      "[1057s] [2019-03-26 17:52:28,988]: Generating oof_lvl1_lgb_img_text oof 4 5\n",
      "[1057s] [2019-03-26 17:52:29,613]: Generating oof_lvl1_lgb_img_text oof 5 5\n",
      "[1058s] [2019-03-26 17:52:30,438]: Generating oof_lvl1_lgb_bagging oof 1 5\n",
      "[1061s] [2019-03-26 17:52:33,863]: Generating oof_lvl1_lgb_bagging oof 2 5\n",
      "[1065s] [2019-03-26 17:52:37,789]: Generating oof_lvl1_lgb_bagging oof 3 5\n",
      "[1069s] [2019-03-26 17:52:41,766]: Generating oof_lvl1_lgb_bagging oof 4 5\n",
      "[1073s] [2019-03-26 17:52:45,604]: Generating oof_lvl1_lgb_bagging oof 5 5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oof_lvl1_lgb_non_img</th>\n",
       "      <th>oof_lvl1_lgb_img_text</th>\n",
       "      <th>oof_lvl1_lgb_bagging</th>\n",
       "      <th>oof_lvl1_lgb_bagging_RescuerID_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>oof_lvl1_lgb_non_img</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.506319</td>\n",
       "      <td>0.807791</td>\n",
       "      <td>0.686448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oof_lvl1_lgb_img_text</th>\n",
       "      <td>0.506319</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.841682</td>\n",
       "      <td>0.628129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oof_lvl1_lgb_bagging</th>\n",
       "      <td>0.807791</td>\n",
       "      <td>0.841682</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.810797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oof_lvl1_lgb_bagging_RescuerID_mean</th>\n",
       "      <td>0.686448</td>\n",
       "      <td>0.628129</td>\n",
       "      <td>0.810797</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     oof_lvl1_lgb_non_img                 ...                   oof_lvl1_lgb_bagging_RescuerID_mean\n",
       "oof_lvl1_lgb_non_img                             1.000000                 ...                                              0.686448\n",
       "oof_lvl1_lgb_img_text                            0.506319                 ...                                              0.628129\n",
       "oof_lvl1_lgb_bagging                             0.807791                 ...                                              0.810797\n",
       "oof_lvl1_lgb_bagging_RescuerID_mean              0.686448                 ...                                              1.000000\n",
       "\n",
       "[4 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oof_lvl1_lgb_non_img</th>\n",
       "      <th>oof_lvl1_lgb_img_text</th>\n",
       "      <th>oof_lvl1_lgb_bagging</th>\n",
       "      <th>oof_lvl1_lgb_bagging_RescuerID_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>oof_lvl1_lgb_non_img</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.512709</td>\n",
       "      <td>0.822752</td>\n",
       "      <td>0.683655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oof_lvl1_lgb_img_text</th>\n",
       "      <td>0.512709</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.846676</td>\n",
       "      <td>0.636665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oof_lvl1_lgb_bagging</th>\n",
       "      <td>0.822752</td>\n",
       "      <td>0.846676</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.811066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oof_lvl1_lgb_bagging_RescuerID_mean</th>\n",
       "      <td>0.683655</td>\n",
       "      <td>0.636665</td>\n",
       "      <td>0.811066</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     oof_lvl1_lgb_non_img                 ...                   oof_lvl1_lgb_bagging_RescuerID_mean\n",
       "oof_lvl1_lgb_non_img                             1.000000                 ...                                              0.683655\n",
       "oof_lvl1_lgb_img_text                            0.512709                 ...                                              0.636665\n",
       "oof_lvl1_lgb_bagging                             0.822752                 ...                                              0.811066\n",
       "oof_lvl1_lgb_bagging_RescuerID_mean              0.683655                 ...                                              1.000000\n",
       "\n",
       "[4 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1080s] [2019-03-26 17:52:52,537]:    Level 2...\n",
      "Fitting pred_lvl2_lgb...\n",
      "[1081s] [2019-03-26 17:52:53,839]: Target match cutoffs: [1.509428095457252, 2.0060230342659584, 2.4750904374885123, 2.8980750995650455, 4]\n",
      "Fitting pred_lvl2_catboost...\n",
      "[1091s] [2019-03-26 17:53:03,777]: Target match cutoffs: [1.5294005746627546, 2.007432051687735, 2.4737538306524107, 2.891172536157699, 4]\n",
      "Fitting pred_lvl2_ridge...\n",
      "[1092s] [2019-03-26 17:53:04,765]: Target match cutoffs: [1.5344504868446596, 2.0741802950525012, 2.4700767458170594, 2.8467574474812722, 4]\n",
      "[1093s] [2019-03-26 17:53:05,187]: Target match cutoffs: [1.5369537284897308, 1.9769057793877423, 2.482930830803186, 2.9114597384870073, 4]\n",
      "[1093s] [2019-03-26 17:53:05,570]: Target match cutoffs: [1.549457948378886, 1.9980909366437154, 2.488234174523754, 2.9196041606222325, 4]\n",
      "[1094s] [2019-03-26 17:53:05,938]: Target match cutoffs: [1.5465929609203384, 2.0702646060423437, 2.487521406804147, 2.8646577083424614, 4]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pred_lvl2_lgb</th>\n",
       "      <th>pred_lvl2_catboost</th>\n",
       "      <th>pred_lvl2_ridge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>pred_lvl2_lgb</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.928902</td>\n",
       "      <td>0.893521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pred_lvl2_catboost</th>\n",
       "      <td>0.928902</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.899538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pred_lvl2_ridge</th>\n",
       "      <td>0.893521</td>\n",
       "      <td>0.899538</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    pred_lvl2_lgb       ...         pred_lvl2_ridge\n",
       "pred_lvl2_lgb            1.000000       ...                0.893521\n",
       "pred_lvl2_catboost       0.928902       ...                0.899538\n",
       "pred_lvl2_ridge          0.893521       ...                1.000000\n",
       "\n",
       "[3 rows x 3 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pred_lvl2_lgb</th>\n",
       "      <th>pred_lvl2_catboost</th>\n",
       "      <th>pred_lvl2_ridge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>pred_lvl2_lgb</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.925930</td>\n",
       "      <td>0.901058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pred_lvl2_catboost</th>\n",
       "      <td>0.925930</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.903801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pred_lvl2_ridge</th>\n",
       "      <td>0.901058</td>\n",
       "      <td>0.903801</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    pred_lvl2_lgb       ...         pred_lvl2_ridge\n",
       "pred_lvl2_lgb            1.000000       ...                0.901058\n",
       "pred_lvl2_catboost       0.925930       ...                0.903801\n",
       "pred_lvl2_ridge          0.901058       ...                1.000000\n",
       "\n",
       "[3 rows x 3 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1094s] [2019-03-26 17:53:06,112]:    Level 3...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([2, 4, 3, ..., 1, 1, 4], dtype=int32)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA64AAAEICAYAAABf3KmoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X+8pXVd7/3XW34o4Q9QPDuYocAjnUJJxbmRDufu3kniiClWZijpYNh0n/CkNZ0au88dpll2jmhBHmsSAg1FIo1JMCJ05/EkKCiBgN5MOMpMA6T80C2GjX7uP9Z3ZM1m75k1sy/WvtbM6/l47Me+1vf6Xt/rs77r2uu7Puv6XtdOVSFJkiRJUl89aqkDkCRJkiRpR0xcJUmSJEm9ZuIqSZIkSeo1E1dJkiRJUq+ZuEqSJEmSes3EVZIkSZLUayau0g4kOSJJJdm3Pf5IklW70c73JZlNsk/3UT5sXzNJXtOWT0vytx22fXOS6bb8xiR/3mHbv5nk3V21J0na8yTZp42n37fUsexIkt9JckFbfkqS2Q7bfneS32zLP55kY4dtTye5uav2pC6ZuGriJdmY5JttILsryQVJHvtI7KuqXlBVF44Y048PbfflqnpsVX37kYhrIVV1UVWdtLN6rc9+Z4T2nlZVM4uNqw2Mm+a0/btV9ZrFti1J6o82Nm/7+c7QeD2b5LRdba+qvt3G0y8/EvHOJ8lrkszs7vZVdXtV7fRzyaj7qarXVNXv7m48Q/vbt305f8RQ2zNV9bTFti09Ekxctad4URsUjgVWAP9tboUMeMzvhm1nnCVJ2hUtyXxsG6O/TBuv289Fc+s73uzYOGZuSX3lh3jtUapqM/AR4Onw3Wmzb0nyv4EHgKckeUKS85JsSbK5TefZp9XfJ8nbknwlye3AC4fbH56G2x7/QpJbk3w9yS1Jjk3yXuD7gL9u3yj/+jxTjg9Lsj7JPUk2JPmFoTbfmOSSJO9p7d6cZMVCzznJ85J8Psn9Sf4IyNC605N8oi0nyTuS3J3ka0luSvL0JKuB04Bfb/H+dau/MclvJLkR+Eb7Zna7M8nAY5J8oMX5mSTPGNp3JXnq0OMLWl8f2F6jw4a+dT9s7tTjJC9uz/2+1u8/NLRuY5JfS3Jje94fSPKYhfpIktRPbVz4QJL3J/k68HNJfiTJNe39f0uSc5Ls1+pvd5YwyZ+39R9pY9Enkxy5wL6+J8n7kny1tf2pJIe0dQcl+bO2v01J3pTkUUmOAf4I+D/bePWVBdp+SpL/1WK4EnjS0LqnJqmhx2e0cezrSW5PcupC+2nP751J/ibJN9r6P0/yxjn7/632vL6Y5NSh8k8kOX3o8fBZ3Y+33ze3ff505kw9TvK0JH/f+uumJC8cWjdy30tdMHHVHiXJ4cDJwGeHil8JrAYeB3wJuADYCjwVeBZwErAtGf0F4Cda+QrgpTvY188AbwReBTweeDHw1ap6Jdt/q/zf59n8YmATcFjbx+8mee7Q+he3OgcB6xkMZvPFcAjwQQZnmA8B/gk4YYGQTwJ+FPgB4AnAy1q864CLgP/e4n3R0DYvZ5C8H1RVW+dp8xTgL4AnAu8D/mrbh4uFVNU3gBcA/zz0rfs/z3lePwC8H3g98GTgCgZfBOw/VO1lwErgSOCHgdN3tF9JUm/9JIMx5AnABxiM0a9jMK6dwOC9/hd3sP0rgP+XwVj0ZeDNC9R7NfA9wHIGieUvAf/a1r0X+Cbw74FnMxj7Xl1VNwGvBf5XG68OWaDtDwDXtJh/j8Fnj4dJ8njg7cDzqupx7fnduJP9vAL4bQafYz45T7PL27rDgDOA84e/ON6BH22/n9b2+ZdzYt0f+DBwOYOx+FeAD8xpe9S+lxbNxFV7ir9Kch/wCeDvgeFrPy6oqptb4vVEBont66vqG1V1N/AOYNu3ky8D/qCq7qiqexgMPgt5DYNk79M1sKGqvrSzQFtyfQLwG1X1r1V1A/BuBgnwNp+oqivaNbHvBZ4xT1O053JzVV1aVf8G/AFw5wJ1/43BwPaDQKrq1qraspNwz2l98c0F1l8/tO+3A48Bjt9Jm6P4WeDyqrqqtf024ADgP86J7Z/b6/TXwDM72K8kafw+UVV/XVXfqapvtnH12qraWlW3A+uA/2sH219aVde18eIiFh4P/o1BYvnUdq3sdVU1m2QZ8OPAr1TVA1V1F4Px9NQF2tlOkqcwGKfPqqoH270grtjBJgU8PcljqmpLVd2yk118qKo+2frnwXnWf2do3x8F/gb4mVFi34kTgP2B/1FV/1ZVf8dgxtRwv4za99KimbhqT/GSqjqoqr6/qn5pTqJ1x9Dy9wP7AVvatJf7gD8B/l1bf9ic+jtKRA9ncIZzVx0G3FNVX5+zn2VDj4eTzwcYTMmd77qf7eKtqmL7+Bla91EGZ27fCdydZF375ndH5m1rvvVV9R0eOou8WIcx1Pet7TvYcR89IjfkkiQ94rYba5L8YJLLk9yZ5GvAmxgknAsZdTy4APg74JIMLhV6axtbvx94NHDX0GeDdwJTI8Z/GIMZTA8Mlc37+aGqvsZgNtOZwJ1JPtxmGe3Izsbi+fbd1Vj85fbZYrhtx2ItCRNX7Q2G33DvAB4EDmmJ7kFV9fihO+htYZCQbrOj2+3fwWBK0c72Odc/A09M8rg5+9m8g20Wsl28ScL28W8fVNU5VfVs4GgGU4b/607i3dHzYM6+H8VgutK2ab8PMJiStc337kK7/8zgg8S2trc9r93pI0lSv80dE/4E+ByDM6OPB36Lofs37PZOqr5VVW+sqh8C/hODKcqnMRjPHwCeOOezwQ8vEN9cW4AnJTlgqGzBzw9V9ZGq+nHgUGADg+e7o/3sbP/z7XvbWPwNFjcWH97G4OG2HYu1JExctVdpU2P/Fjg7yePbjRf+fZJtU5AuAX45yfIkBwNrd9Dcu4FfS/LsDDw1ybZk6y7gKQvEcAfwD8DvJXlMkh9mcE3K7vxP1MuBpyX5qfat8S+z/aD0XUn+jyTPadegfoPBdT3f2Vm8O/HsoX2/nsGXAte0dTcAr8jghlcr2X6a110MBtonLNDuJcALk5zY4l3T2v6H3YhRkjRZHgfcz+DGgD/Ejq9vHVmS52ZwU8JHAV9jMHX4O21c/nvgbUOfDZ6aZNs1oHcByxe6h0NV/RNwI/DGJPu37V44X90khyZ5UZLvAb7FYDweHosX3M8OPGpo39MM7iNxaVt3A/DTSQ5oZ3Z/fijubwNfZeHx/x8YXG+8Jsl+7V4cJzO4nlcaOxNX7Y1exeCajVuAexm8uR/a1v0pcCXwj8BnGNz4aF5V9RfAWxjcUOLrwF8xuIYWBtfG/rc25ejX5tn85cARDL7N/BCDa1P+blefSFV9hcF1LG9lMPgcBfzvBao/nsHzu5fBVJ+vAv+jrTsPOLrF+1e7EMJlDK5HvZfBjSh+ql3nAoMba7wIuI/BN9rfbbeqPs/g5ku3t31uN6Wpqr4A/BxwLvCV1s6LqupbuxCbJGkyrQFWMRhb/4TuEqXDGIzrXwNuZjBt+H1t3c8BB/LQZ4O/4KEvgq8CbmMwlXih+0icyuCa0HuA/4fB/Snmsw+D2U5bGIzD/5HBtOFR9zOfTQwS4C3AhcBrquq2tu5tDM6s3g2cz8O/JD8LeF8bi39qeEW7nvZFDG7E+BXgHOAVQ21LY5Xtp61LkiRJktQvnnGVJEmSJPWaiaskSZIkqddMXCVJkiRJvWbiKkmSJEnqtX2XOoAdOeSQQ+qII47opK1vfOMbHHjggZ20NU7GPV7GPV7GPX6TGntXcV9//fVfqaondxDSXsux2bjHzbjHy7jHb1JjH/fY3OvE9YgjjuC6667rpK2ZmRmmp6c7aWucjHu8jHu8jHv8JjX2ruJO8qXFR7N3c2w27nEz7vEy7vGb1NjHPTY7VViSJEmS1GsmrpIkSZKkXjNxlSRJkiT1momrJEmSJKnXTFwlSZIkSb1m4ipJkiRJ6jUTV0mSJElSr5m4SpIkSZJ6zcRVkiRJktRr+y51AJKkfjli7eVLHcLDXLDywKUOQZKkJePY7BlXSZIkSVLPmbhKkiRJknrNxFWSJEmS1Gte4yrtAW7afD+n9+zah41vfeFShyBJkqQ9hGdcJUmSJEm9ZuIqSdKESrJPks8m+XB7fGSSa5NsSPKBJPu38ke3xxva+iOG2nhDK/9CkucvzTORJGnHTFwlSZpcrwNuHXr8+8A7quqpwL3AGa38DODeVv6OVo8kRwOnAk8DVgL/M8k+Y4pdkqSReY2rJEkTKMly4IXAW4BfTRLgucArWpULgTcC7wJOacsAlwJ/1OqfAlxcVQ8CX0yyATgO+OSYnoa0x/L+E1K3TFwlSZpMfwD8OvC49vhJwH1VtbU93gQsa8vLgDsAqmprkvtb/WXANUNtDm/zXUlWA6sBpqammJmZ6eQJzM7OdtbWOBn3eE1q3FMHwJpjtu684hiN0o+T2t+TGjeMFnvfjiUYf5+buEqSNGGS/ARwd1Vdn2T6kd5fVa0D1gGsWLGipqe72eXMzAxdtTVOxj1ekxr3uRddxtk39euj9sbTpndaZ1L7e1LjhtFi79vZe4ALVh441j7v11+TJEkaxQnAi5OcDDwGeDzwh8BBSfZtZ12XA5tb/c3A4cCmJPsCTwC+OlS+zfA2kiT1hjdnkiRpwlTVG6pqeVUdweDmSh+tqtOAjwEvbdVWAZe15fXtMW39R6uqWvmp7a7DRwJHAZ8a09OQJGlknnGVJGnP8RvAxUl+B/gscF4rPw94b7v50j0Mkl2q6uYklwC3AFuBM6vq2+MPW5KkHTNxlSRpglXVDDDTlm9ncFfguXX+FfiZBbZ/C4M7E0uS1FtOFZYkSZIk9dpOE9ck5ye5O8nnhsqemOSqJLe13we38iQ5J8mGJDcmOXZom1Wt/m1JVs23L0mSJEmS5hrljOsFwMo5ZWuBq6vqKODq9hjgBQxu7HAUg//39i4YJLrAWcBzGExhOmtbsitJkiRJ0o7sNHGtqo8zuJHDsFOAC9vyhcBLhsrfUwPXMLgt/6HA84GrquqeqroXuIqHJ8OSJEmSJD3M7t6caaqqtrTlO4GptrwMuGOo3qZWtlD5wyRZzeBsLVNTU8zMzOxmiNubnZ3trK1xMu7xmtS4pw6ANcdsXeowtjNKP05qf09q3DBa7H07lmCy+1ySJC3eou8qXFWVpLoIprW3DlgHsGLFipqenu6k3ZmZGbpqa5yMe7wmNe5zL7qMs2/q103CN542vdM6k9rfkxo3jBb76WsvH08wu+CClQdObJ9LkqTF2927Ct/VpgDTft/dyjcDhw/VW97KFiqXJEmSJGmHdjdxXQ9suzPwKuCyofJXtbsLHw/c36YUXwmclOTgdlOmk1qZJEmSJEk7tNO5hUneD0wDhyTZxODuwG8FLklyBvAl4GWt+hXAycAG4AHg1QBVdU+SNwOfbvXeVFVzb/gkSZIkSdLD7DRxraqXL7DqxHnqFnDmAu2cD5y/S9FJkiRJkvZ6uztVWJIkSZKksTBxlSRJkiT1momrJEmSJKnXTFwlSZIkSb1m4ipJkiRJ6jUTV0mSJElSr5m4SpI0YZI8JsmnkvxjkpuT/HYrvyDJF5Pc0H6e2cqT5JwkG5LcmOTYobZWJbmt/axaquckSdKO7PT/uEqSpN55EHhuVc0m2Q/4RJKPtHX/taounVP/BcBR7ec5wLuA5yR5InAWsAIo4Pok66vq3rE8C0mSRuQZV0mSJkwNzLaH+7Wf2sEmpwDvadtdAxyU5FDg+cBVVXVPS1avAlY+krFLkrQ7POMqSdIESrIPcD3wVOCdVXVtkv8MvCXJbwFXA2ur6kFgGXDH0OabWtlC5XP3tRpYDTA1NcXMzEwnz2F2draztsbJuMdrUuOeOgDWHLN1qcPYzij9OKn9Palxw2ix9+1YgvH3uYmrJEkTqKq+DTwzyUHAh5I8HXgDcCewP7AO+A3gTR3sa11rjxUrVtT09PRimwQGH6K7amucjHu8JjXucy+6jLNv6tdH7Y2nTe+0zqT296TGDaPFfvray8cTzC64YOWBY+1zpwpLkjTBquo+4GPAyqra0qYDPwj8GXBcq7YZOHxos+WtbKFySZJ6xcRVkqQJk+TJ7UwrSQ4Angd8vl23SpIALwE+1zZZD7yq3V34eOD+qtoCXAmclOTgJAcDJ7UySZJ6pV/zFyRJ0igOBS5s17k+Crikqj6c5KNJngwEuAH4v1v9K4CTgQ3AA8CrAarqniRvBj7d6r2pqu4Z4/OQJGkkJq6SJE2YqroReNY85c9doH4BZy6w7nzg/E4DlCSpY04VliRJkiT1momrJEmSJKnXTFwlSZIkSb1m4ipJkiRJ6jUTV0mSJElSr5m4SpIkSZJ6zX+HI0mSJGlJHLH28k7aWXPMVk7vqK2Nb31hJ+2oW55xlSRJkiT1mmdcJe01/FZXkiRpMnnGVZIkSZLUayaukiRJkqRec6qwttO3qZROo5QkSZLkGVdJkiRJUq8tKnFN8itJbk7yuSTvT/KYJEcmuTbJhiQfSLJ/q/vo9nhDW39EF09AkiRJkrRn2+2pwkmWAb8MHF1V30xyCXAqcDLwjqq6OMkfA2cA72q/762qpyY5Ffh94GcX/QwkSZL2YF7GI0mLnyq8L3BAkn2B7wG2AM8FLm3rLwRe0pZPaY9p609MkkXuX5IkSZK0h9vtM65VtTnJ24AvA98E/ha4Hrivqra2apuAZW15GXBH23ZrkvuBJwFfGW43yWpgNcDU1BQzMzO7G+J2ZmdnO2trnMYd95pjtu680gimDuimrXG/ZpN6nHTV310apR/39uMb+nmM9+1Ygsn923ykJHkM8HHg0QzG8kur6qwkRwIXMxhfrwdeWVXfSvJo4D3As4GvAj9bVRtbW29gMCvq28AvV9WV434+kiTtzGKmCh/M4CzqkcB9wF8AKxcbUFWtA9YBrFixoqanpxfbJDD4cNhVW+M07ri7mEIEgw++Z9+0+JtWbzxtevHB7IJJPU7OveiyTvq7S6O8dnv78Q39PMa76qcuXbDywIn823wEPQg8t6pmk+wHfCLJR4BfZRcu10lyNIPLfJ4GHAb8XZIfqKpvL8WTkiRpIYuZKvzjwBer6l+q6t+ADwInAAe1qcMAy4HNbXkzcDhAW/8EBt/6SpKkXVADs+3hfu2n2PXLdU4BLq6qB6vqi8AG4LgxPAVJknbJYk4ZfBk4Psn3MJgqfCJwHfAx4KUMpiqtAi5r9de3x59s6z9aVbWI/UuStNdKsg+D6cBPBd4J/BO7frnOMuCaoWaHtxnel5fxDNnbL3Po4yUOfeRlPKPp2/EN/TzG+3YswfiPlcVc43ptkkuBzwBbgc8ymOJ7OXBxkt9pZee1Tc4D3ptkA3APg6lJkiRpN7TpvM9MchDwIeAHH8F9eRnPkL39Moc+XuLQR17GM5q+Hd/Qz2Pcy3gWd8aVqjoLOGtO8e3MM82oqv4V+JnF7E+SJG2vqu5L8jHgR2iX67SzrvNdrrNpzuU6372MpxneRpKk3ljsv8ORJEljluTJ7UwrSQ4AngfcykOX68D8l+vA9pfrrAdOTfLodkfio4BPjedZSJI0un7NX5AkSaM4FLiwXef6KOCSqvpwklvYhct1qurmJJcAtzC47OdM7ygsSeojE1dJkiZMVd0IPGue8l2+XKeq3gK8pesYJUnqklOFJUmSJEm9ZuIqSZIkSeo1E1dJkiRJUq+ZuEqSJEmSes3EVZIkSZLUayaukiRJkqReM3GVJEmSJPWaiaskSZIkqddMXCVJkiRJvWbiKkmSJEnqNRNXSZIkSVKvmbhKkiRJknrNxFWSJEmS1GsmrpIkSZKkXjNxlSRJkiT1momrJEkTJsnhST6W5JYkNyd5XSt/Y5LNSW5oPycPbfOGJBuSfCHJ84fKV7ayDUnWLsXzkSRpZ/Zd6gAkSdIu2wqsqarPJHkccH2Sq9q6d1TV24YrJzkaOBV4GnAY8HdJfqCtfifwPGAT8Okk66vqlrE8C0mSRmTiKknShKmqLcCWtvz1JLcCy3awySnAxVX1IPDFJBuA49q6DVV1O0CSi1tdE1dJUq+YuEqSNMGSHAE8C7gWOAF4bZJXAdcxOCt7L4Ok9pqhzTbxUKJ7x5zy58yzj9XAaoCpqSlmZmY6iX12draztsZp3HGvOWZrJ+1MHdBNW+N+zSb1OOmqv7s0Sj/u7cc39PMY79uxBOM/VkxcJUmaUEkeC/wl8Pqq+lqSdwFvBqr9Phv4+cXup6rWAesAVqxYUdPT04ttEhh8OOyqrXEad9ynr728k3bWHLOVs29a/Ee/jadNLz6YXTCpx8m5F13WSX93aZTXbm8/vqGfx3hX/dSlC1YeONZjpV9/TZIkaSRJ9mOQtF5UVR8EqKq7htb/KfDh9nAzcPjQ5stbGTsolySpN7yrsCRJEyZJgPOAW6vq7UPlhw5V+0ngc215PXBqkkcnORI4CvgU8GngqCRHJtmfwQ2c1o/jOUiStCs84ypJ0uQ5AXglcFOSG1rZbwIvT/JMBlOFNwK/CFBVNye5hMFNl7YCZ1bVtwGSvBa4EtgHOL+qbh7nE5EkaRQmrpIkTZiq+gSQeVZdsYNt3gK8ZZ7yK3a0nSRJfeBUYUmSJElSr5m4SpIkSZJ6bVGJa5KDklya5PNJbk3yI0memOSqJLe13we3uklyTpINSW5Mcmw3T0GSJEmStCdb7BnXPwT+pqp+EHgGcCuwFri6qo4Crm6PAV7A4C6GRzH4J+bvWuS+JUmSJEl7gd1OXJM8AfhRBrfjp6q+VVX3AacAF7ZqFwIvacunAO+pgWuAg+bctl+SJEmSpIdZzF2FjwT+BfizJM8ArgdeB0xV1ZZW505gqi0vA+4Y2n5TK9syVEaS1QzOyDI1NcXMzMwiQnzI7OxsZ22N07jjXnPM1k7amTqgm7bG/ZpN6nHSVX93aZR+3NuPb+jnMd63Ywkm929TkiR1YzGJ677AscB/qaprk/whD00LBqCqKkntSqNVtQ5YB7BixYqanp5eRIgPmZmZoau2xmnccZ++9vJO2llzzFbOvmnx/21p42nTiw9mF0zqcXLuRZd10t9dGuW129uPb+jnMd5VP3XpgpUHTuTfpiRJ6sZirnHdBGyqqmvb40sZJLJ3bZsC3H7f3dZvBg4f2n55K5MkSZIkaUG7nbhW1Z3AHUn+Qys6EbgFWA+samWrgMva8nrgVe3uwscD9w9NKZYkSZIkaV6Lnev2X4CLkuwP3A68mkEyfEmSM4AvAS9rda8ATgY2AA+0upIkSZIk7dCiEtequgFYMc+qE+epW8CZi9mfJEmSJGnvs9j/4ypJkiRJ0iPKxFWSJEmS1GsmrpIkSZKkXjNxlSRJkiT1momrJEmSJKnXTFwlSZowSQ5P8rEktyS5OcnrWvkTk1yV5Lb2++BWniTnJNmQ5MYkxw61tarVvy3JqoX2KUnSUjJxlSRp8mwF1lTV0cDxwJlJjgbWAldX1VHA1e0xwAuAo9rPauBdMEh0gbOA5wDHAWdtS3YlSeoTE1dJkiZMVW2pqs+05a8DtwLLgFOAC1u1C4GXtOVTgPfUwDXAQUkOBZ4PXFVV91TVvcBVwMoxPhVJkkay71IHIEmSdl+SI4BnAdcCU1W1pa26E5hqy8uAO4Y229TKFiqfu4/VDM7UMjU1xczMTCexz87OdtbWOI077jXHbO2knakDumlr3K/ZpB4nXfV3l0bpx739+IZ+HuN9O5Zg/MeKiaskSRMqyWOBvwReX1VfS/LddVVVSaqL/VTVOmAdwIoVK2p6erqLZpmZmaGrtsZp3HGfvvbyTtpZc8xWzr5p8R/9Np42vfhgdsGkHifnXnRZJ/3dpVFeu739+IZ+HuNd9VOXLlh54FiPFacKS5I0gZLsxyBpvaiqPtiK72pTgGm/727lm4HDhzZf3soWKpckqVdMXCVJmjAZnFo9D7i1qt4+tGo9sO3OwKuAy4bKX9XuLnw8cH+bUnwlcFKSg9tNmU5qZZIk9Uq/5i9IkqRRnAC8ErgpyQ2t7DeBtwKXJDkD+BLwsrbuCuBkYAPwAPBqgKq6J8mbgU+3em+qqnvG8xQkSRqdiaskSROmqj4BZIHVJ85Tv4AzF2jrfOD87qKTJKl7ThWWJEmSJPWaiaskSZIkqddMXCVJkiRJvWbiKkmSJEnqNRNXSZIkSVKvmbhKkiRJknrNxFWSJEmS1GsmrpIkSZKkXjNxlSRJkiT1momrJEmSJKnXTFwlSZIkSb1m4ipJkiRJ6jUTV0mSJElSr5m4SpIkSZJ6bdGJa5J9knw2yYfb4yOTXJtkQ5IPJNm/lT+6Pd7Q1h+x2H1LkiRJkvZ8XZxxfR1w69Dj3wfeUVVPBe4FzmjlZwD3tvJ3tHqSJEmSJO3QohLXJMuBFwLvbo8DPBe4tFW5EHhJWz6lPaatP7HVlyRJuyDJ+UnuTvK5obI3Jtmc5Ib2c/LQuje0GU9fSPL8ofKVrWxDkrXjfh6SJI1qsWdc/wD4deA77fGTgPuqamt7vAlY1paXAXcAtPX3t/qSJGnXXACsnKf8HVX1zPZzBUCSo4FTgae1bf5nu8xnH+CdwAuAo4GXt7qSJPXOvru7YZKfAO6uquuTTHcVUJLVwGqAqakpZmZmOml3dna2s7bGadxxrzlm684rjWDqgG7aGvdrNqnHSVf93aVR+nFvP76hn8d4344lmNy/zUdKVX18F+4VcQpwcVU9CHwxyQbguLZuQ1XdDpDk4lb3lo7DlSRp0VJVu7dh8nvAK4GtwGOAxwMfAp4PfG9VbU3yI8Abq+r5Sa5sy59Msi9wJ/Dk2kEAK1asqOuuu2634ptrZmaG6enpTtoap3HHfcTayztpZ80xWzn7pt3+XuS7Nr71hR1EM7pJPU7OveiyTvq7S6O8dnv78Q39PMa76qcuXbDywE6OlSTXV9WKxUe09Fri+uGqenp7/EbgdOBrwHXAmqq6N8kfAddU1Z+3eucBH2nNrKyq17TyVwLPqarXzrOv4S+Vn33xxRd38hxmZ2d57GMf20lb4zTuuG/afH8n7UwdAHd9c/HtHLPsCYtvZBdM6nFy9z33d9LfXRrEkyqOAAAOvElEQVTltdvbj2/o5zHeVT916cgn7NPJsfJjP/ZjI43Nu/3Jq6reALwBoJ1x/bWqOi3JXwAvBS4GVgGXtU3Wt8efbOs/uqOkVZIk7ZJ3AW8Gqv0+G/j5LhquqnXAOhh8qdzVF06T+mXhuOM+vWdfum08bXrxweyCST1Oevml8giv3d5+fEM/j/Gu+qlLXX2pPKpH4v+4/gbwq20q0pOA81r5ecCTWvmvAt4EQpKkjlTVXVX17ar6DvCnPDQdeDNw+FDV5a1soXJJknqnk68lqmoGmGnLt/PQYDlc51+Bn+lif5IkaXtJDq2qLe3hTwLb7ji8HnhfkrcDhwFHAZ8CAhyV5EgGCeupwCvGG7UkSaPp1/wFSZK0U0neD0wDhyTZBJwFTCd5JoOpwhuBXwSoqpuTXMLgpktbgTOr6tutndcCVwL7AOdX1c1jfiqSJI3ExFWSpAlTVS+fp/i8ecq21X8L8JZ5yq8ArugwNEmSHhGPxDWukiRJkiR1xsRVkiRJktRrJq6SJEmSpF4zcZUkSZIk9ZqJqyRJkiSp10xcJUmSJEm9ZuIqSZIkSeo1E1dJkiRJUq+ZuEqSJEmSes3EVZIkSZLUayaukiRJkqReM3GVJEmSJPWaiaskSZIkqddMXCVJkiRJvWbiKkmSJEnqNRNXSZImTJLzk9yd5HNDZU9MclWS29rvg1t5kpyTZEOSG5McO7TNqlb/tiSrluK5SJI0ChNXSZImzwXAyjlla4Grq+oo4Or2GOAFwFHtZzXwLhgkusBZwHOA44CztiW7kiT1jYmrJEkTpqo+Dtwzp/gU4MK2fCHwkqHy99TANcBBSQ4Fng9cVVX3VNW9wFU8PBmWJKkX9l3qACRJUiemqmpLW74TmGrLy4A7huptamULlT9MktUMztYyNTXFzMxMJwHPzs521tY4jTvuNcds7aSdqQO6aWvcr9mkHidd9XeXRunHvf34hn4e4307lmD8x4qJqyRJe5iqqiTVYXvrgHUAK1asqOnp6U7anZmZoau2xmnccZ++9vJO2llzzFbOvmnxH/02nja9+GB2waQeJ+dedFkn/d2lUV67vf34hn4e4131U5cuWHngWI8VpwpLkrRnuKtNAab9vruVbwYOH6q3vJUtVC5JUu+YuEqStGdYD2y7M/Aq4LKh8le1uwsfD9zfphRfCZyU5OB2U6aTWpkkSb3Tr/kLkiRpp5K8H5gGDkmyicHdgd8KXJLkDOBLwMta9SuAk4ENwAPAqwGq6p4kbwY+3eq9qarm3vBJkqReMHGVJGnCVNXLF1h14jx1CzhzgXbOB87vMDRJkh4RThWWJEmSJPWaiaskSZIkqddMXCVJkiRJvbbbiWuSw5N8LMktSW5O8rpW/sQkVyW5rf0+uJUnyTlJNiS5McmxXT0JSZIkSdKeazFnXLcCa6rqaOB44MwkRwNrgaur6ijg6vYY4AXAUe1nNfCuRexbkiRJkrSX2O3Etaq2VNVn2vLXgVuBZcApwIWt2oXAS9ryKcB7auAa4KBt/yhdkiRJkqSFdPLvcJIcATwLuBaYav/YHOBOYKotLwPuGNpsUyvbMlRGktUMzsgyNTXFzMxMFyEyOzvbWVvjNO641xyztZN2pg7opq1xv2aTepx01d9dGqUf9/bjG/p5jPftWILJ/duUJEndWHTimuSxwF8Cr6+qryX57rqqqiS1K+1V1TpgHcCKFStqenp6sSECgw+HXbU1TuOO+/S1l3fSzppjtnL2TYv/XmTjadOLD2YXTOpxcu5Fl3XS310a5bXb249v6Ocx3lU/demClQdO5N+mJEnqxqLuKpxkPwZJ60VV9cFWfNe2KcDt992tfDNw+NDmy1uZJEmSJEkLWsxdhQOcB9xaVW8fWrUeWNWWVwGXDZW/qt1d+Hjg/qEpxZIkSZIkzWsxc91OAF4J3JTkhlb2m8BbgUuSnAF8CXhZW3cFcDKwAXgAePUi9i1JkiRJ2kvsduJaVZ8AssDqE+epX8CZu7s/SZIkSdLeaVHXuEqSJEmS9EgzcZUkSZIk9ZqJqyRJkiSp10xcJUmSJEm9ZuIqSdIeJMnGJDcluSHJda3siUmuSnJb+31wK0+Sc5JsSHJjkmOXNnpJkuZn4ipJ0p7nx6rqmVW1oj1eC1xdVUcBV7fHAC8Ajmo/q4F3jT1SSZJGYOIqSdKe7xTgwrZ8IfCSofL31MA1wEFJDl2KACVJ2pHd/j+ukiSplwr42yQF/ElVrQOmqmpLW38nMNWWlwF3DG27qZVtGSojyWoGZ2SZmppiZmamk0BnZ2c7a2ucxh33mmO2dtLO1AHdtDXu12xSj5Ou+rtLo/Tj3n58Qz+P8b4dSzD+Y8XEVZKkPct/qqrNSf4dcFWSzw+vrKpqSe3IWvK7DmDFihU1PT3dSaAzMzN01dY4jTvu09de3kk7a47Zytk3Lf6j38bTphcfzC6Y1OPk3Isu66S/uzTKa7e3H9/Qz2O8q37q0gUrDxzrseJUYUmS9iBVtbn9vhv4EHAccNe2KcDt992t+mbg8KHNl7cySZJ6xcRVkqQ9RJIDkzxu2zJwEvA5YD2wqlVbBVzWltcDr2p3Fz4euH9oSrEkSb3Rr/kLkiRpMaaADyWBwRj/vqr6mySfBi5JcgbwJeBlrf4VwMnABuAB4NXjD1mSpJ0zcZUkaQ9RVbcDz5in/KvAifOUF3DmGEKb102b7+/ddVsb3/rCpQ5BkjQPpwpLkiRJknptrznj6re6kiRJkjSZPOMqSZIkSeo1E1dJkiRJUq+ZuEqSJEmSes3EVZIkSZLUayaukiRJkqReM3GVJEmSJPWaiaskSZIkqddMXCVJkiRJvWbiKkmSJEnqNRNXSZIkSVKvmbhKkiRJknrNxFWSJEmS1GsmrpIkSZKkXjNxlSRJkiT12tgT1yQrk3whyYYka8e9f0mStD3HZklS3401cU2yD/BO4AXA0cDLkxw9zhgkSdJDHJslSZNg3GdcjwM2VNXtVfUt4GLglDHHIEmSHuLYLEnqvVTV+HaWvBRYWVWvaY9fCTynql47VGc1sLo9/A/AFzra/SHAVzpqa5yMe7yMe7yMe/wmNfau4v7+qnpyB+3sMRybd4txj5dxj5dxj9+kxj7WsXnfDnbUqapaB6zrut0k11XViq7bfaQZ93gZ93gZ9/hNauyTGveewrF5e8Y9XsY9XsY9fpMa+7jjHvdU4c3A4UOPl7cySZK0NBybJUm9N+7E9dPAUUmOTLI/cCqwfswxSJKkhzg2S5J6b6xThatqa5LXAlcC+wDnV9XNY9p951OcxsS4x8u4x8u4x29SY5/UuHvPsXm3GPd4Gfd4Gff4TWrsY417rDdnkiRJkiRpV417qrAkSZIkSbvExFWSJEmS1Gt7XOKaZGWSLyTZkGTtPOsfneQDbf21SY4Yf5QPN0Lcpyf5lyQ3tJ/XLEWcc2I6P8ndST63wPokOac9pxuTHDvuGOczQtzTSe4f6uvfGneM80lyeJKPJbklyc1JXjdPnd71+Yhx967PkzwmyaeS/GOL+7fnqdO795MR4+7d+8k2SfZJ8tkkH55nXe/6W6NxbB4fx+bxcmweL8fmpdGbsbmq9pgfBjeV+CfgKcD+wD8CR8+p80vAH7flU4EPTEjcpwN/tNSxzonpR4Fjgc8tsP5k4CNAgOOBa5c65hHjngY+vNRxzhPXocCxbflxwP83z3HSuz4fMe7e9Xnrw8e25f2Aa4Hj59Tp4/vJKHH37v1kKLZfBd433/HQx/72Z6TX1LF5vHE7No83bsfm8cbt2Lw08fdibN7TzrgeB2yoqtur6lvAxcApc+qcAlzYli8FTkySMcY4n1Hi7p2q+jhwzw6qnAK8pwauAQ5Kcuh4olvYCHH3UlVtqarPtOWvA7cCy+ZU612fjxh377Q+nG0P92s/c+9m17v3kxHj7qUky4EXAu9eoErv+lsjcWweI8fm8XJsHi/H5vHr09i8pyWuy4A7hh5v4uF/hN+tU1VbgfuBJ40luoWNEjfAT7cpJpcmOXye9X0z6vPqox9p0zk+kuRpSx3MXG0axrMYfGM3rNd9voO4oYd93qbG3ADcDVxVVQv2d4/eT0aJG/r5fvIHwK8D31lgfS/7Wzvl2NwvvR4ndqJ348Qwx+bxcGweu96MzXta4ron+2vgiKr6YeAqHvpmQ937DPD9VfUM4Fzgr5Y4nu0keSzwl8Drq+prSx3PqHYSdy/7vKq+XVXPBJYDxyV5+lLHNIoR4u7d+0mSnwDurqrrlzoWaRf07m9pD9bLcWIbx+bxcWwen76NzXta4roZGP52Ynkrm7dOkn2BJwBfHUt0C9tp3FX11ap6sD18N/DsMcW2GKO8Hr1TVV/bNp2jqq4A9ktyyBKHBUCS/RgMMBdV1QfnqdLLPt9Z3H3uc4Cqug/4GLByzqo+vp9810Jx9/T95ATgxUk2MpiS+dwkfz6nTq/7WwtybO6XXo4TO9PnccKxeWk4No9Fr8bmPS1x/TRwVJIjk+zP4ALh9XPqrAdWteWXAh+tqqWeY77TuOdcC/FiBtci9N164FUZOB64v6q2LHVQO5Pke7fNzU9yHIO/kyV/w2sxnQfcWlVvX6Ba7/p8lLj72OdJnpzkoLZ8APA84PNzqvXu/WSUuPv4flJVb6iq5VV1BIP3wI9W1c/Nqda7/tZIHJv7pXfjxCj6OE60WBybx8ixebz6Njbv+0g0ulSqamuS1wJXMrgb4PlVdXOSNwHXVdV6Bn+k702ygcFNAE5duogHRoz7l5O8GNjKIO7TlyzgJsn7Gdxx7pAkm4CzGFxsTlX9MXAFgzvpbQAeAF69NJFub4S4Xwr85yRbgW8Cpy71G15zAvBK4KZ2jQTAbwLfB73u81Hi7mOfHwpcmGQfBoP1JVX14b6/nzBa3L17P1nIBPS3dsKxebwcm8fOsXm8HJt7YKn6O0t//EmSJEmStLA9baqwJEmSJGkPY+IqSZIkSeo1E1dJkiRJUq+ZuEqSJEmSes3EVZIkSZLUayaukiRJkqReM3GVJEmSJPXa/w/KAn2/bmuJBQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def run_level0(X_train, y_train, X_test):\n",
    "    text_feats = PandasFeatureUnion([\n",
    "        ('Original', PandasTransform(lambda x: x)),\n",
    "        ('Raw description + TFIDF + SVD10', PandasPipeline([\n",
    "            ('select', SelectColumns(['Description'])),\n",
    "            ('preprocess', PandasTransform(lambda x: x.values.flatten().tolist())),\n",
    "            ('tfidf', TfidfVectorizer(min_df=2, max_features=None, \n",
    "                strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\\b\\w+\\b', \n",
    "                 ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)),\n",
    "            ('svd', TruncatedSVD(n_components=10, random_state=1337)),\n",
    "            ('df', PandasTransform(lambda x: \n",
    "                pd.DataFrame(x, columns=[f'lvl0_text_raw_desc_tfidf_svd10_{i}' for i in range(10)])))\n",
    "        ])),\n",
    "        ('Stemmed description + TFIDF + SVD5', PandasPipeline([\n",
    "            ('select', SelectColumns(['Description'])),\n",
    "            ('preprocess', PandasTransform(lambda x: x.fillna('').values.flatten().tolist())),\n",
    "            ('tfidf', StemmedTfidfVectorizer(SnowballStemmer('english', ignore_stopwords=False),\n",
    "                min_df=2, max_features=None, strip_accents='unicode', analyzer='word', \n",
    "                token_pattern=r'(?u)\\b\\w+\\b', ngram_range=(1, 3), use_idf=1, smooth_idf=1, \n",
    "                sublinear_tf=1)),\n",
    "            ('svd', TruncatedSVD(n_components=5, random_state=1337)),\n",
    "            ('df', PandasTransform(lambda x: \n",
    "                pd.DataFrame(x, columns=[f'lvl0_text_stem_desc_tfidf_svd5_{i}' for i in range(5)])))\n",
    "        ]))\n",
    "    ], n_jobs=1)\n",
    "    X_train = text_feats.fit_transform(X_train, y_train)\n",
    "    X_test = text_feats.transform(X_test)\n",
    "    \n",
    "    img_feats = PandasFeatureUnion([\n",
    "        ('Original', PandasTransform(lambda x: x)),\n",
    "        ('NasNet + PCA5', PandasPipeline([\n",
    "            ('select cols', SelectColumns(regex=r'(nasnetlarge_)')),\n",
    "            ('impute', SimpleImputer(fill_value=0.)),\n",
    "            ('var_thres', VarianceThreshold(0.)),\n",
    "            ('pca', PCA(5, random_state=234)),\n",
    "            ('df', PandasTransform(lambda x: \n",
    "                pd.DataFrame(x, columns=[f'lvl0_img_nasnet_pca5_{i}' for i in range(5)])))\n",
    "        ])),\n",
    "        ('NasNet + PCA400 + LR', OOFTransformer(make_pipeline(\n",
    "            SelectColumns(regex=r'(nasnetlarge_)'),\n",
    "            SimpleImputer(fill_value=0.),\n",
    "            VarianceThreshold(0.),\n",
    "            PCA(400, random_state=234),\n",
    "            LinearRegression(\n",
    "            )\n",
    "        ), name='lvl0_img_nasnet_pca400_lr')),\n",
    "        ('Xception + SVD150 + ET', OOFTransformer(make_pipeline(\n",
    "            SelectColumns(regex=r'(xception_)'),\n",
    "            SimpleImputer(fill_value=0.),\n",
    "            VarianceThreshold(0.),\n",
    "            TruncatedSVD(150, random_state=113120),\n",
    "            ExtraTreesRegressor(\n",
    "                n_estimators=200,\n",
    "                random_state=10233\n",
    "            )\n",
    "        ), name='lvl0_img_xception_svd150_pca400_lr')),\n",
    "    ], n_jobs=1)\n",
    "    X_train = img_feats.fit_transform(X_train, y_train)\n",
    "    X_test = img_feats.transform(X_test)\n",
    "    \n",
    "    if SUBMISSION or VERBOSE:\n",
    "        display(X_train.filter(like='oof_lvl0').corr())\n",
    "        display(X_test.filter(like='oof_lvl0').corr())\n",
    "    if SUBMISSION:\n",
    "        X_test.filter(like='oof_lvl0').to_csv('X_test_lvl0.csv', index=False)\n",
    "    return X_train,X_test\n",
    "\n",
    "def run_level1(X_train, y_train, X_test):\n",
    "    normal_feats = [\n",
    "        'Type', 'Age', 'Breed1', 'Breed2', 'Gender',\n",
    "        'Color1', 'Color2', 'Color3', 'MaturitySize', \n",
    "        'FurLength', 'Vaccinated', 'Dewormed', 'Sterilized',\n",
    "        'Health', 'Quantity', 'Fee', 'State', 'VideoAmt', 'PhotoAmt',\n",
    "        'Population2', '2016 GDP', 'adopted', 'MoreThanOneRescuerPet',\n",
    "    ]\n",
    "    lvl1 = PandasFeatureUnion([\n",
    "        ('Original', PandasTransform(lambda x: x)),\n",
    "        ('LGB + Non-image', OOFTransformer(make_pipeline(\n",
    "            SelectColumns(normal_feats),\n",
    "            LGBMRegressor(\n",
    "                n_estimators=100,\n",
    "                num_leaves=16,\n",
    "                learning_rate=0.1,\n",
    "                max_depth=5,\n",
    "                colsample_bytree=0.5,\n",
    "                min_split_gain=0.02,\n",
    "                min_child_samples=100,\n",
    "                min_child_weight=0.02,\n",
    "                reg_lambda=0.01,\n",
    "                random_state=SEED+374,\n",
    "                objective='rmse'\n",
    "                #verbose=1\n",
    "            )\n",
    "        ), name='lvl1_lgb_non_img')),\n",
    "        ('LGB + Image and text', OOFTransformer(make_pipeline(\n",
    "            SelectColumns(regex=r'(lvl0_text|lvl0_img|PhotoAmt|label_score|vertex_y)'),\n",
    "            LGBMRegressor(\n",
    "                n_estimators=100,\n",
    "                num_leaves=16,\n",
    "                learning_rate=0.1,\n",
    "                max_depth=5,\n",
    "                colsample_bytree=0.5,\n",
    "                min_split_gain=0.02,\n",
    "                min_child_samples=100,\n",
    "                min_child_weight=0.02,\n",
    "                reg_lambda=0.01,\n",
    "                random_state=SEED+274,\n",
    "                objective='rmse'\n",
    "                #verbose=1\n",
    "            )\n",
    "        ), name='lvl1_lgb_img_text')),\n",
    "        ('LGB + All features bagging', OOFTransformer(make_pipeline(\n",
    "            SelectColumns(regex=r'(lvl0|label_score|vertex_y|%s)' % '|'.join(normal_feats)),\n",
    "            BaggingRegressor(\n",
    "                LGBMRegressor(\n",
    "                    n_estimators=100,\n",
    "                    num_leaves=16,\n",
    "                    learning_rate=0.1,\n",
    "                    max_depth=5,\n",
    "                    colsample_bytree=0.5,\n",
    "                    min_split_gain=0.02,\n",
    "                    min_child_samples=100,\n",
    "                    min_child_weight=0.02,\n",
    "                    reg_lambda=0.01,\n",
    "                    random_state=SEED+743,\n",
    "                    objective='rmse'\n",
    "                    #verbose=1\n",
    "                ),\n",
    "                n_estimators=20,\n",
    "                max_samples=0.8,\n",
    "                max_features=0.4,\n",
    "                bootstrap=True,\n",
    "                random_state=9843\n",
    "            )\n",
    "        ), name='lvl1_lgb_bagging'))\n",
    "    ], n_jobs=1)\n",
    "    X_train = lvl1.fit_transform(X_train, y_train)\n",
    "    X_test = lvl1.transform(X_test)\n",
    "    \n",
    "    lvl1_rescuer_oof_mean = PandasFeatureUnion([\n",
    "        ('original', PandasTransform(lambda x: x)),\n",
    "        ('rescuermean', PandasTransform(lambda x: \n",
    "            grouped_feature(x, ['RescuerID'], ['oof_lvl1_lgb_bagging'], ['mean'])))\n",
    "    ])\n",
    "    X_train = lvl1_rescuer_oof_mean.fit_transform(X_train)\n",
    "    X_test = lvl1_rescuer_oof_mean.transform(X_test)\n",
    "    \n",
    "    if SUBMISSION or VERBOSE:\n",
    "        display(X_train.filter(regex='(oof_lvl1_|pred_)').corr())\n",
    "        display(X_test.filter(regex='(oof_lvl1_|pred_)').corr())\n",
    "    if SUBMISSION:\n",
    "        X_test.filter(regex='(oof_lvl1_|pred_)').to_csv('X_test_lvl1.csv', index=False)\n",
    "    return X_train,X_test\n",
    "\n",
    "def run_level2(X_train, y_train, X_test):\n",
    "    normal_feats = [\n",
    "        'Type', 'Age', 'Breed1', 'Breed2', 'Gender',\n",
    "        'Color1', 'Color2', 'Color3', 'MaturitySize', \n",
    "        'FurLength', 'Vaccinated', 'Dewormed', 'Sterilized',\n",
    "        'Health', 'Quantity', 'Fee', 'State', 'VideoAmt', 'PhotoAmt',\n",
    "        'Population2', '2016 GDP', 'adopted', 'MoreThanOneRescuerPet',\n",
    "    ]\n",
    "    lvl2 = PandasFeatureUnion([\n",
    "        ('Best LGB', PredictTransformer(make_pipeline(\n",
    "            SelectColumns(regex=r'(lvl0|lvl1|label_score|vertex_y|%s)' % '|'.join(normal_feats)),\n",
    "            LGBMRegressor(\n",
    "                n_estimators=100,\n",
    "                num_leaves=16,\n",
    "                learning_rate=0.1,\n",
    "                max_depth=5,\n",
    "                colsample_bytree=0.5,\n",
    "                min_split_gain=0.02,\n",
    "                min_child_samples=100,\n",
    "                min_child_weight=0.02,\n",
    "                reg_lambda=0.01,\n",
    "                random_state=SEED+74,\n",
    "                objective='rmse'\n",
    "                #verbose=1\n",
    "            )\n",
    "        ), name='lvl2_lgb')),\n",
    "        ('Best CatBoost', PredictTransformer(make_pipeline(\n",
    "            SelectColumns(regex=r'(lvl0|lvl1|label_score|vertex_y|%s)' % '|'.join(normal_feats)),\n",
    "            CatBoostRegressor(\n",
    "                n_estimators=250,\n",
    "                learning_rate=0.1,\n",
    "                max_depth=6,\n",
    "                colsample_bylevel=0.5,\n",
    "                l2_leaf_reg=1.,\n",
    "                random_state=SEED+1932,\n",
    "                verbose=-1\n",
    "            )\n",
    "        ), name='lvl2_catboost')),\n",
    "        ('Best Ridge', PredictTransformer(make_pipeline(\n",
    "            SelectColumns(regex=r'(oof_lvl1_lgb)'),\n",
    "            DropColumns(regex=r'(RescuerID)'),\n",
    "            BaggingRegressor(\n",
    "                BayesianRidge(\n",
    "                    normalize=False,\n",
    "                    fit_intercept=True\n",
    "                ),\n",
    "                n_estimators=100,\n",
    "                max_samples=0.8,\n",
    "                max_features=1.0,\n",
    "                bootstrap=True,\n",
    "                random_state=93843\n",
    "            )\n",
    "        ), name='lvl2_ridge'))\n",
    "    ], n_jobs=1)\n",
    "    X_train = lvl2.fit_transform(X_train, y_train)\n",
    "    X_test = lvl2.transform(X_test)\n",
    "    \n",
    "    if SUBMISSION or VERBOSE:\n",
    "        display(X_train.filter(regex='(oof_lvl2_|pred_)').corr())\n",
    "        display(X_test.filter(regex='(oof_lvl2_|pred_)').corr())\n",
    "    if SUBMISSION:\n",
    "        X_test.filter(regex='(oof_lvl2_|pred_)').to_csv('X_test_lvl2.csv', index=False)\n",
    "    return X_train,X_test\n",
    "\n",
    "def run_level3(X_train, y_train, X_test):\n",
    "    y_pred = mode(X_test.values.T)[0].T.flatten().astype(np.int32)\n",
    "    return y_pred\n",
    "\n",
    "def run_all(X_train, y_train, X_test):\n",
    "    X_train = X_train.reset_index(drop=True).copy()\n",
    "    X_test = X_test.reset_index(drop=True).copy()\n",
    "    \n",
    "    # Common preprocessing\n",
    "    if VERBOSE or SUBMISSION:\n",
    "        logger.info('>> Fitting basic pipeline...')\n",
    "    pl = PandasPipeline([\n",
    "        ('extract_features', PandasFeatureUnion([\n",
    "            ('Original features', PandasTransform(lambda x: x)),\n",
    "            ('Row features', PandasTransform(row_features)),\n",
    "            ('Leaky features', ExtractLeaky(X_train, X_test))\n",
    "        ])),\n",
    "        ('Common values', FilterCommonValues(X_train, X_test, ['Breed1','Breed2','State'])),\n",
    "        ('Categoricals', ConvertType([\n",
    "            'Type','Breed1','Breed2','Gender','Color1','Color2','Color3','MaturitySize',\n",
    "            'FurLength','Vaccinated','Dewormed','Sterilized','Health','State','adopted'], \n",
    "            'category'))\n",
    "    ])\n",
    "    X_train = pl.fit_transform(X_train)\n",
    "    X_test = pl.transform(X_test)\n",
    "\n",
    "    use_feats = (X_train.iloc[0:0].select_dtypes(include=['category','number'])\n",
    "                 .columns.tolist() + ['RescuerID','Description'])\n",
    "    X_train,X_test = X_train[use_feats],X_test[use_feats]\n",
    "    \n",
    "    if VERBOSE or SUBMISSION:\n",
    "        logger.info('>> Level 0...')\n",
    "    X_train,X_test = run_level0(X_train, y_train, X_test)\n",
    "    \n",
    "    if VERBOSE:\n",
    "        logger.info('>> Level 1...')\n",
    "    X_train,X_test = run_level1(X_train, y_train, X_test)\n",
    "    \n",
    "    if VERBOSE:\n",
    "        logger.info('>> Level 2...')\n",
    "    X_train,X_test = run_level2(X_train, y_train, X_test)\n",
    "    \n",
    "    if VERBOSE:\n",
    "        logger.info('>> Level 3...')\n",
    "    y_pred = run_level3(X_train, y_train, X_test)\n",
    "    if VERBOSE or SUBMISSION:\n",
    "        display(y_pred)\n",
    "    return y_pred\n",
    "    \n",
    "USE_LB = False\n",
    "    \n",
    "if not SUBMISSION:\n",
    "    validate(X_train, y_train, rescuer_ids)\n",
    "    \n",
    "if LB:\n",
    "    USE_LB = True\n",
    "    y_pred = run_all(X_train, y_train, X_test)\n",
    "    msg = 'LB score: %.3f' % qwk(y_pred, y_test)\n",
    "    logger.info(msg)\n",
    "    \n",
    "if SUBMISSION and not LB:\n",
    "    y_pred = pd.DataFrame(run_all(X_train, y_train, X_test),\n",
    "                          columns=['AdoptionSpeed'], \n",
    "                          index=test['PetID']).reset_index()\n",
    "    y_pred.to_csv('submission.csv', index=False)\n",
    "    \n",
    "    fig,ax = plt.subplots(1,2,figsize=(16,4))\n",
    "    y_pred['AdoptionSpeed'].hist(ax=ax[0])\n",
    "    train['AdoptionSpeed'].hist(ax=ax[1])\n",
    "    ax[0].set_title('Prediction distribution')\n",
    "    ax[1].set_title('Train set distribution')\n",
    "else:\n",
    "    logger.info('Will not make submission without full dataset, turn LB off!')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}