4-modeling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "import cPickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "from sklearn.cross_validation import cross_val_score, train_test_split\n",
    "from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier\n",
    "from sklearn.grid_search import GridSearchCV, ParameterGrid\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from time import time\n",
    "import warnings\n",
    "import xgboost as xgb\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Grab the engineered data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def read_pickle(file_name):\n",
    "    f = open(file_name, 'rb')\n",
    "    p = cPickle.load(f)\n",
    "    f.close()\n",
    "    return p\n",
    "\n",
    "\n",
    "train = np.array(read_pickle('data/train.engineered'))\n",
    "test = np.array(read_pickle('data/test.engineered'))\n",
    "outcomes = read_pickle('data/outcomes.engineered')\n",
    "outcomes_le = read_pickle('data/outcomes_le.engineered')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Do a quick check to see how many trees a random forest would need for the test error to settle down."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 18s, sys: 1.56 s, total: 2min 20s\n",
      "Wall time: 9min\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "rf = GridSearchCV(\n",
    "    RandomForestClassifier(random_state = 25), {\n",
    "        'n_estimators': [int(x) for x in np.linspace(100, 3000, 11)]},\n",
    "    n_jobs = -1, scoring = 'log_loss').fit(train, outcomes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[mean: -1.01710, std: 0.02760, params: {'n_estimators': 100},\n",
       " mean: -0.89021, std: 0.01241, params: {'n_estimators': 390},\n",
       " mean: -0.87188, std: 0.01040, params: {'n_estimators': 680},\n",
       " mean: -0.86587, std: 0.01177, params: {'n_estimators': 970},\n",
       " mean: -0.86333, std: 0.01397, params: {'n_estimators': 1260},\n",
       " mean: -0.86109, std: 0.01484, params: {'n_estimators': 1550},\n",
       " mean: -0.85698, std: 0.01294, params: {'n_estimators': 1840},\n",
       " mean: -0.85459, std: 0.01143, params: {'n_estimators': 2130},\n",
       " mean: -0.85363, std: 0.01038, params: {'n_estimators': 2420},\n",
       " mean: -0.85172, std: 0.00872, params: {'n_estimators': 2710},\n",
       " mean: -0.85168, std: 0.00869, params: {'n_estimators': 3000}]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.grid_scores_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Estimate test error of various models w/o doing hyperparameter optimization to get a sense of which models work better than others."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Gradient boosted trees\n",
      "Log loss: 0.831070681851\n",
      "Time elapsed (min): 1.13\n",
      "--------------------------------------------------------------\n",
      "Model: Extra trees\n",
      "Log loss: 1.0903370461\n",
      "Time elapsed (min): 4.3\n",
      "--------------------------------------------------------------\n",
      "Model: SVM (RBF kernal)\n",
      "Log loss: 0.909677745361\n",
      "Time elapsed (min): 13.62\n",
      "--------------------------------------------------------------\n",
      "Model: Logistic regression\n",
      "Log loss: 0.89151937115\n",
      "Time elapsed (min): 0.06\n",
      "--------------------------------------------------------------\n",
      "Model: Random forest\n",
      "Log loss: 0.842249054112\n",
      "Time elapsed (min): 2.41\n",
      "--------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Score</th>\n",
       "      <th>Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.831071</td>\n",
       "      <td>Gradient boosted trees</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.842249</td>\n",
       "      <td>Random forest</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.891519</td>\n",
       "      <td>Logistic regression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.909678</td>\n",
       "      <td>SVM (RBF kernal)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.090337</td>\n",
       "      <td>Extra trees</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Score                   Model\n",
       "0  0.831071  Gradient boosted trees\n",
       "1  0.842249           Random forest\n",
       "2  0.891519     Logistic regression\n",
       "3  0.909678        SVM (RBF kernal)\n",
       "4  1.090337             Extra trees"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def try_models(X, y, estimators, scoring = 'log_loss', cv = 5, n_jobs = -1,\n",
    "              verbose = True):\n",
    "    \"\"\"Estimate test error of models using kfold-cv w/ default hyperparameters,\n",
    "    no hyperparameter tuning at all.\n",
    "    \"\"\"\n",
    "    scores = []\n",
    "    for name, estimator in estimators.items():\n",
    "        try:\n",
    "            start = time()\n",
    "            score = np.absolute(np.mean(cross_val_score(\n",
    "                        estimator, X, y, cv = cv, n_jobs = n_jobs, scoring = scoring)))\n",
    "            scores.append([score, name])\n",
    "            time_elapsed = round((time() - start) / 60, 2)\n",
    "            if verbose:\n",
    "                print 'Model:', name\n",
    "                print 'Log loss:', score\n",
    "                print 'Time elapsed (min):', time_elapsed\n",
    "                print '--------------------------------------------------------------'\n",
    "        except:\n",
    "            print 'Failure to run model:', name\n",
    "            print '--------------------------------------------------------------'\n",
    "    return pd.DataFrame(sorted(scores), columns = ['Score', 'Model'])\n",
    "\n",
    "\n",
    "try_models(train, outcomes, {\n",
    "        'Logistic regression' : LogisticRegression(random_state = 50), \n",
    "        'SVM (RBF kernal)' : SVC(random_state = 25, probability = True, kernel = 'rbf'),\n",
    "        'Random forest' : RandomForestClassifier(random_state = 25, n_estimators = 3000),\n",
    "        'Extra trees' : ExtraTreesClassifier(random_state = 25, n_estimators = 3000),\n",
    "        'Gradient boosted trees' : xgb.XGBClassifier(seed = 25)\n",
    "    })"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's take the best performing models and get test errors after we optimize for their hyperparamters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    train, outcomes, test_size = 0.3, random_state = 1)\n",
    "\n",
    "\n",
    "def hold_out_cv(model, hyperparameters, X_train, X_test, y_train, y_test,\n",
    "                folds = 5):\n",
    "    \"\"\"Given a [model] and a set of possible [hyperparameters], along with \n",
    "    matricies corresponding to hold-out cross-validation, returns a model w/ \n",
    "    optimized hyperparameters, and prints out model evaluation metrics.\n",
    "    \"\"\"\n",
    "    optimized_model = GridSearchCV(model, hyperparameters, cv = folds, \n",
    "                                   n_jobs = -1, scoring = 'log_loss')\n",
    "    optimized_model.fit(X_train, y_train)\n",
    "    print 'Optimized parameters:', optimized_model.best_params_\n",
    "    print 'Mean absolute error:', np.absolute(\n",
    "        optimized_model.score(X_test, y_test))\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimized parameters: {'C': 0.10000000000000001}\n",
      "Mean absolute error: 0.897359484785\n",
      "CPU times: user 2 s, sys: 548 ms, total: 2.54 s\n",
      "Wall time: 17.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "hold_out_cv(LogisticRegression(random_state = 50), {'C': np.logspace(-3, 3, 7)}, \n",
    "            X_train, X_test, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimized parameters: {'min_samples_split': 10, 'max_depth': None, 'min_samples_leaf': 1}\n",
      "Mean absolute error: 0.837451136547\n",
      "CPU times: user 1min 8s, sys: 1.44 s, total: 1min 9s\n",
      "Wall time: 14min 47s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "hold_out_cv(\n",
    "    RandomForestClassifier(n_estimators = 3000, random_state = 25), {\n",
    "        'min_samples_split': [1, 3, 10],\n",
    "        'min_samples_leaf': [1, 3, 10],\n",
    "        'max_depth': [3, None]}, \n",
    "    X_train, X_test, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-320bf3cc0afd>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mget_ipython\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mu'time'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34mu''\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34mu\"hold_out_cv(\\n    xgb.XGBClassifier(learning_rate = 0.05, n_estimators = 200, seed = 25), {\\n        'max_depth': range(3, 10, 2), \\n        'min_child_weight': range(1, 6, 2),\\n        'gamma': [i / 10.0 for i in range(0, 5)], \\n        'reg_alpha': [0.001, 0.01, 0.1, 1, 10, 100, 1000]}, \\n    X_train, X_test, y_train, y_test)\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m/home/jake/miniconda2/envs/shelter-animals/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc\u001b[0m in \u001b[0;36mrun_cell_magic\u001b[1;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[0;32m   2118\u001b[0m             \u001b[0mmagic_arg_s\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvar_expand\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mline\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstack_depth\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2119\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2120\u001b[1;33m                 \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmagic_arg_s\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2121\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2122\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<decorator-gen-60>\u001b[0m in \u001b[0;36mtime\u001b[1;34m(self, line, cell, local_ns)\u001b[0m\n",
      "\u001b[1;32m/home/jake/miniconda2/envs/shelter-animals/lib/python2.7/site-packages/IPython/core/magic.pyc\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(f, *a, **k)\u001b[0m\n\u001b[0;32m    191\u001b[0m     \u001b[1;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    192\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 193\u001b[1;33m         \u001b[0mcall\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    194\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    195\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/jake/miniconda2/envs/shelter-animals/lib/python2.7/site-packages/IPython/core/magics/execution.pyc\u001b[0m in \u001b[0;36mtime\u001b[1;34m(self, line, cell, local_ns)\u001b[0m\n\u001b[0;32m   1171\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m==\u001b[0m\u001b[1;34m'eval'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1172\u001b[0m             \u001b[0mst\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1173\u001b[1;33m             \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcode\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mglob\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlocal_ns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1174\u001b[0m             \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1175\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<timed eval>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32m<ipython-input-8-cecc59a0809d>\u001b[0m in \u001b[0;36mhold_out_cv\u001b[1;34m(model, hyperparameters, X_train, X_test, y_train, y_test, folds)\u001b[0m\n\u001b[0;32m     11\u001b[0m     optimized_model = GridSearchCV(model, hyperparameters, cv = folds, \n\u001b[0;32m     12\u001b[0m                                    n_jobs = -1, scoring = 'log_loss')\n\u001b[1;32m---> 13\u001b[1;33m     \u001b[0moptimized_model\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     14\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[1;34m'Optimized parameters:'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimized_model\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbest_params_\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m     print 'Mean absolute error:', np.absolute(\n",
      "\u001b[1;32m/home/jake/miniconda2/envs/shelter-animals/lib/python2.7/site-packages/sklearn/grid_search.pyc\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m    802\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    803\u001b[0m         \"\"\"\n\u001b[1;32m--> 804\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mParameterGrid\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    805\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    806\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/jake/miniconda2/envs/shelter-animals/lib/python2.7/site-packages/sklearn/grid_search.pyc\u001b[0m in \u001b[0;36m_fit\u001b[1;34m(self, X, y, parameter_iterable)\u001b[0m\n\u001b[0;32m    551\u001b[0m                                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreturn_parameters\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    552\u001b[0m                                     error_score=self.error_score)\n\u001b[1;32m--> 553\u001b[1;33m                 \u001b[1;32mfor\u001b[0m \u001b[0mparameters\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mparameter_iterable\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    554\u001b[0m                 for train, test in cv)\n\u001b[0;32m    555\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/jake/miniconda2/envs/shelter-animals/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m    808\u001b[0m                 \u001b[1;31m# consumption.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    809\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 810\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    811\u001b[0m             \u001b[1;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    812\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/jake/miniconda2/envs/shelter-animals/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36mretrieve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    755\u001b[0m                     \u001b[1;31m# a working pool as they expect.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    756\u001b[0m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_initialize_pool\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 757\u001b[1;33m                 \u001b[1;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    758\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    759\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0miterable\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "%%time\n",
    "hold_out_cv(\n",
    "    xgb.XGBClassifier(learning_rate = 0.05, n_estimators = 200, seed = 25), {\n",
    "        'max_depth': range(3, 10, 2), \n",
    "        'min_child_weight': range(1, 6, 2),\n",
    "        'gamma': [i / 10.0 for i in range(0, 5)], \n",
    "        'reg_alpha': [0.001, 0.01, 0.1, 1, 10, 100, 1000]}, \n",
    "    X_train, X_test, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}