3-transfer-learning-model/Transfer learning notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Introduction\n",
    "\n",
    "In this notebook, we will use transfer learning to create a model for the Kaggle Aerial Cactus Detection competition. This notebook has several sections:\n",
    "1. Setting up\n",
    "    - Import packages\n",
    "    - Set directories\n",
    "- Data preparation\n",
    "    - Import data\n",
    "    - Prepare data for modelling\n",
    "- Import transfer model\n",
    "    - Download VGG16 model structure and weights\n",
    "    - Freeze weights of lower layers\n",
    "- Training our own custom layers on top of transfer model\n",
    "    - Conduct grid search to find optimal structure of top classification layers\n",
    "    - Train top classification layers\n",
    "- Testing model and prepare submission\n",
    "    - Run model on test data\n",
    "    - Prepare submission file for Kaggle competition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setting up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# Import packages\n",
    "import cv2\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "import os\n",
    "from tqdm import tqdm, tqdm_notebook\n",
    "from keras.models import Sequential, Model, load_model\n",
    "from keras.layers import Activation, Dropout, Flatten, Dense\n",
    "from keras.applications import VGG16\n",
    "from keras import applications\n",
    "from keras.optimizers import Adam\n",
    "from keras import regularizers\n",
    "from keras.optimizers import Adam\n",
    "from keras.optimizers import SGD\n",
    "\n",
    "from keras.constraints import maxnorm\n",
    "from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\n",
    "\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, roc_auc_score, classification_report\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import RandomizedSearchCV,KFold\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv2D, Dense, Flatten, BatchNormalization, Dropout, LeakyReLU, Flatten\n",
    "from keras import regularizers\n",
    "from keras.optimizers import Adam\n",
    "from keras.optimizers import SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>has_cactus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0004be2cfeaba1c0361d39e2b000257b.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000c8a36845c0208e833c79c1bffedd1.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000d1e9a533f62e55c289303b072733d.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0011485b40695e9138e92d0b3fb55128.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0014d7a11e90b62848904c1418fc8cf2.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     id  has_cactus\n",
       "0  0004be2cfeaba1c0361d39e2b000257b.jpg           1\n",
       "1  000c8a36845c0208e833c79c1bffedd1.jpg           1\n",
       "2  000d1e9a533f62e55c289303b072733d.jpg           1\n",
       "3  0011485b40695e9138e92d0b3fb55128.jpg           1\n",
       "4  0014d7a11e90b62848904c1418fc8cf2.jpg           1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Set train and test paths to data. Import training labels as a dataframe\n",
    "train_dir = '../input/train/train/'\n",
    "test_dir = '../input/test/test/'\n",
    "train_df = pd.read_csv('../input/train.csv')\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef0b8dcfe982463db129149aee4ba2cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=17500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Prepare training images for modeling by appending them to a list and converting to np arrays\n",
    "X_tr = []\n",
    "Y_tr = []\n",
    "imges = train_df['id'].values\n",
    "for img_id in tqdm_notebook(imges):\n",
    "    X_tr.append(cv2.imread(train_dir + img_id))    \n",
    "    Y_tr.append(train_df[train_df['id'] == img_id]['has_cactus'].values[0])  \n",
    "X_tr = np.asarray(X_tr)\n",
    "X_tr = X_tr.astype('float32')\n",
    "X_tr /= 255\n",
    "Y_tr = np.asarray(Y_tr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0b02d9d5de247399db97d1e0b10b46d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "CPU times: user 524 ms, sys: 468 ms, total: 992 ms\n",
      "Wall time: 2.59 s\n"
     ]
    }
   ],
   "source": [
    "# Prepare test images for testing by appending them to a list and converting to np arrays\n",
    "X_tst = []\n",
    "Test_imgs = []\n",
    "for img_id in tqdm_notebook(os.listdir(test_dir)):\n",
    "    X_tst.append(cv2.imread(test_dir + img_id))     \n",
    "    Test_imgs.append(img_id)\n",
    "X_tst = np.asarray(X_tst)\n",
    "X_tst = X_tst.astype('float32')\n",
    "X_tst /= 255"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import transfer model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5\n",
      "58892288/58889256 [==============================] - 2s 0us/step\n"
     ]
    }
   ],
   "source": [
    "# Download the VGG16 model that we are going to use for transfer learning\n",
    "# Include_top=False means that we are not going to include the top 3 classification layers\n",
    "# We are going to create our own top classification layers based on our cactus data\n",
    "base_model = applications.VGG16(weights='imagenet', include_top=False, input_shape=(32, 32, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 32, 32, 3)         0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 32, 32, 64)        1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 32, 32, 64)        36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 16, 16, 128)       73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 16, 16, 128)       147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 8, 8, 256)         295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 8, 8, 256)         590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 8, 8, 256)         590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 4, 4, 512)         1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 4, 4, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 4, 4, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 2, 2, 512)         0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 1, 1, 512)         0         \n",
      "=================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 0\n",
      "Non-trainable params: 14,714,688\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Freeze the weights of the convolutional layers and pooling layers - we will use the pre-trained weights\n",
    "base_model.trainable = False \n",
    "base_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training our own custom layers on top of transfer model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a function to create the model in order to be able to do grid search\n",
    "def create_model(optimizer='adam',\n",
    "                 nn=128,  lr=0.01, l1=0.01, l2=0.01,\n",
    "                activation_1 = 'relu', activation_2='sigmoid', dropout=0):  #Set default values for hyperparameters\n",
    "    \n",
    "    reg = regularizers.l1_l2(l1=l1, l2=l2)\n",
    "    model = Sequential()\n",
    "    model.add(base_model)\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(nn,activation = activation_1, kernel_regularizer=reg))\n",
    "    model.add(Dropout(dropout))\n",
    "    model.add(Dense(1, activation = activation_2))\n",
    "    \n",
    "    \n",
    "    # Define optimizer. \n",
    "    if optimizer =='sgd':\n",
    "        optimizer = SGD(lr=lr)\n",
    "    elif optimizer == 'adam':\n",
    "        optimizer = Adam(lr=lr)\n",
    "    elif optimizer == 'rmspop':\n",
    "        optimizer = RMSprop(lr=lr)\n",
    "        \n",
    "    # Compile model\n",
    "    model.compile(optimizer = optimizer , loss= 'binary_crossentropy', metrics = ['accuracy'])\n",
    "    \n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Definie paramter grid values for random grid search.\n",
    "\n",
    "# Optimizer options\n",
    "optimizer = ['rmsprop','sgd','adam']\n",
    "\n",
    "# Learning algorithm parameters\n",
    "lr=[1e-2, 1e-3, 1e-4, 1e-5]\n",
    "\n",
    "\n",
    "# Activation\n",
    "activation_1 =['relu', 'sigmoid', 'tanh']\n",
    "activation_2 =['relu', 'sigmoid', 'tanh']\n",
    "\n",
    "# Neurons in each layer\n",
    "nn=[128,256,512,1024]\n",
    "\n",
    "\n",
    "# Dropout and regularisation\n",
    "dropout = [0, 0.1, 0.2, 0.3, 0.4 ,0.5]\n",
    "\n",
    "l1 = [0, 0.01, 0.003, 0.001,0.0001]\n",
    "l2 = [0, 0.01, 0.003, 0.001,0.0001]\n",
    "\n",
    "# Dictionary summary\n",
    "param_grid = dict(\n",
    "                  nn=nn,\n",
    "                  activation_1=activation_1, activation_2=activation_2, l1=l1, l2=l2, lr=lr, dropout =dropout, \n",
    "                  optimizer=optimizer\n",
    "                 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model class to use in the scikit random search CV \n",
    "# Grid search only on 10 epochs -- this is all the Kaggle kernel can handle in the allowed time\n",
    "model = KerasClassifier(build_fn=create_model, epochs=10, batch_size=128, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define grid search\n",
    "# We use 3-fold cross validation and will be using the entire dataset for the gridsearch\n",
    "grid = RandomizedSearchCV(estimator=model, cv=KFold(3), param_distributions=param_grid, \n",
    "                          verbose=10,  n_iter=10, n_jobs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define where weights will be stored and callbacks for training. \n",
    "file_path = 'weights-aerial-cactus.h5'\n",
    "\n",
    "callbacks = [\n",
    "        ModelCheckpoint(file_path, monitor = 'val_acc', verbose = 1, save_best_only = True, mode = 'max'),\n",
    "        ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, patience = 3, verbose = 1, mode = 'min', min_lr = 0.00001),\n",
    "        EarlyStopping(monitor = 'val_loss', min_delta = 1e-10, patience = 15, verbose = 1, restore_best_weights = True)\n",
    "        ]\n",
    "\n",
    "training_path = train_dir\n",
    "test_path = test_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n",
      "[CV] optimizer=adam, nn=512, lr=0.001, l2=0.01, l1=0, dropout=0.5, activation_2=sigmoid, activation_1=relu \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "11666/11666 [==============================] - 5s 440us/step - loss: 1.6744 - acc: 0.9024\n",
      "Epoch 2/10\n",
      " 1664/11666 [===>..........................] - ETA: 1s - loss: 0.3387 - acc: 0.9303"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/keras/callbacks.py:434: RuntimeWarning: Can save best model only with val_acc available, skipping.\n",
      "  'skipping.' % (self.monitor), RuntimeWarning)\n",
      "/opt/conda/lib/python3.6/site-packages/keras/callbacks.py:1109: RuntimeWarning: Reduce LR on plateau conditioned on metric `val_loss` which is not available. Available metrics are: loss,acc,lr\n",
      "  (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning\n",
      "/opt/conda/lib/python3.6/site-packages/keras/callbacks.py:569: RuntimeWarning: Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss,acc,lr\n",
      "  (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11666/11666 [==============================] - 1s 106us/step - loss: 0.2738 - acc: 0.9434\n",
      "Epoch 3/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 0.2245 - acc: 0.9441\n",
      "Epoch 4/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 0.2096 - acc: 0.9446\n",
      "Epoch 5/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 0.1897 - acc: 0.9513\n",
      "Epoch 6/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 0.1893 - acc: 0.9480\n",
      "Epoch 7/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 0.1739 - acc: 0.9535\n",
      "Epoch 8/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 0.1729 - acc: 0.9526\n",
      "Epoch 9/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 0.1682 - acc: 0.9535\n",
      "Epoch 10/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 0.1813 - acc: 0.9480\n",
      "5834/5834 [==============================] - 1s 125us/step\n",
      "[CV]  optimizer=adam, nn=512, lr=0.001, l2=0.01, l1=0, dropout=0.5, activation_2=sigmoid, activation_1=relu, score=0.957, total=  17.3s\n",
      "[CV] optimizer=adam, nn=512, lr=0.001, l2=0.01, l1=0, dropout=0.5, activation_2=sigmoid, activation_1=relu \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:   17.3s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "11667/11667 [==============================] - 2s 137us/step - loss: 1.7575 - acc: 0.8846\n",
      "Epoch 2/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.3067 - acc: 0.9469\n",
      "Epoch 3/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.2390 - acc: 0.9433\n",
      "Epoch 4/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 0.2034 - acc: 0.9529\n",
      "Epoch 5/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 0.1932 - acc: 0.9521\n",
      "Epoch 6/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.1881 - acc: 0.9505\n",
      "Epoch 7/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.1831 - acc: 0.9497\n",
      "Epoch 8/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.1721 - acc: 0.9544\n",
      "Epoch 9/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.1692 - acc: 0.9537\n",
      "Epoch 10/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.1701 - acc: 0.9531\n",
      "5833/5833 [==============================] - 1s 128us/step\n",
      "[CV]  optimizer=adam, nn=512, lr=0.001, l2=0.01, l1=0, dropout=0.5, activation_2=sigmoid, activation_1=relu, score=0.957, total=  13.8s\n",
      "[CV] optimizer=adam, nn=512, lr=0.001, l2=0.01, l1=0, dropout=0.5, activation_2=sigmoid, activation_1=relu \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:   31.1s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "11667/11667 [==============================] - 2s 137us/step - loss: 1.7601 - acc: 0.8805\n",
      "Epoch 2/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.3078 - acc: 0.9409\n",
      "Epoch 3/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.2395 - acc: 0.9445\n",
      "Epoch 4/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.2098 - acc: 0.9464\n",
      "Epoch 5/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.1945 - acc: 0.9505\n",
      "Epoch 6/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.1868 - acc: 0.9505\n",
      "Epoch 7/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.1797 - acc: 0.9525\n",
      "Epoch 8/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.1799 - acc: 0.9506\n",
      "Epoch 9/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.1748 - acc: 0.9521\n",
      "Epoch 10/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.1707 - acc: 0.9517\n",
      "5833/5833 [==============================] - 1s 111us/step\n",
      "[CV]  optimizer=adam, nn=512, lr=0.001, l2=0.01, l1=0, dropout=0.5, activation_2=sigmoid, activation_1=relu, score=0.957, total=  13.6s\n",
      "[CV] optimizer=sgd, nn=1024, lr=1e-05, l2=0, l1=0.003, dropout=0.5, activation_2=sigmoid, activation_1=relu \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:   44.7s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "11666/11666 [==============================] - 2s 136us/step - loss: 49.8823 - acc: 0.4583\n",
      "Epoch 2/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 49.8583 - acc: 0.4865\n",
      "Epoch 3/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 49.8421 - acc: 0.5018\n",
      "Epoch 4/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 49.8235 - acc: 0.5289\n",
      "Epoch 5/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 49.8082 - acc: 0.5485\n",
      "Epoch 6/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 49.7915 - acc: 0.5719\n",
      "Epoch 7/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 49.7810 - acc: 0.5826\n",
      "Epoch 8/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 49.7641 - acc: 0.6100\n",
      "Epoch 9/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 49.7572 - acc: 0.6103\n",
      "Epoch 10/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 49.7422 - acc: 0.6321\n",
      "5834/5834 [==============================] - 1s 114us/step\n",
      "[CV]  optimizer=sgd, nn=1024, lr=1e-05, l2=0, l1=0.003, dropout=0.5, activation_2=sigmoid, activation_1=relu, score=0.744, total=  13.5s\n",
      "[CV] optimizer=sgd, nn=1024, lr=1e-05, l2=0, l1=0.003, dropout=0.5, activation_2=sigmoid, activation_1=relu \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:   58.2s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "11667/11667 [==============================] - 2s 137us/step - loss: 49.8151 - acc: 0.5078\n",
      "Epoch 2/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 49.7961 - acc: 0.5322\n",
      "Epoch 3/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 49.7869 - acc: 0.5450\n",
      "Epoch 4/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 49.7676 - acc: 0.5714\n",
      "Epoch 5/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 49.7517 - acc: 0.5906\n",
      "Epoch 6/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 49.7416 - acc: 0.6020\n",
      "Epoch 7/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 49.7323 - acc: 0.6175\n",
      "Epoch 8/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 49.7191 - acc: 0.6318\n",
      "Epoch 9/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 49.7022 - acc: 0.6539\n",
      "Epoch 10/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 49.6965 - acc: 0.6537\n",
      "5833/5833 [==============================] - 1s 116us/step\n",
      "[CV]  optimizer=sgd, nn=1024, lr=1e-05, l2=0, l1=0.003, dropout=0.5, activation_2=sigmoid, activation_1=relu, score=0.751, total=  13.4s\n",
      "[CV] optimizer=sgd, nn=1024, lr=1e-05, l2=0, l1=0.003, dropout=0.5, activation_2=sigmoid, activation_1=relu \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:  1.2min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "11667/11667 [==============================] - 2s 140us/step - loss: 49.7917 - acc: 0.6602\n",
      "Epoch 2/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 49.7837 - acc: 0.6631\n",
      "Epoch 3/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 49.7753 - acc: 0.6736\n",
      "Epoch 4/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 49.7673 - acc: 0.6778\n",
      "Epoch 5/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 49.7598 - acc: 0.6804\n",
      "Epoch 6/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 49.7555 - acc: 0.6852\n",
      "Epoch 7/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 49.7469 - acc: 0.6920\n",
      "Epoch 8/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 49.7393 - acc: 0.6953\n",
      "Epoch 9/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 49.7286 - acc: 0.7052\n",
      "Epoch 10/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 49.7221 - acc: 0.7010\n",
      "5833/5833 [==============================] - 1s 120us/step\n",
      "[CV]  optimizer=sgd, nn=1024, lr=1e-05, l2=0, l1=0.003, dropout=0.5, activation_2=sigmoid, activation_1=relu, score=0.754, total=  13.5s\n",
      "[CV] optimizer=sgd, nn=1024, lr=0.01, l2=0.01, l1=0, dropout=0.3, activation_2=relu, activation_1=tanh \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:  1.4min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "11666/11666 [==============================] - 2s 144us/step - loss: 10.7379 - acc: 0.0025\n",
      "Epoch 2/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 10.4078 - acc: 0.0000e+00\n",
      "Epoch 3/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 10.1739 - acc: 0.0000e+00\n",
      "Epoch 4/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 9.9485 - acc: 0.0000e+00\n",
      "Epoch 5/10\n",
      "11666/11666 [==============================] - 1s 104us/step - loss: 9.7312 - acc: 0.0000e+00\n",
      "Epoch 6/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 9.5218 - acc: 0.0000e+00\n",
      "Epoch 7/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 9.3200 - acc: 0.0000e+00\n",
      "Epoch 8/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 9.1254 - acc: 0.0000e+00\n",
      "Epoch 9/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 8.9379 - acc: 0.0000e+00\n",
      "Epoch 10/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 8.7571 - acc: 0.0000e+00\n",
      "5834/5834 [==============================] - 1s 123us/step\n",
      "[CV]  optimizer=sgd, nn=1024, lr=0.01, l2=0.01, l1=0, dropout=0.3, activation_2=relu, activation_1=tanh, score=0.000, total=  13.7s\n",
      "[CV] optimizer=sgd, nn=1024, lr=0.01, l2=0.01, l1=0, dropout=0.3, activation_2=relu, activation_1=tanh \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:  1.6min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "11667/11667 [==============================] - 2s 147us/step - loss: 10.6884 - acc: 0.0049\n",
      "Epoch 2/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 10.4442 - acc: 0.0000e+00\n",
      "Epoch 3/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 10.2105 - acc: 0.0000e+00\n",
      "Epoch 4/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 9.9852 - acc: 0.0000e+00\n",
      "Epoch 5/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 9.7681 - acc: 0.0000e+00\n",
      "Epoch 6/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 9.5588 - acc: 0.0000e+00\n",
      "Epoch 7/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 9.3571 - acc: 0.0000e+00\n",
      "Epoch 8/10\n",
      "11667/11667 [==============================] - 1s 100us/step - loss: 9.1627 - acc: 0.0000e+00\n",
      "Epoch 9/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 8.9753 - acc: 0.0000e+00\n",
      "Epoch 10/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 8.7946 - acc: 0.0000e+00\n",
      "5833/5833 [==============================] - 1s 126us/step\n",
      "[CV]  optimizer=sgd, nn=1024, lr=0.01, l2=0.01, l1=0, dropout=0.3, activation_2=relu, activation_1=tanh, score=0.000, total=  13.6s\n",
      "[CV] optimizer=sgd, nn=1024, lr=0.01, l2=0.01, l1=0, dropout=0.3, activation_2=relu, activation_1=tanh \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:  1.9min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "11667/11667 [==============================] - 2s 149us/step - loss: 10.7270 - acc: 0.0129\n",
      "Epoch 2/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 10.4889 - acc: 0.0000e+00\n",
      "Epoch 3/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 10.2550 - acc: 0.0000e+00\n",
      "Epoch 4/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 10.0295 - acc: 0.0000e+00\n",
      "Epoch 5/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 9.8122 - acc: 0.0000e+00\n",
      "Epoch 6/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 9.6027 - acc: 0.0000e+00\n",
      "Epoch 7/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 9.4007 - acc: 0.0000e+00\n",
      "Epoch 8/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 9.2061 - acc: 0.0000e+00\n",
      "Epoch 9/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 9.0185 - acc: 0.0000e+00\n",
      "Epoch 10/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 8.8377 - acc: 0.0000e+00\n",
      "5833/5833 [==============================] - 1s 127us/step\n",
      "[CV]  optimizer=sgd, nn=1024, lr=0.01, l2=0.01, l1=0, dropout=0.3, activation_2=relu, activation_1=tanh, score=0.000, total=  13.7s\n",
      "[CV] optimizer=sgd, nn=512, lr=0.0001, l2=0.0001, l1=0.0001, dropout=0.4, activation_2=tanh, activation_1=tanh \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:  2.1min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "11666/11666 [==============================] - 2s 155us/step - loss: 2.0427 - acc: 0.6968\n",
      "Epoch 2/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 1.9070 - acc: 0.7209\n",
      "Epoch 3/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 1.8656 - acc: 0.7272\n",
      "Epoch 4/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 1.8249 - acc: 0.7294\n",
      "Epoch 5/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 1.8127 - acc: 0.7340\n",
      "Epoch 6/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 1.7754 - acc: 0.7337\n",
      "Epoch 7/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 1.7688 - acc: 0.7518\n",
      "Epoch 8/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 1.7205 - acc: 0.7528\n",
      "Epoch 9/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 1.7032 - acc: 0.7542\n",
      "Epoch 10/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 1.6909 - acc: 0.7554\n",
      "5834/5834 [==============================] - 1s 131us/step\n",
      "[CV]  optimizer=sgd, nn=512, lr=0.0001, l2=0.0001, l1=0.0001, dropout=0.4, activation_2=tanh, activation_1=tanh, score=0.754, total=  13.9s\n",
      "[CV] optimizer=sgd, nn=512, lr=0.0001, l2=0.0001, l1=0.0001, dropout=0.4, activation_2=tanh, activation_1=tanh \n",
      "Epoch 1/10\n",
      "11667/11667 [==============================] - 2s 160us/step - loss: 1.9606 - acc: 0.7224\n",
      "Epoch 2/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.7821 - acc: 0.7213\n",
      "Epoch 3/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 1.7606 - acc: 0.7303\n",
      "Epoch 4/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.7780 - acc: 0.7401\n",
      "Epoch 5/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.7276 - acc: 0.7355\n",
      "Epoch 6/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.7073 - acc: 0.7355\n",
      "Epoch 7/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 1.6893 - acc: 0.7395\n",
      "Epoch 8/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.6784 - acc: 0.7405\n",
      "Epoch 9/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 1.6707 - acc: 0.7409\n",
      "Epoch 10/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 1.6602 - acc: 0.7429\n",
      "5833/5833 [==============================] - 1s 135us/step\n",
      "[CV]  optimizer=sgd, nn=512, lr=0.0001, l2=0.0001, l1=0.0001, dropout=0.4, activation_2=tanh, activation_1=tanh, score=0.752, total=  13.9s\n",
      "[CV] optimizer=sgd, nn=512, lr=0.0001, l2=0.0001, l1=0.0001, dropout=0.4, activation_2=tanh, activation_1=tanh \n",
      "Epoch 1/10\n",
      "11667/11667 [==============================] - 2s 160us/step - loss: 2.0625 - acc: 0.6889\n",
      "Epoch 2/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.7740 - acc: 0.7327\n",
      "Epoch 3/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.7513 - acc: 0.7362\n",
      "Epoch 4/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.7336 - acc: 0.7398\n",
      "Epoch 5/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.7140 - acc: 0.7370\n",
      "Epoch 6/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 1.7103 - acc: 0.7421\n",
      "Epoch 7/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.6709 - acc: 0.7497\n",
      "Epoch 8/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 1.6686 - acc: 0.7522\n",
      "Epoch 9/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.6328 - acc: 0.7559\n",
      "Epoch 10/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.6422 - acc: 0.7669\n",
      "5833/5833 [==============================] - 1s 139us/step\n",
      "[CV]  optimizer=sgd, nn=512, lr=0.0001, l2=0.0001, l1=0.0001, dropout=0.4, activation_2=tanh, activation_1=tanh, score=0.756, total=  13.9s\n",
      "[CV] optimizer=adam, nn=1024, lr=0.0001, l2=0.01, l1=0.003, dropout=0, activation_2=sigmoid, activation_1=relu \n",
      "Epoch 1/10\n",
      "11666/11666 [==============================] - 2s 171us/step - loss: 48.6133 - acc: 0.7543\n",
      "Epoch 2/10\n",
      "11666/11666 [==============================] - 1s 104us/step - loss: 34.3360 - acc: 0.7890\n",
      "Epoch 3/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 22.9807 - acc: 0.8550\n",
      "Epoch 4/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 14.2527 - acc: 0.8885\n",
      "Epoch 5/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 7.8750 - acc: 0.9028\n",
      "Epoch 6/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 3.6168 - acc: 0.9067\n",
      "Epoch 7/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 1.2663 - acc: 0.9067\n",
      "Epoch 8/10\n",
      "11666/11666 [==============================] - 1s 104us/step - loss: 0.5511 - acc: 0.9031\n",
      "Epoch 9/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 0.4877 - acc: 0.9061\n",
      "Epoch 10/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 0.4631 - acc: 0.9064\n",
      "5834/5834 [==============================] - 1s 140us/step\n",
      "[CV]  optimizer=adam, nn=1024, lr=0.0001, l2=0.01, l1=0.003, dropout=0, activation_2=sigmoid, activation_1=relu, score=0.900, total=  14.3s\n",
      "[CV] optimizer=adam, nn=1024, lr=0.0001, l2=0.01, l1=0.003, dropout=0, activation_2=sigmoid, activation_1=relu \n",
      "Epoch 1/10\n",
      "11667/11667 [==============================] - 2s 177us/step - loss: 48.5707 - acc: 0.7500\n",
      "Epoch 2/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 34.3022 - acc: 0.7891\n",
      "Epoch 3/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 22.9563 - acc: 0.8578\n",
      "Epoch 4/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 14.2301 - acc: 0.8884\n",
      "Epoch 5/10\n",
      "11667/11667 [==============================] - 1s 104us/step - loss: 7.8581 - acc: 0.9030\n",
      "Epoch 6/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 3.6013 - acc: 0.9079\n",
      "Epoch 7/10\n",
      "11667/11667 [==============================] - 1s 104us/step - loss: 1.2535 - acc: 0.9083\n",
      "Epoch 8/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.5472 - acc: 0.9050\n",
      "Epoch 9/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.4844 - acc: 0.9059\n",
      "Epoch 10/10\n",
      "11667/11667 [==============================] - 1s 104us/step - loss: 0.4623 - acc: 0.9064\n",
      "5833/5833 [==============================] - 1s 145us/step\n",
      "[CV]  optimizer=adam, nn=1024, lr=0.0001, l2=0.01, l1=0.003, dropout=0, activation_2=sigmoid, activation_1=relu, score=0.890, total=  14.6s\n",
      "[CV] optimizer=adam, nn=1024, lr=0.0001, l2=0.01, l1=0.003, dropout=0, activation_2=sigmoid, activation_1=relu \n",
      "Epoch 1/10\n",
      "11667/11667 [==============================] - 2s 179us/step - loss: 48.6394 - acc: 0.7365\n",
      "Epoch 2/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 34.4192 - acc: 0.8025\n",
      "Epoch 3/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 23.0933 - acc: 0.8619\n",
      "Epoch 4/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 14.3730 - acc: 0.8871\n",
      "Epoch 5/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 7.9915 - acc: 0.8995\n",
      "Epoch 6/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 3.7172 - acc: 0.9055\n",
      "Epoch 7/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 1.3429 - acc: 0.9025\n",
      "Epoch 8/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.6035 - acc: 0.9014\n",
      "Epoch 9/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.5251 - acc: 0.9001\n",
      "Epoch 10/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 0.4912 - acc: 0.9040\n",
      "5833/5833 [==============================] - 1s 148us/step\n",
      "[CV]  optimizer=adam, nn=1024, lr=0.0001, l2=0.01, l1=0.003, dropout=0, activation_2=sigmoid, activation_1=relu, score=0.905, total=  14.4s\n",
      "[CV] optimizer=sgd, nn=256, lr=0.001, l2=0.001, l1=0.0001, dropout=0.4, activation_2=relu, activation_1=tanh \n",
      "Epoch 1/10\n",
      "11666/11666 [==============================] - 2s 177us/step - loss: 8.7959 - acc: 0.3063\n",
      "Epoch 2/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 10.2243 - acc: 0.2987\n",
      "Epoch 3/10\n",
      "11666/11666 [==============================] - 1s 101us/step - loss: 5.3740 - acc: 0.1579\n",
      "Epoch 4/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 7.1222 - acc: 0.0637\n",
      "Epoch 5/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 6.7874 - acc: 0.0467\n",
      "Epoch 6/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 6.1503 - acc: 0.0157\n",
      "Epoch 7/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 6.0561 - acc: 0.0166\n",
      "Epoch 8/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 6.0675 - acc: 0.0161\n",
      "Epoch 9/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 5.9520 - acc: 0.0094\n",
      "Epoch 10/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 5.9864 - acc: 0.0079\n",
      "5834/5834 [==============================] - 1s 156us/step\n",
      "[CV]  optimizer=sgd, nn=256, lr=0.001, l2=0.001, l1=0.0001, dropout=0.4, activation_2=relu, activation_1=tanh, score=0.000, total=  14.3s\n",
      "[CV] optimizer=sgd, nn=256, lr=0.001, l2=0.001, l1=0.0001, dropout=0.4, activation_2=relu, activation_1=tanh \n",
      "Epoch 1/10\n",
      "11667/11667 [==============================] - 2s 183us/step - loss: 3.8030 - acc: 0.3617\n",
      "Epoch 2/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 4.7130 - acc: 0.0418\n",
      "Epoch 3/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 4.8964 - acc: 0.0000e+00\n",
      "Epoch 4/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 4.8959 - acc: 0.0000e+00\n",
      "Epoch 5/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 4.8955 - acc: 0.0000e+00\n",
      "Epoch 6/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 4.8950 - acc: 0.0000e+00\n",
      "Epoch 7/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 4.8946 - acc: 0.0000e+00\n",
      "Epoch 8/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 4.8941 - acc: 0.0000e+00\n",
      "Epoch 9/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 4.8936 - acc: 0.0000e+00\n",
      "Epoch 10/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 4.8932 - acc: 0.0000e+00\n",
      "5833/5833 [==============================] - 1s 156us/step\n",
      "[CV]  optimizer=sgd, nn=256, lr=0.001, l2=0.001, l1=0.0001, dropout=0.4, activation_2=relu, activation_1=tanh, score=0.000, total=  14.3s\n",
      "[CV] optimizer=sgd, nn=256, lr=0.001, l2=0.001, l1=0.0001, dropout=0.4, activation_2=relu, activation_1=tanh \n",
      "Epoch 1/10\n",
      "11667/11667 [==============================] - 2s 184us/step - loss: 8.2046 - acc: 0.3262\n",
      "Epoch 2/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 3.4194 - acc: 0.4058\n",
      "Epoch 3/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 7.2389 - acc: 0.2374\n",
      "Epoch 4/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 9.2936 - acc: 0.1947\n",
      "Epoch 5/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 4.9335 - acc: 0.0000e+00\n",
      "Epoch 6/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 4.9319 - acc: 8.5712e-05\n",
      "Epoch 7/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 4.9326 - acc: 8.5712e-05\n",
      "Epoch 8/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 4.9309 - acc: 8.5712e-05\n",
      "Epoch 9/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 4.9304 - acc: 1.7142e-04\n",
      "Epoch 10/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 4.9312 - acc: 0.0000e+00\n",
      "5833/5833 [==============================] - 1s 160us/step\n",
      "[CV]  optimizer=sgd, nn=256, lr=0.001, l2=0.001, l1=0.0001, dropout=0.4, activation_2=relu, activation_1=tanh, score=0.000, total=  14.3s\n",
      "[CV] optimizer=sgd, nn=256, lr=0.001, l2=0.001, l1=0.0001, dropout=0.1, activation_2=tanh, activation_1=relu \n",
      "Epoch 1/10\n",
      "11666/11666 [==============================] - 2s 192us/step - loss: 1.4245 - acc: 0.7608\n",
      "Epoch 2/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 1.3088 - acc: 0.8280\n",
      "Epoch 3/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 1.3053 - acc: 0.8326\n",
      "Epoch 4/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 1.2344 - acc: 0.8802\n",
      "Epoch 5/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 1.2036 - acc: 0.8973\n",
      "Epoch 6/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 1.1903 - acc: 0.9027\n",
      "Epoch 7/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 1.1771 - acc: 0.9104\n",
      "Epoch 8/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 1.1659 - acc: 0.9104\n",
      "Epoch 9/10\n",
      "11666/11666 [==============================] - 1s 103us/step - loss: 1.1557 - acc: 0.9176\n",
      "Epoch 10/10\n",
      "11666/11666 [==============================] - 1s 102us/step - loss: 1.1513 - acc: 0.9229\n",
      "5834/5834 [==============================] - 1s 166us/step\n",
      "[CV]  optimizer=sgd, nn=256, lr=0.001, l2=0.001, l1=0.0001, dropout=0.1, activation_2=tanh, activation_1=relu, score=0.935, total=  14.8s\n",
      "[CV] optimizer=sgd, nn=256, lr=0.001, l2=0.001, l1=0.0001, dropout=0.1, activation_2=tanh, activation_1=relu \n",
      "Epoch 1/10\n",
      "11667/11667 [==============================] - 2s 195us/step - loss: 1.4325 - acc: 0.7561\n",
      "Epoch 2/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.3281 - acc: 0.8174\n",
      "Epoch 3/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 1.2819 - acc: 0.8503\n",
      "Epoch 4/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.2418 - acc: 0.8762\n",
      "Epoch 5/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 1.2221 - acc: 0.8947\n",
      "Epoch 6/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.1949 - acc: 0.8960\n",
      "Epoch 7/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.1849 - acc: 0.8957\n",
      "Epoch 8/10\n",
      "11667/11667 [==============================] - 1s 101us/step - loss: 1.1599 - acc: 0.9172\n",
      "Epoch 9/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 1.1484 - acc: 0.9195\n",
      "Epoch 10/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 1.1473 - acc: 0.9230\n",
      "5833/5833 [==============================] - 1s 171us/step\n",
      "[CV]  optimizer=sgd, nn=256, lr=0.001, l2=0.001, l1=0.0001, dropout=0.1, activation_2=tanh, activation_1=relu, score=0.931, total=  14.6s\n",
      "[CV] optimizer=sgd, nn=256, lr=0.001, l2=0.001, l1=0.0001, dropout=0.1, activation_2=tanh, activation_1=relu \n",
      "Epoch 1/10\n",
      "11667/11667 [==============================] - 2s 205us/step - loss: 5.0067 - acc: 0.7376\n",
      "Epoch 2/10\n",
      "11667/11667 [==============================] - 1s 105us/step - loss: 4.9418 - acc: 0.7482\n",
      "Epoch 3/10\n",
      "11667/11667 [==============================] - 1s 104us/step - loss: 4.9413 - acc: 0.7482\n",
      "Epoch 4/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 4.9409 - acc: 0.7482\n",
      "Epoch 5/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 4.9404 - acc: 0.7482\n",
      "Epoch 6/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 4.9400 - acc: 0.7482\n",
      "Epoch 7/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 4.9395 - acc: 0.7482\n",
      "Epoch 8/10\n",
      "11667/11667 [==============================] - 1s 103us/step - loss: 4.9390 - acc: 0.7482\n",
      "Epoch 9/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 4.9386 - acc: 0.7482\n",
      "Epoch 10/10\n",
      "11667/11667 [==============================] - 1s 102us/step - loss: 4.9381 - acc: 0.7482\n",
      "5833/5833 [==============================] - 1s 173us/step\n",
      "[CV]  optimizer=sgd, nn=256, lr=0.001, l2=0.001, l1=0.0001, dropout=0.1, activation_2=tanh, activation_1=relu, score=0.756, total=  14.8s\n",
      "[CV] optimizer=sgd, nn=128, lr=0.001, l2=0.003, l1=0, dropout=0.3, activation_2=sigmoid, activation_1=sigmoid \n",
      "Epoch 1/10\n",
      "11666/11666 [==============================] - 2s 202us/step - loss: 1.3955 - acc: 0.4457\n",
      "Epoch 2/10\n",
      " 2176/11666 [====>.........................] - ETA: 0s - loss: 1.2410 - acc: 0.6783"
     ]
    }
   ],
   "source": [
    "# Perform grid search\n",
    "grid_result = grid.fit(X_tr, Y_tr, callbacks = callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'optimizer': 'adam', 'nn': 512, 'lr': 0.001, 'l2': 0.01, 'l1': 0, 'dropout': 0.5, 'activation_2': 'sigmoid', 'activation_1': 'relu'}\n"
     ]
    }
   ],
   "source": [
    "# Print parameters of model with best results\n",
    "print(grid_result.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save best model .\n",
    "best_model = grid_result.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing model and prepare submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4000/4000 [==============================] - 1s 289us/step\n"
     ]
    }
   ],
   "source": [
    "# Prediction\n",
    "test_predictions = best_model.predict(X_tst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create predictions data frame\n",
    "sub_df = pd.DataFrame(test_predictions, columns=['has_cactus'])\n",
    "\n",
    "# we predict a cactus if according to the model there is over 75% chance of there being a cactus\n",
    "sub_df['has_cactus'] = sub_df['has_cactus'].apply(lambda x: 1 if x > 0.75 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create id column and rearrange\n",
    "sub_df['id'] = ''\n",
    "cols = sub_df.columns.tolist()\n",
    "cols = cols[-1:] + cols[:-1]\n",
    "sub_df=sub_df[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>has_cactus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3970</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3971</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3972</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3973</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3974</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3975</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3976</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3977</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3978</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3979</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3980</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3981</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3982</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3983</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3984</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3985</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3986</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3987</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3988</th>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3989</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3990</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3991</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3992</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3993</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3994</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3995</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3996</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3997</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3998</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     id  has_cactus\n",
       "0                 1\n",
       "1                 1\n",
       "2                 1\n",
       "3                 1\n",
       "4                 1\n",
       "5                 1\n",
       "6                 0\n",
       "7                 1\n",
       "8                 1\n",
       "9                 0\n",
       "10                0\n",
       "11                0\n",
       "12                1\n",
       "13                0\n",
       "14                1\n",
       "15                1\n",
       "16                1\n",
       "17                1\n",
       "18                1\n",
       "19                1\n",
       "20                1\n",
       "21                1\n",
       "22                1\n",
       "23                1\n",
       "24                0\n",
       "25                1\n",
       "26                1\n",
       "27                1\n",
       "28                1\n",
       "29                0\n",
       "...  ..         ...\n",
       "3970              1\n",
       "3971              1\n",
       "3972              0\n",
       "3973              0\n",
       "3974              1\n",
       "3975              0\n",
       "3976              1\n",
       "3977              0\n",
       "3978              1\n",
       "3979              1\n",
       "3980              0\n",
       "3981              1\n",
       "3982              1\n",
       "3983              0\n",
       "3984              0\n",
       "3985              1\n",
       "3986              1\n",
       "3987              1\n",
       "3988              0\n",
       "3989              1\n",
       "3990              1\n",
       "3991              1\n",
       "3992              1\n",
       "3993              1\n",
       "3994              1\n",
       "3995              1\n",
       "3996              1\n",
       "3997              1\n",
       "3998              1\n",
       "3999              1\n",
       "\n",
       "[4000 rows x 2 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:2: FutureWarning: set_value is deprecated and will be removed in a future release. Please use .at[] or .iat[] accessors instead\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "for i, img in enumerate(Test_imgs):\n",
    "    sub_df.set_value(i,'id',img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>has_cactus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6a571b6df250e9575fb82f8904c325a1.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2edb0bf826248b088d57e22799464c41.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ea9422f63363a362ba6f482617006e76.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1021509e308bf12f71a01cac2ddca97f.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5eacbb413e5cd4e73cb7b1936758abf1.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     id  has_cactus\n",
       "0  6a571b6df250e9575fb82f8904c325a1.jpg           1\n",
       "1  2edb0bf826248b088d57e22799464c41.jpg           1\n",
       "2  ea9422f63363a362ba6f482617006e76.jpg           1\n",
       "3  1021509e308bf12f71a01cac2ddca97f.jpg           1\n",
       "4  5eacbb413e5cd4e73cb7b1936758abf1.jpg           1"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_df.to_csv('submission.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "1f971f28f11d40b59208f02024efd0d2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a3bf42273cfd49c9b5d014bcc1da2ee2",
       "placeholder": "​",
       "style": "IPY_MODEL_e2b2d226f56d4aa9992e7380e4839e4f",
       "value": "100% 4000/4000 [00:02&lt;00:00, 1591.99it/s]"
      }
     },
     "46607bc963d042678b76af833d3018a2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ecd23094382044dc90aa32bddd8d6d52",
       "max": 17500,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_eb7addab87f34562856a6b3844c25c37",
       "value": 17500
      }
     },
     "5d089e09a4a741e6a8a984292de328af": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7978f57e98de4d49bdd6b6f61f75f1af": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "8512c279d35a443ea6bfb80ea255ab58": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a0e7e79e03a74b69ba2d05fb8de20a11": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "a3bf42273cfd49c9b5d014bcc1da2ee2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b0b02d9d5de247399db97d1e0b10b46d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_bfe9b906d825425bb8291442bdee613a",
        "IPY_MODEL_1f971f28f11d40b59208f02024efd0d2"
       ],
       "layout": "IPY_MODEL_ccfb744bfd3f4748b59b4b548072d62b"
      }
     },
     "bfe9b906d825425bb8291442bdee613a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_8512c279d35a443ea6bfb80ea255ab58",
       "max": 4000,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_a0e7e79e03a74b69ba2d05fb8de20a11",
       "value": 4000
      }
     },
     "ccfb744bfd3f4748b59b4b548072d62b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e2b2d226f56d4aa9992e7380e4839e4f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "eb7addab87f34562856a6b3844c25c37": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "ecd23094382044dc90aa32bddd8d6d52": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ef0b8dcfe982463db129149aee4ba2cc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_46607bc963d042678b76af833d3018a2",
        "IPY_MODEL_f56ae0f406344977acb852ba25f79a49"
       ],
       "layout": "IPY_MODEL_f0927de5e29e4a5f91356acd57c998b1"
      }
     },
     "f0927de5e29e4a5f91356acd57c998b1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f56ae0f406344977acb852ba25f79a49": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_5d089e09a4a741e6a8a984292de328af",
       "placeholder": "​",
       "style": "IPY_MODEL_7978f57e98de4d49bdd6b6f61f75f1af",
       "value": "100% 17500/17500 [00:46&lt;00:00, 375.59it/s]"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}