Adversarial Drug Reaction-ADE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: Quadro K2000 (CNMeM is disabled, cuDNN 5005)\n",
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "%matplotlib notebook\n",
    "from matplotlib import pyplot as plt\n",
    "import pandas\n",
    "import re\n",
    "import nltk\n",
    "import numpy as np\n",
    "from numpy import *\n",
    "import csv\n",
    "import theano.tensor as T\n",
    "import os.path\n",
    "from nltk.collocations import *\n",
    "from optparse import OptionParser\n",
    "from collections import Counter\n",
    "from copy import copy\n",
    "import cPickle\n",
    "import csv\n",
    "import warnings\n",
    "\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.metrics import roc_auc_score, f1_score\n",
    "\n",
    "from keras.layers.convolutional import MaxPooling1D, Convolution1D\n",
    "from keras.layers.recurrent import LSTM, GRU\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.models import Sequential, Graph\n",
    "from keras.engine.training import slice_X\n",
    "from keras.layers.core import Layer, Dense, Dropout, Activation,\\\n",
    "    Reshape, Flatten, Lambda\n",
    "from keras.regularizers import Regularizer\n",
    "from keras.optimizers import SGD\n",
    "from keras.constraints import maxnorm\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.utils import np_utils\n",
    "from keras.optimizers import Adadelta\n",
    "from keras.callbacks import Callback\n",
    "\n",
    "\n",
    "from IPython.utils.io import CapturedIO\n",
    "from gensim.models import Word2Vec\n",
    "from pkg_resources import resource_filename\n",
    "import utils\n",
    "import datasets\n",
    "from unidecode import unidecode\n",
    "\n",
    "# Yoon Kim's tokenization\n",
    "def my_process(string):\n",
    "    \"\"\"\n",
    "    Tokenization/string cleaning for all datasets except for SST.\n",
    "    Every dataset is lower cased except for TREC\n",
    "    \"\"\"\n",
    "    string = re.sub(r\"[^\\w(),|!?\\'\\`\\:\\-\\.;\\$%#]\", \" \", string)\n",
    "    string = re.sub(r\"\\'s\", \" is\", string)\n",
    "    string = re.sub(r\"\\'ve\", \" have\", string)\n",
    "    string = re.sub(r\"n\\'t\", \" not\", string)\n",
    "    string = re.sub(r\"\\'re\", \" are\", string)\n",
    "    string = re.sub(r\"\\'d\", \" would\", string)\n",
    "    string = re.sub(r\"\\'ll\", \" will\", string)\n",
    "    string = re.sub(r\"(?<=\\w)\\.\\.\\.\", \" ... \", string)\n",
    "    string = re.sub(r\"(?<=\\w)\\.\", \" . \", string)\n",
    "    string = re.sub(r\"(?<=\\w),\", \" , \", string)\n",
    "    string = re.sub(r\"(?<=\\w);\", \" ; \", string)\n",
    "    string = re.sub(r\"(?<=\\w)!\", \" ! \", string)\n",
    "    string = re.sub(r\"\\((?=\\w)\", \" ( \", string)\n",
    "    string = re.sub(r\"(?<=\\w)\\)\", \" ) \", string)\n",
    "    string = re.sub(r\"(?<=\\w)\\?\", \" ? \", string)\n",
    "    string = re.sub(r\"\\s{2,}\", \" \", string)\n",
    "    return string.strip()\n",
    "\n",
    "def mixed_score(y_true, y_probs, th):\n",
    "    y_probs = asarray(y_probs)\n",
    "    return {\n",
    "        \"precision\": y_true[y_probs >= th].sum()*1./(y_probs >= th).sum(),\n",
    "        \"recall\": y_true[y_probs >= th].sum()*1./y_true.sum(),\n",
    "        \"f1\": f1_score(y_true, (y_probs >= th)*1),\n",
    "        \"auc\": roc_auc_score(y_true, y_probs)\n",
    "    }\n",
    "\n",
    "# This function chooses the best threshold based on f1 of validation.\n",
    "def seq_score(model, X, y):\n",
    "    val_split = model.last_fit_params.get('validation_split', 0.)\n",
    "    split_at = int(model.last_fit_X.shape[0] * (1. - val_split))\n",
    "    X_val, y_val = model.last_fit_X[split_at:], model.last_fit_y[split_at:]\n",
    "    val_probs = model.predict(X_val).flatten()\n",
    "    thresholds = sorted(unique(val_probs))\n",
    "    max_f1, best_threshold = 0, 0\n",
    "    for threshold in thresholds:\n",
    "        f1 = f1_score(y_val, (val_probs >= threshold)*1)\n",
    "        if f1 > max_f1:\n",
    "            max_f1 = f1\n",
    "            best_threshold = threshold\n",
    "    return mixed_score(y, model.predict(X).flatten(), best_threshold)\n",
    "    \n",
    "# Same as seq_f1 but for graph model\n",
    "def graph_score(model, data):\n",
    "    val_split = model.last_fit_params.get('validation_split', 0.)\n",
    "    split_at = int(model.last_fit_data['output'].shape[0] * (1. - val_split))\n",
    "    data_val = {k: slice_X(v, split_at) for k, v in model.last_fit_data.items()}\n",
    "    val_probs = model.predict(data_val)['output'].flatten()\n",
    "    thresholds = sorted(unique(val_probs))\n",
    "    max_f1, best_threshold = 0, 0\n",
    "    for threshold in thresholds:\n",
    "        f1 = f1_score(data_val['output'], (val_probs >= threshold)*1)\n",
    "        if f1 > max_f1:\n",
    "            max_f1 = f1\n",
    "            best_threshold = threshold\n",
    "    return mixed_score(data['output'], \n",
    "                       model.predict(data)['output'].flatten(), \n",
    "                       best_threshold)\n",
    "\n",
    "def seq_auc(model, X, y):\n",
    "    preds = model.predict(X).flatten()\n",
    "    return roc_auc_score(y, preds)\n",
    "    \n",
    "def graph_auc(model, data):\n",
    "    preds = model.predict(data)['output'].flatten()\n",
    "    return roc_auc_score(data['output'], preds)\n",
    "\n",
    "seq_eval_f = seq_score\n",
    "graph_eval_f = graph_score\n",
    "results = pandas.DataFrame()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load ADE data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_path = \"ADE-Corpus-V2/\"\n",
    "texts, labels = [], []\n",
    "with open(os.path.join(data_path, 'DRUG-AE.rel')) as f:\n",
    "    for line in f:\n",
    "        pubmed_id, text = line.strip().split('|')[:2]\n",
    "        texts.append(unidecode(text.decode('utf-8')))\n",
    "        labels.append(1)\n",
    "\n",
    "with open(os.path.join(data_path, 'ADE-NEG.txt')) as f:\n",
    "    for line in f:\n",
    "        pubmed_id, neg = line.strip().split(' ')[:2]\n",
    "        text = ' '.join(line.strip().split(' ')[2:])\n",
    "        texts.append(unidecode(text.decode('utf-8')))\n",
    "        labels.append(0)\n",
    "        \n",
    "np.random.seed(0)\n",
    "# Shuffle the data as Keras won't shuffle validation data.\n",
    "# This can make the training ends early as we are using\n",
    "# early stop for regularisation.\n",
    "idx = np.random.permutation(len(labels))        \n",
    "labels = asarray(labels)[idx]\n",
    "texts = asarray(texts, dtype='str')[idx]\n",
    "skf = list(StratifiedKFold(labels, n_folds=10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The baselines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from zhang_adr.concept_matching import run_cm\n",
    "from zhang_adr.maxent_tfidf import run_tfidf\n",
    "from zhang_adr.maxent_nblcr import run_nblcr\n",
    "from zhang_adr.maxent_we import run_we\n",
    "from zhang_adr.tweetnlp import tweet_tagger\n",
    "from zhang_adr.preprocess import clean_tweet\n",
    "\n",
    "tokens, tags = tweet_tagger.runtagger_parse(texts)\n",
    "zhang_clean_texts = []\n",
    "for token, tag in zip(tokens, tags):\n",
    "    zhang_clean_texts.append(clean_tweet(token, tag))\n",
    "zhang_clean_texts = asarray(zhang_clean_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "### Fold 1:\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "clf = LogisticRegression(class_weight=\"auto\")\n",
    "bm_results = []\n",
    "with warnings.catch_warnings():\n",
    "    warnings.simplefilter(\"ignore\")\n",
    "    for i, (train_idx, test_idx) in enumerate(skf, 1):\n",
    "        print \"### Fold {}:\".format(i)\n",
    "        train, test = [], []\n",
    "        for train_id in train_idx:\n",
    "            train.append({\"id\": None, \"label\": labels[train_id], \"text\": zhang_clean_texts[train_id]})\n",
    "        train = pandas.DataFrame(train)\n",
    "        for test_id in test_idx:\n",
    "            test.append({\"id\": None, \"label\": labels[test_id], \"text\": zhang_clean_texts[test_id]})\n",
    "        test = pandas.DataFrame(test)\n",
    "\n",
    "        result = {}\n",
    "\n",
    "        y_pred_cm = run_cm(train, test, resource_filename('zhang_adr', 'data/ADR-lexicon.txt'))\n",
    "        result = mixed_score(test['label'].values, y_pred_cm, 0.5)\n",
    "        result['model'] = 'CM'\n",
    "        results = pandas.concat([results, pandas.DataFrame([result])])\n",
    "\n",
    "        _, y_prob_tfidf = run_tfidf(train, test, grams='123', n_dim=40000, clf=clf)\n",
    "        result = mixed_score(test['label'].values, asarray(y_prob_tfidf[:, 1]), 0.5)\n",
    "        result['model'] = 'ME-TFIDF'\n",
    "        results = pandas.concat([results, pandas.DataFrame([result])])\n",
    "\n",
    "        _, y_prob_nblcr = run_nblcr(train, test, 'nblcr', grams='123', clf=clf)\n",
    "        result = mixed_score(test['label'].values, y_prob_nblcr[:, 1], 0.5)\n",
    "        result['model'] = 'ME-NBLCR'\n",
    "        results = pandas.concat([results, pandas.DataFrame([result])])\n",
    "\n",
    "        _, y_prob_we = run_we(train, test, resource_filename('zhang_adr', 'data/w2v_150.txt'), 150, clf=clf)\n",
    "        result = mixed_score(test['label'].values,  y_prob_we[:, 1], 0.5)\n",
    "        result['model'] = 'ME-WE'\n",
    "        results = pandas.concat([results, pandas.DataFrame([result])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "results.groupby(\"model\").mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Our methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the embedding and features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "w2v = Word2Vec.load_word2vec_format(\n",
    "    '/home/trung/data/embeddings/glovec/tmp',\n",
    "    binary=False\n",
    ")\n",
    "\n",
    "dim = w2v.layer1_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from zhang_adr.TextUtility import TextUtility\n",
    "\n",
    "MOST_FREQUENT_WORDS = 20000\n",
    "USE_CACHE = False\n",
    "INCLUDE_UNKNOWN_WORDS = False\n",
    "\n",
    "docs = [[w for w in TextUtility.text_to_wordlist(text)\\\n",
    "         if INCLUDE_UNKNOWN_WORDS or w in w2v.index2word]\\\n",
    "         for text in zhang_clean_texts]\n",
    "all_words = Counter([w for doc in docs for w in doc])\n",
    "top_words = sorted(all_words.items(), key=lambda t: t[1], reverse=True)\n",
    "top_words = top_words[:MOST_FREQUENT_WORDS]\n",
    "V = {w:i for i, (w, freq) in enumerate(top_words)}\n",
    "X = utils.vectorize(docs, V)\n",
    "\n",
    "# initialize embedding matrix\n",
    "my_embeddings = np.random.normal(-.25, .25, size=(X.max() + 1, dim))\n",
    "for w in V:\n",
    "    if w in w2v:\n",
    "        my_embeddings[V[w]] = w2v[w]\n",
    "        \n",
    "# set embedding of padded character as 0s.\n",
    "my_embeddings[len(V) + 1] = np.zeros((dim, ))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/trung/anaconda2/lib/python2.7/site-packages/keras/models.py:540: UserWarning: \"class_mode\" argument is deprecated, please remove it.\n",
      "  warnings.warn('\"class_mode\" argument is deprecated, '\n"
     ]
    }
   ],
   "source": [
    "# sequential model\n",
    "early_stopper = utils.MyEarlyStopping(monitor='val_loss', patience=5, verbose=0)\n",
    "scores = utils.seq_cross_validate(\n",
    "    utils.mk_yk_model_f(X.shape[1], my_embeddings, n_filters=300),\n",
    "    X, labels, \n",
    "    skf, eval_f=seq_eval_f,\n",
    "    fit_params={\n",
    "        \"callbacks\": [early_stopper],\n",
    "        \"validation_split\": .1,\n",
    "        \"batch_size\": 50\n",
    "    }, \n",
    "    verbose=1)\n",
    "df = pandas.DataFrame(scores)\n",
    "model_name = \"CNN\"\n",
    "df[\"model\"] = model_name\n",
    "results = pandas.concat([results[results[\"model\"] != model_name], df])\n",
    "df.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'model'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-13-abf64790a5d7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mresults\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"model\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m/home/trung/anaconda2/lib/python2.7/site-packages/pandas/core/generic.pyc\u001b[0m in \u001b[0;36mgroupby\u001b[1;34m(self, by, axis, level, as_index, sort, group_keys, squeeze, **kwargs)\u001b[0m\n\u001b[0;32m   3776\u001b[0m         return groupby(self, by=by, axis=axis, level=level, as_index=as_index,\n\u001b[0;32m   3777\u001b[0m                        \u001b[0msort\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msort\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgroup_keys\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mgroup_keys\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msqueeze\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3778\u001b[1;33m                        **kwargs)\n\u001b[0m\u001b[0;32m   3779\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3780\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0masfreq\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfreq\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhow\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnormalize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/trung/anaconda2/lib/python2.7/site-packages/pandas/core/groupby.pyc\u001b[0m in \u001b[0;36mgroupby\u001b[1;34m(obj, by, **kwds)\u001b[0m\n\u001b[0;32m   1425\u001b[0m         \u001b[1;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'invalid type: %s'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1426\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1427\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mklass\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1428\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1429\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/trung/anaconda2/lib/python2.7/site-packages/pandas/core/groupby.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, obj, keys, axis, level, grouper, exclusions, selection, as_index, sort, group_keys, squeeze, **kwargs)\u001b[0m\n\u001b[0;32m    352\u001b[0m                                                     \u001b[0mlevel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    353\u001b[0m                                                     \u001b[0msort\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msort\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 354\u001b[1;33m                                                     mutated=self.mutated)\n\u001b[0m\u001b[0;32m    355\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    356\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mobj\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/trung/anaconda2/lib/python2.7/site-packages/pandas/core/groupby.pyc\u001b[0m in \u001b[0;36m_get_grouper\u001b[1;34m(obj, key, axis, level, sort, mutated)\u001b[0m\n\u001b[0;32m   2381\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2382\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mis_in_axis\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgpr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# df.groupby('name')\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2383\u001b[1;33m             \u001b[0min_axis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgpr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgpr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mgpr\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2384\u001b[0m             \u001b[0mexclusions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2385\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/trung/anaconda2/lib/python2.7/site-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   1995\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1996\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1997\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_column\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1998\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1999\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_getitem_column\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/trung/anaconda2/lib/python2.7/site-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m_getitem_column\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   2002\u001b[0m         \u001b[1;31m# get column\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2003\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_unique\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2004\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_item_cache\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2005\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2006\u001b[0m         \u001b[1;31m# duplicate columns & possible reduce dimensionality\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/trung/anaconda2/lib/python2.7/site-packages/pandas/core/generic.pyc\u001b[0m in \u001b[0;36m_get_item_cache\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m   1348\u001b[0m         \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcache\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1349\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1350\u001b[1;33m             \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1351\u001b[0m             \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_box_item_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1352\u001b[0m             \u001b[0mcache\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mres\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/trung/anaconda2/lib/python2.7/site-packages/pandas/core/internals.pyc\u001b[0m in \u001b[0;36mget\u001b[1;34m(self, item, fastpath)\u001b[0m\n\u001b[0;32m   3288\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3289\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misnull\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3290\u001b[1;33m                 \u001b[0mloc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3291\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3292\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0misnull\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/trung/anaconda2/lib/python2.7/site-packages/pandas/indexes/base.pyc\u001b[0m in \u001b[0;36mget_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   1945\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1946\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1947\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1948\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1949\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas/index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_loc (pandas/index.c:4154)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_loc (pandas/index.c:4018)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/hashtable.pyx\u001b[0m in \u001b[0;36mpandas.hashtable.PyObjectHashTable.get_item (pandas/hashtable.c:12368)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/hashtable.pyx\u001b[0m in \u001b[0;36mpandas.hashtable.PyObjectHashTable.get_item (pandas/hashtable.c:12322)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'model'"
     ]
    }
   ],
   "source": [
    "results.groupby(\"model\").mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CNN + [Doc features: has_adrs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# def mk_mixture_model(max_len, embedding, doc_feature_size):\n",
    "    \n",
    "#     def mixture_model():\n",
    "#         graph = Graph()\n",
    "#         graph.add_input(name='tokens', input_shape=(max_len, ), dtype='int')\n",
    "#         utils.add_yk_node(graph, 'tokens', max_len, embedding)\n",
    "#         graph.add_input(name='doc_features', input_shape=(doc_feature_size, ), dtype='int')\n",
    "#         graph.add_node(Dense(1, activation='sigmoid', W_constraint=maxnorm(9)), \n",
    "#                        name='perceptron', inputs=['yk', 'doc_features'])\n",
    "#         graph.add_output('output', input='perceptron')\n",
    "#         graph.compile(optimizer='adadelta', loss={'output': 'binary_crossentropy'})\n",
    "#         return graph\n",
    "    \n",
    "#     return mixture_model\n",
    "\n",
    "# class MyReport(Callback):\n",
    "    \n",
    "#     def on_epoch_end(self, logs={}):\n",
    "#         pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# # Create a feature that tells whether a tweet contains a phrase in ADR lexicon\n",
    "# adr_lexicon = datasets.load_ADR_lexicon(\"/home/trung/data/lexicons/ADR/ADR_lexicon.tsv\")\n",
    "# has_adrs = []\n",
    "# for text in zhang_clean_texts:\n",
    "#     has_adr = 0\n",
    "#     for p in adr_lexicon['phrase']:\n",
    "#         if p in text:\n",
    "#             has_adr = 1\n",
    "#     has_adrs.append(has_adr)                \n",
    "# has_adrs = asarray(has_adrs).reshape(-1, 1)\n",
    "\n",
    "# # graph model\n",
    "# early_stopper = utils.MyEarlyStopping(monitor='val_loss', patience=5, verbose=1)\n",
    "# scores = utils.graph_cross_validate(\n",
    "#     mk_mixture_model(X.shape[1], my_embeddings, has_adrs.shape[1]),\n",
    "#     {'tokens': X, 'doc_features': has_adrs, 'output': labels}, \n",
    "#     skf, \n",
    "#     eval_f=graph_eval_f,\n",
    "#     fit_params={\n",
    "#         \"callbacks\": [early_stopper],\n",
    "#         \"validation_split\": .1,\n",
    "#         \"batch_size\": 50\n",
    "#     })\n",
    "# results[\"my-cnn-dynamic-embedding-has_adr-inc_unkwn_{}\".format(INCLUDE_UNKNOWN_WORDS)] = scores\n",
    "# mean(scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 7150/19047 [==========>...................] - ETA: 105s - loss: 0.1434"
     ]
    }
   ],
   "source": [
    "# sequential model\n",
    "early_stopper = utils.MyEarlyStopping(monitor='val_loss', patience=5, verbose=0)\n",
    "scores = utils.seq_cross_validate(\n",
    "    utils.mk_gru_model_f(X.shape[1], my_embeddings),\n",
    "    X[:, ::-1], labels,\n",
    "    skf, eval_f=seq_eval_f,\n",
    "    verbose=0,\n",
    "    fit_params={\n",
    "        \"callbacks\": [early_stopper],\n",
    "        \"validation_split\": .1,\n",
    "        \"batch_size\": 50\n",
    "    })\n",
    "df = pandas.DataFrame(scores)\n",
    "model_name = \"GRU\"\n",
    "df[\"model\"] = model_name\n",
    "results = pandas.concat([results[results[\"model\"] != model_name], df])\n",
    "df.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CRNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO (theano.gof.compilelock): Refreshing lock /home/trung/.theano/compiledir_Linux-2.6-el6.x86_64-x86_64-with-centos-6.6-Final-x86_64-2.7.12-64/lock_dir/lock\n",
      "INFO (theano.gof.compilelock): Refreshing lock /home/trung/.theano/compiledir_Linux-2.6-el6.x86_64-x86_64-with-centos-6.6-Final-x86_64-2.7.12-64/lock_dir/lock\n"
     ]
    }
   ],
   "source": [
    "# sequential model\n",
    "early_stopper = utils.MyEarlyStopping(monitor='val_loss', patience=5, verbose=0)\n",
    "scores = utils.seq_cross_validate(\n",
    "    utils.mk_cgru_model_f(X.shape[1], my_embeddings, nb_filter=300, rnn_output=300),\n",
    "    X[:, ::-1], labels,\n",
    "    skf, eval_f=seq_eval_f,\n",
    "    verbose=0,\n",
    "    fit_params={\n",
    "        \"callbacks\": [early_stopper],\n",
    "        \"validation_split\": .1,\n",
    "        \"batch_size\": 50\n",
    "    })\n",
    "df = pandas.DataFrame(scores)\n",
    "model_name = \"CRNN\"\n",
    "df[\"model\"] = model_name\n",
    "results = pandas.concat([results[results[\"model\"] != model_name], df])\n",
    "df.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 19046 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19046/19046 [==============================] - 89s - loss: 0.4313 - val_loss: 0.3082\n",
      "Epoch 2/10\n",
      "19046/19046 [==============================] - 89s - loss: 0.2576 - val_loss: 0.2248\n",
      "Epoch 3/10\n",
      "19046/19046 [==============================] - 89s - loss: 0.1896 - val_loss: 0.2000\n",
      "Epoch 4/10\n",
      "19046/19046 [==============================] - 89s - loss: 0.1418 - val_loss: 0.1980\n",
      "Epoch 5/10\n",
      "19046/19046 [==============================] - 89s - loss: 0.1020 - val_loss: 0.2058\n",
      "Epoch 6/10\n",
      "19046/19046 [==============================] - 89s - loss: 0.0687 - val_loss: 0.2453\n",
      "Epoch 7/10\n",
      "19046/19046 [==============================] - 89s - loss: 0.0427 - val_loss: 0.2689\n",
      "Epoch 8/10\n",
      "19046/19046 [==============================] - 89s - loss: 0.0239 - val_loss: 0.2988\n",
      "Epoch 9/10\n",
      "19046/19046 [==============================] - 89s - loss: 0.0130 - val_loss: 0.4708\n",
      "Train on 19047 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.4088 - val_loss: 0.2770\n",
      "Epoch 2/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.2491 - val_loss: 0.2162\n",
      "Epoch 3/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.1834 - val_loss: 0.2001\n",
      "Epoch 4/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.1372 - val_loss: 0.2797\n",
      "Epoch 5/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0995 - val_loss: 0.2218\n",
      "Epoch 6/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0681 - val_loss: 0.2397\n",
      "Epoch 7/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0439 - val_loss: 0.2501\n",
      "Epoch 8/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0255 - val_loss: 0.3381\n",
      "Train on 19047 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.4034 - val_loss: 0.2513\n",
      "Epoch 2/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.2401 - val_loss: 0.2060\n",
      "Epoch 3/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.1752 - val_loss: 0.2082\n",
      "Epoch 4/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.1270 - val_loss: 0.1876\n",
      "Epoch 5/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0923 - val_loss: 0.1914\n",
      "Epoch 6/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0633 - val_loss: 0.2778\n",
      "Epoch 7/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0400 - val_loss: 0.2556\n",
      "Epoch 8/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0228 - val_loss: 0.3188\n",
      "Epoch 9/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0130 - val_loss: 0.3288\n",
      "Train on 19047 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.4206 - val_loss: 0.2645\n",
      "Epoch 2/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.2502 - val_loss: 0.2239\n",
      "Epoch 3/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.1872 - val_loss: 0.2239\n",
      "Epoch 4/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.1394 - val_loss: 0.1986\n",
      "Epoch 5/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.1012 - val_loss: 0.1870\n",
      "Epoch 6/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0680 - val_loss: 0.2018\n",
      "Epoch 7/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0423 - val_loss: 0.2392\n",
      "Epoch 8/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0250 - val_loss: 0.2822\n",
      "Epoch 9/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0140 - val_loss: 0.3150\n",
      "Epoch 10/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0086 - val_loss: 0.3838\n",
      "Train on 19047 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.4691 - val_loss: 0.2709\n",
      "Epoch 2/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.2541 - val_loss: 0.2743\n",
      "Epoch 3/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.1894 - val_loss: 0.2018\n",
      "Epoch 4/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.1387 - val_loss: 0.2059\n",
      "Epoch 5/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.1023 - val_loss: 0.2096\n",
      "Epoch 6/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0708 - val_loss: 0.2334\n",
      "Epoch 7/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0480 - val_loss: 0.2709\n",
      "Epoch 8/10\n",
      "19047/19047 [==============================] - 89s - loss: 0.0334 - val_loss: 0.2686\n",
      "Train on 19048 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.4378 - val_loss: 0.3023\n",
      "Epoch 2/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.2499 - val_loss: 0.2452\n",
      "Epoch 3/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.1842 - val_loss: 0.2343\n",
      "Epoch 4/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.1370 - val_loss: 0.2215\n",
      "Epoch 5/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0987 - val_loss: 0.2166\n",
      "Epoch 6/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0677 - val_loss: 0.2458\n",
      "Epoch 7/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0419 - val_loss: 0.2850\n",
      "Epoch 8/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0246 - val_loss: 0.3374\n",
      "Epoch 9/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0141 - val_loss: 0.3698\n",
      "Epoch 10/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0086 - val_loss: 0.4786\n",
      "Train on 19048 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.4514 - val_loss: 0.2945\n",
      "Epoch 2/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.2659 - val_loss: 0.2374\n",
      "Epoch 3/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.1903 - val_loss: 0.2103\n",
      "Epoch 4/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.1455 - val_loss: 0.2039\n",
      "Epoch 5/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.1027 - val_loss: 0.4342\n",
      "Epoch 6/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0702 - val_loss: 0.2220\n",
      "Epoch 7/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0456 - val_loss: 0.2523\n",
      "Epoch 8/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0277 - val_loss: 0.2982\n",
      "Epoch 9/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0169 - val_loss: 0.3085\n",
      "Train on 19048 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19048/19048 [==============================] - 89s - loss: 8.3720 - val_loss: 11.1604\n",
      "Epoch 2/10\n",
      "19048/19048 [==============================] - 90s - loss: 11.3358 - val_loss: 11.1604\n",
      "Epoch 3/10\n",
      "19048/19048 [==============================] - 90s - loss: 11.3358 - val_loss: 11.1604\n",
      "Epoch 4/10\n",
      "19048/19048 [==============================] - 90s - loss: 11.3349 - val_loss: 11.1604\n",
      "Epoch 5/10\n",
      "19048/19048 [==============================] - 90s - loss: 11.3358 - val_loss: 11.1604\n",
      "Epoch 6/10\n",
      "19048/19048 [==============================] - 90s - loss: 11.3358 - val_loss: 11.1604\n",
      "Train on 19048 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.4129 - val_loss: 0.2716\n",
      "Epoch 2/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.2493 - val_loss: 0.2261\n",
      "Epoch 3/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.1855 - val_loss: 0.1950\n",
      "Epoch 4/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.1356 - val_loss: 0.2486\n",
      "Epoch 5/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0980 - val_loss: 0.1802\n",
      "Epoch 6/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0675 - val_loss: 0.2369\n",
      "Epoch 7/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0422 - val_loss: 0.2423\n",
      "Epoch 8/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0244 - val_loss: 0.3035\n",
      "Epoch 9/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0177 - val_loss: 0.2894\n",
      "Epoch 10/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0093 - val_loss: 0.3419\n",
      "Train on 19048 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.4139 - val_loss: 0.2798\n",
      "Epoch 2/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.2515 - val_loss: 0.2841\n",
      "Epoch 3/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.1864 - val_loss: 0.2106\n",
      "Epoch 4/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.1385 - val_loss: 0.2063\n",
      "Epoch 5/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.1006 - val_loss: 0.2231\n",
      "Epoch 6/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0724 - val_loss: 0.2286\n",
      "Epoch 7/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0485 - val_loss: 0.2452\n",
      "Epoch 8/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0294 - val_loss: 0.3244\n",
      "Epoch 9/10\n",
      "19048/19048 [==============================] - 89s - loss: 0.0194 - val_loss: 0.3245\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'result' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-14-410f77863ba8>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[0mmodel_name\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"RCNN\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"model\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel_name\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 15\u001b[1;33m \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpandas\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mresults\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"model\"\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mmodel_name\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     16\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'result' is not defined"
     ]
    }
   ],
   "source": [
    "early_stopper = utils.MyEarlyStopping(monitor='val_loss', patience=5, verbose=0)\n",
    "scores = utils.seq_cross_validate(\n",
    "    utils.mk_rcnn_model_f(X.shape[1], my_embeddings, rnn_output=300, nb_filter=300, filter_length=5),\n",
    "    X, labels,\n",
    "    skf, eval_f=seq_eval_f,\n",
    "    verbose=0,\n",
    "    fit_params={\n",
    "        \"callbacks\": [early_stopper],\n",
    "        \"validation_split\": .1,\n",
    "        \"batch_size\": 50\n",
    "    })\n",
    "df = pandas.DataFrame(scores)\n",
    "model_name = \"RCNN\"\n",
    "df[\"model\"] = model_name\n",
    "results = pandas.concat([results[results[\"model\"] != model_name], df])\n",
    "df.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "auc          0.922562\n",
       "f1           0.829314\n",
       "precision    0.811351\n",
       "recall       0.887119\n",
       "dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = pandas.concat([results[results[\"model\"] != model_name], df])\n",
    "df.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CNNA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'utils' from 'utils.py'>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reload(utils)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 19046 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19046/19046 [==============================] - 48s - loss: 0.5645 - val_loss: 0.3962\n",
      "Epoch 2/10\n",
      "19046/19046 [==============================] - 48s - loss: 0.3827 - val_loss: 0.3192\n",
      "Epoch 3/10\n",
      "19046/19046 [==============================] - 48s - loss: 0.3132 - val_loss: 0.2929\n",
      "Epoch 4/10\n",
      "19046/19046 [==============================] - 48s - loss: 0.2776 - val_loss: 0.2854\n",
      "Epoch 5/10\n",
      "19046/19046 [==============================] - 48s - loss: 0.2548 - val_loss: 0.2510\n",
      "Epoch 6/10\n",
      "19046/19046 [==============================] - 48s - loss: 0.2344 - val_loss: 0.2503\n",
      "Epoch 7/10\n",
      "19046/19046 [==============================] - 48s - loss: 0.2197 - val_loss: 0.2433\n",
      "Epoch 8/10\n",
      "19046/19046 [==============================] - 48s - loss: 0.2075 - val_loss: 0.2402\n",
      "Epoch 9/10\n",
      "19046/19046 [==============================] - 48s - loss: 0.1960 - val_loss: 0.2404\n",
      "Epoch 10/10\n",
      "19046/19046 [==============================] - 48s - loss: 0.1871 - val_loss: 0.2418\n",
      "Train on 19047 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.5694 - val_loss: 0.3966\n",
      "Epoch 2/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.3811 - val_loss: 0.3080\n",
      "Epoch 3/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.3206 - val_loss: 0.3152\n",
      "Epoch 4/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2807 - val_loss: 0.2713\n",
      "Epoch 5/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2568 - val_loss: 0.2535\n",
      "Epoch 6/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2384 - val_loss: 0.2470\n",
      "Epoch 7/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2236 - val_loss: 0.2436\n",
      "Epoch 8/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2107 - val_loss: 0.2460\n",
      "Epoch 9/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.1994 - val_loss: 0.2397\n",
      "Epoch 10/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.1881 - val_loss: 0.2459\n",
      "Train on 19047 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.5702 - val_loss: 0.4029\n",
      "Epoch 2/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.3785 - val_loss: 0.3119\n",
      "Epoch 3/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.3146 - val_loss: 0.2790\n",
      "Epoch 4/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2772 - val_loss: 0.2682\n",
      "Epoch 5/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2538 - val_loss: 0.2618\n",
      "Epoch 6/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2357 - val_loss: 0.2518\n",
      "Epoch 7/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2213 - val_loss: 0.2491\n",
      "Epoch 8/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2077 - val_loss: 0.2490\n",
      "Epoch 9/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.1978 - val_loss: 0.2477\n",
      "Epoch 10/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.1886 - val_loss: 0.2633\n",
      "Train on 19047 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.5745 - val_loss: 0.3976\n",
      "Epoch 2/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.3816 - val_loss: 0.3321\n",
      "Epoch 3/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.3164 - val_loss: 0.2780\n",
      "Epoch 4/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2811 - val_loss: 0.2617\n",
      "Epoch 5/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2529 - val_loss: 0.2517\n",
      "Epoch 6/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2343 - val_loss: 0.2512\n",
      "Epoch 7/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2200 - val_loss: 0.2501\n",
      "Epoch 8/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2066 - val_loss: 0.2455\n",
      "Epoch 9/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.1964 - val_loss: 0.2410\n",
      "Epoch 10/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.1865 - val_loss: 0.2445\n",
      "Train on 19047 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.5673 - val_loss: 0.4070\n",
      "Epoch 2/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.3795 - val_loss: 0.3036\n",
      "Epoch 3/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.3142 - val_loss: 0.2859\n",
      "Epoch 4/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2811 - val_loss: 0.2802\n",
      "Epoch 5/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2557 - val_loss: 0.2596\n",
      "Epoch 6/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2383 - val_loss: 0.2528\n",
      "Epoch 7/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2231 - val_loss: 0.2476\n",
      "Epoch 8/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.2097 - val_loss: 0.2450\n",
      "Epoch 9/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.1985 - val_loss: 0.2494\n",
      "Epoch 10/10\n",
      "19047/19047 [==============================] - 48s - loss: 0.1887 - val_loss: 0.2415\n",
      "Train on 19048 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.5704 - val_loss: 0.4161\n",
      "Epoch 2/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.3830 - val_loss: 0.3103\n",
      "Epoch 3/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.3175 - val_loss: 0.2784\n",
      "Epoch 4/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2789 - val_loss: 0.2641\n",
      "Epoch 5/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2540 - val_loss: 0.2647\n",
      "Epoch 6/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2348 - val_loss: 0.2496\n",
      "Epoch 7/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2179 - val_loss: 0.2590\n",
      "Epoch 8/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2069 - val_loss: 0.2470\n",
      "Epoch 9/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.1968 - val_loss: 0.2633\n",
      "Epoch 10/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.1858 - val_loss: 0.2448\n",
      "Train on 19048 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.5691 - val_loss: 0.4061\n",
      "Epoch 2/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.3794 - val_loss: 0.3062\n",
      "Epoch 3/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.3185 - val_loss: 0.2741\n",
      "Epoch 4/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2784 - val_loss: 0.2611\n",
      "Epoch 5/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2545 - val_loss: 0.2541\n",
      "Epoch 6/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2389 - val_loss: 0.2486\n",
      "Epoch 7/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2196 - val_loss: 0.2482\n",
      "Epoch 8/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2068 - val_loss: 0.2435\n",
      "Epoch 9/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.1983 - val_loss: 0.2644\n",
      "Epoch 10/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.1865 - val_loss: 0.2406\n",
      "Train on 19048 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.5688 - val_loss: 0.3987\n",
      "Epoch 2/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.3812 - val_loss: 0.3027\n",
      "Epoch 3/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.3173 - val_loss: 0.2771\n",
      "Epoch 4/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2777 - val_loss: 0.2758\n",
      "Epoch 5/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2548 - val_loss: 0.2547\n",
      "Epoch 6/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2357 - val_loss: 0.2460\n",
      "Epoch 7/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2216 - val_loss: 0.2413\n",
      "Epoch 8/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2092 - val_loss: 0.2429\n",
      "Epoch 9/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.1962 - val_loss: 0.2421\n",
      "Epoch 10/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.1842 - val_loss: 0.2372\n",
      "Train on 19048 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.5635 - val_loss: 0.3894\n",
      "Epoch 2/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.3763 - val_loss: 0.3046\n",
      "Epoch 3/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.3146 - val_loss: 0.2766\n",
      "Epoch 4/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2783 - val_loss: 0.2663\n",
      "Epoch 5/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2556 - val_loss: 0.2514\n",
      "Epoch 6/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2376 - val_loss: 0.2723\n",
      "Epoch 7/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2210 - val_loss: 0.2448\n",
      "Epoch 8/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2084 - val_loss: 0.2638\n",
      "Epoch 9/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.1989 - val_loss: 0.2459\n",
      "Epoch 10/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.1861 - val_loss: 0.2489\n",
      "Train on 19048 samples, validate on 2117 samples\n",
      "Epoch 1/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.5641 - val_loss: 0.4234\n",
      "Epoch 2/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.3801 - val_loss: 0.3296\n",
      "Epoch 3/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.3161 - val_loss: 0.3000\n",
      "Epoch 4/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2797 - val_loss: 0.2935\n",
      "Epoch 5/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2545 - val_loss: 0.2753\n",
      "Epoch 6/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2355 - val_loss: 0.2709\n",
      "Epoch 7/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2209 - val_loss: 0.2648\n",
      "Epoch 8/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.2084 - val_loss: 0.2659\n",
      "Epoch 9/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.1950 - val_loss: 0.2844\n",
      "Epoch 10/10\n",
      "19048/19048 [==============================] - 48s - loss: 0.1868 - val_loss: 0.2585\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "auc          0.950869\n",
       "f1           0.826228\n",
       "precision    0.815253\n",
       "recall       0.838439\n",
       "dtype: float64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "early_stopper = utils.MyEarlyStopping(monitor='val_loss', patience=5, verbose=0)\n",
    "scores = utils.graph_cross_validate(\n",
    "    utils.mk_attention_based_model_f(X.shape[1], my_embeddings, attention_l2=0.1),\n",
    "    {\"tokens\": X, \"output\": labels},\n",
    "    skf,\n",
    "    eval_f=graph_eval_f,\n",
    "    verbose=0,\n",
    "    fit_params={\n",
    "        \"callbacks\": [early_stopper],\n",
    "        \"validation_split\": .1,\n",
    "        \"batch_size\": 50\n",
    "    })\n",
    "df = pandas.DataFrame(scores)\n",
    "model_name = \"CNNA\"\n",
    "df[\"model\"] = model_name\n",
    "results = pandas.concat([results[results[\"model\"] != model_name], df])\n",
    "df.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>auc</th>\n",
       "      <th>f1</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CM</th>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CNN</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CNNA</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CRNN</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ME-NBLCR</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ME-TFIDF</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ME-WE</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RCNN</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          auc  f1  precision  recall\n",
       "model                               \n",
       "CM         11  11         11      11\n",
       "CNN        10  10         10      10\n",
       "CNNA       10  10         10      10\n",
       "CRNN       10  10         10      10\n",
       "ME-NBLCR   10  10         10      10\n",
       "ME-TFIDF   10  10         10      10\n",
       "ME-WE      10  10         10      10\n",
       "RCNN       10  10         10      10"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"model\").count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>auc</th>\n",
       "      <th>f1</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CM</th>\n",
       "      <td>0.528312</td>\n",
       "      <td>0.463361</td>\n",
       "      <td>0.302339</td>\n",
       "      <td>0.991338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CNN</th>\n",
       "      <td>0.970146</td>\n",
       "      <td>0.865692</td>\n",
       "      <td>0.845420</td>\n",
       "      <td>0.887404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CNNA</th>\n",
       "      <td>0.950869</td>\n",
       "      <td>0.826228</td>\n",
       "      <td>0.815253</td>\n",
       "      <td>0.838439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CRNN</th>\n",
       "      <td>0.956478</td>\n",
       "      <td>0.837676</td>\n",
       "      <td>0.816188</td>\n",
       "      <td>0.861015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ME-NBLCR</th>\n",
       "      <td>0.953812</td>\n",
       "      <td>0.843834</td>\n",
       "      <td>0.905228</td>\n",
       "      <td>0.790352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ME-TFIDF</th>\n",
       "      <td>0.938561</td>\n",
       "      <td>0.796262</td>\n",
       "      <td>0.742770</td>\n",
       "      <td>0.858230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ME-WE</th>\n",
       "      <td>0.760576</td>\n",
       "      <td>0.572706</td>\n",
       "      <td>0.482441</td>\n",
       "      <td>0.704592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RCNN</th>\n",
       "      <td>0.922562</td>\n",
       "      <td>0.829314</td>\n",
       "      <td>0.811351</td>\n",
       "      <td>0.887119</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               auc        f1  precision    recall\n",
       "model                                            \n",
       "CM        0.528312  0.463361   0.302339  0.991338\n",
       "CNN       0.970146  0.865692   0.845420  0.887404\n",
       "CNNA      0.950869  0.826228   0.815253  0.838439\n",
       "CRNN      0.956478  0.837676   0.816188  0.861015\n",
       "ME-NBLCR  0.953812  0.843834   0.905228  0.790352\n",
       "ME-TFIDF  0.938561  0.796262   0.742770  0.858230\n",
       "ME-WE     0.760576  0.572706   0.482441  0.704592\n",
       "RCNN      0.922562  0.829314   0.811351  0.887119"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"model\").mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results.to_csv(\"ADE-results.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}