01_FastText-Applier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "import gzip\n",
    "import json\n",
    "import re\n",
    "\n",
    "from urllib.parse import urlparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy\n",
    "from scipy import interpolate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.cluster\n",
    "import sklearn.feature_extraction \n",
    "import sklearn.feature_extraction.text\n",
    "import sklearn.metrics\n",
    "import sklearn.preprocessing\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.semi_supervised import LabelPropagation\n",
    "from sklearn.semi_supervised import LabelSpreading\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/clb617/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/nltk/twitter/__init__.py:20: UserWarning: The twython library has not been installed. Some functionality from the twitter package will not be available.\n",
      "  warnings.warn(\"The twython library has not been installed. \"\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download(\"stopwords\")\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer as VS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models.fasttext import FastText\n",
    "from gensim.models import KeyedVectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Tweet Count: 49147\n"
     ]
    }
   ],
   "source": [
    "tweet_id_map = {}\n",
    "with open(\"../data/2019b-testing.json\", \"r\") as in_file:\n",
    "    for line in in_file:\n",
    "        tweet_top = json.loads(line)\n",
    "        tweet = json.loads(tweet_top[\"allProperties\"][\"srcjson\"])\n",
    "        tweet_id_map[np.int64(tweet[\"id\"])] = tweet\n",
    "        \n",
    "with open(\"../data/2019b-training.json\", \"r\") as in_file:\n",
    "    for line in in_file:\n",
    "        tweet = json.loads(line)\n",
    "        tweet_id_map[np.int64(tweet[\"id\"])] = tweet\n",
    "\n",
    "print(\"Total Tweet Count:\", len(tweet_id_map))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'en': 48776,\n",
       " 'es': 60,\n",
       " 'tl': 153,\n",
       " 'it': 3,\n",
       " 'fr': 13,\n",
       " 'pt': 9,\n",
       " 'und': 95,\n",
       " 'hi': 10,\n",
       " 'nl': 3,\n",
       " 'ca': 2,\n",
       " 'eu': 1,\n",
       " 'in': 6,\n",
       " 'ro': 3,\n",
       " 'et': 1,\n",
       " 'de': 1,\n",
       " 'cs': 1,\n",
       " 'ht': 1,\n",
       " 'ja': 4,\n",
       " 'th': 1,\n",
       " 'cy': 1,\n",
       " 'lt': 1,\n",
       " 'pl': 1,\n",
       " 'tr': 1}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lang_count_map = {}\n",
    "for lang in [tweet[\"lang\"] for tweet in tweet_id_map.values()]:\n",
    "    lang_count_map[lang] = lang_count_map.get(lang, 0) + 1\n",
    "lang_count_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sources: 1011\n",
      "Mapped Sources: 97\n"
     ]
    }
   ],
   "source": [
    "tweet_sources_map = {}\n",
    "\n",
    "for tweet in tweet_id_map.values():\n",
    "    tweet_sources_map[tweet[\"source\"]] = tweet_sources_map.get(tweet[\"source\"], 0) + 1\n",
    "\n",
    "print(\"Sources:\", len(tweet_sources_map))\n",
    "tweet_sources_index = {x:i+1 for i, x in enumerate([x for x in tweet_sources_map.keys()if tweet_sources_map[x] > 10])}\n",
    "\n",
    "print(\"Mapped Sources:\", len(tweet_sources_index))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_priority_map = {}\n",
    "priority_df = pd.read_csv(\"tweet_to_priority.csv\", dtype={\"tweet_id\": np.int64})\n",
    "for row in priority_df.itertuples():\n",
    "    tweet_id = row.tweet_id\n",
    "    \n",
    "    tweet_priority_map[tweet_id] = {\n",
    "        \"score\": row.score_mean,\n",
    "        \"weight\": 1.0 - row.score_std\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_category_map = {}\n",
    "category_df = pd.read_csv(\"tweet_to_category.csv\")\n",
    "\n",
    "cat_update_map = {\n",
    "    \"ContinuingNews\": \"News\",\n",
    "    \"PastNews\": \"ContextualInformation\",\n",
    "    \"KnownAlready\": \"OriginalEvent\",\n",
    "    \"SignificantEventChange\": \"NewSubEvent\",\n",
    "}\n",
    "\n",
    "category_df[\"category\"] = category_df[\"category\"].apply(lambda x: cat_update_map.get(x, x))\n",
    "\n",
    "for category, tweets in category_df.groupby(\"category\"):\n",
    "    tweet_category_map[category] = list(tweets[\"tweet_id\"])\n",
    "    \n",
    "# Deleted in 2019\n",
    "del(tweet_category_map[\"Unknown\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Labels: 63346\n"
     ]
    }
   ],
   "source": [
    "print(\"Labels:\", sum([len(v) for v in tweet_category_map.values()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Category: Advice\n",
      "\tTweet Count: 1706 Retrieved Fraction: 1.0\n",
      "\t {'en': 1705, 'und': 1}\n",
      "Category: CleanUp\n",
      "\tTweet Count: 135 Retrieved Fraction: 1.0\n",
      "\t {'en': 135}\n",
      "Category: ContextualInformation\n",
      "\tTweet Count: 1610 Retrieved Fraction: 1.0\n",
      "\t {'en': 1561, 'hi': 1, 'es': 39, 'pt': 2, 'it': 2, 'fr': 2, 'und': 2, 'tl': 1}\n",
      "Category: Discussion\n",
      "\tTweet Count: 2921 Retrieved Fraction: 1.0\n",
      "\t {'en': 2878, 'und': 5, 'hi': 1, 'es': 27, 'pt': 3, 'it': 1, 'fr': 2, 'tl': 2, 'nl': 2}\n",
      "Category: Donations\n",
      "\tTweet Count: 894 Retrieved Fraction: 1.0\n",
      "\t {'en': 894}\n",
      "Category: EmergingThreats\n",
      "\tTweet Count: 1192 Retrieved Fraction: 1.0\n",
      "\t {'en': 1191, 'fr': 1}\n",
      "Category: Factoid\n",
      "\tTweet Count: 3525 Retrieved Fraction: 1.0\n",
      "\t {'en': 3511, 'tl': 8, 'und': 2, 'ca': 1, 'es': 3}\n",
      "Category: FirstPartyObservation\n",
      "\tTweet Count: 4216 Retrieved Fraction: 1.0\n",
      "\t {'en': 4211, 'hi': 2, 'und': 2, 'tl': 1}\n",
      "Category: GoodsServices\n",
      "\tTweet Count: 146 Retrieved Fraction: 1.0\n",
      "\t {'en': 146}\n",
      "Category: Hashtags\n",
      "\tTweet Count: 7159 Retrieved Fraction: 1.0\n",
      "\t {'en': 7043, 'es': 12, 'it': 1, 'fr': 2, 'und': 36, 'ca': 1, 'eu': 1, 'pt': 1, 'hi': 2, 'tl': 50, 'cy': 2, 'in': 4, 'pl': 1, 'lt': 1, 'ja': 1, 'tr': 1}\n",
      "Category: InformationWanted\n",
      "\tTweet Count: 259 Retrieved Fraction: 1.0\n",
      "\t {'en': 259}\n",
      "Category: Irrelevant\n",
      "\tTweet Count: 5294 Retrieved Fraction: 1.0\n",
      "\t {'en': 5223, 'und': 22, 'pt': 3, 'in': 1, 'es': 6, 'tl': 32, 'fr': 1, 'nl': 1, 'hi': 3, 'pl': 1, 'tr': 1}\n",
      "Category: Location\n",
      "\tTweet Count: 1506 Retrieved Fraction: 1.0\n",
      "\t {'en': 1506}\n",
      "Category: MovePeople\n",
      "\tTweet Count: 157 Retrieved Fraction: 1.0\n",
      "\t {'en': 157}\n",
      "Category: MultimediaShare\n",
      "\tTweet Count: 5696 Retrieved Fraction: 1.0\n",
      "\t {'en': 5642, 'tl': 3, 'und': 9, 'hi': 1, 'es': 33, 'it': 2, 'fr': 2, 'pt': 1, 'pl': 1, 'ja': 1, 'cy': 1}\n",
      "Category: NewSubEvent\n",
      "\tTweet Count: 591 Retrieved Fraction: 1.0\n",
      "\t {'en': 585, 'tl': 4, 'et': 2}\n",
      "Category: News\n",
      "\tTweet Count: 6837 Retrieved Fraction: 1.0\n",
      "\t {'en': 6819, 'ja': 2, 'tl': 7, 'und': 4, 'hi': 1, 'fr': 1, 'es': 1, 'pt': 1, 'cy': 1}\n",
      "Category: Official\n",
      "\tTweet Count: 705 Retrieved Fraction: 1.0\n",
      "\t {'en': 702, 'fr': 1, 'pt': 1, 'es': 1}\n",
      "Category: OriginalEvent\n",
      "\tTweet Count: 2111 Retrieved Fraction: 1.0\n",
      "\t {'en': 2054, 'ja': 2, 'hi': 1, 'es': 30, 'it': 1, 'pt': 1, 'tl': 9, 'fr': 2, 'und': 9, 'cy': 1, 'lt': 1}\n",
      "Category: SearchAndRescue\n",
      "\tTweet Count: 307 Retrieved Fraction: 1.0\n",
      "\t {'en': 307}\n",
      "Category: Sentiment\n",
      "\tTweet Count: 8806 Retrieved Fraction: 1.0\n",
      "\t {'en': 8694, 'tl': 49, 'und': 50, 'hi': 4, 'es': 1, 'eu': 1, 'in': 4, 'pl': 1, 'lt': 1, 'tr': 1}\n",
      "Category: ServiceAvailable\n",
      "\tTweet Count: 1285 Retrieved Fraction: 1.0\n",
      "\t {'en': 1281, 'hi': 2, 'ro': 2}\n",
      "Category: ThirdPartyObservation\n",
      "\tTweet Count: 4520 Retrieved Fraction: 1.0\n",
      "\t {'en': 4517, 'ca': 1, 'tl': 1, 'pt': 1}\n",
      "Category: Volunteer\n",
      "\tTweet Count: 145 Retrieved Fraction: 1.0\n",
      "\t {'en': 145}\n",
      "Category: Weather\n",
      "\tTweet Count: 1623 Retrieved Fraction: 1.0\n",
      "\t {'en': 1621, 'tl': 2}\n"
     ]
    }
   ],
   "source": [
    "for category, tweet_ids in tweet_category_map.items():\n",
    "    retrieved_count = sum([1 if np.int64(tid) in tweet_id_map else 0 in tweet_id_map for tid in tweet_ids])\n",
    "    print(\"Category:\", category)\n",
    "    print(\"\\tTweet Count:\", len(tweet_ids), \"Retrieved Fraction:\", retrieved_count/len(tweet_ids))\n",
    "    \n",
    "    lang_count_map = {}\n",
    "    for lang in [tweet_id_map[np.int64(tid)][\"lang\"] for tid in tweet_ids if np.int64(tid) in tweet_id_map]:\n",
    "        lang_count_map[lang] = lang_count_map.get(lang, 0) + 1\n",
    "    print(\"\\t\", str(lang_count_map))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Skip stop words, retweet signs, @ symbols, and URL headers\n",
    "stopList = [\"http\", \"https\", \"rt\", \"@\", \":\", \"t.co\", \"co\", \"amp\", \"&amp;\", \"...\", \"\\n\", \"\\r\"]\n",
    "stopList.extend(string.punctuation)\n",
    "# stopList.extend(stopwords.words(\"english\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def tokenizer_wrapper(text):\n",
    "#     return [t.lemma_ for t in nlp(text)]\n",
    "\n",
    "local_tokenizer = TweetTokenizer()\n",
    "def tokenizer_wrapper(text):\n",
    "    return local_tokenizer.tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate Additional Features\n",
    "sentiment_analyzer = VS()\n",
    "\n",
    "def count_twitter_objs(text_string):\n",
    "    \"\"\"\n",
    "    Accepts a text string and replaces:\n",
    "    1) urls with URLHERE\n",
    "    2) lots of whitespace with one instance\n",
    "    3) mentions with MENTIONHERE\n",
    "    4) hashtags with HASHTAGHERE\n",
    "\n",
    "    This allows us to get standardized counts of urls and mentions\n",
    "    Without caring about specific people mentioned.\n",
    "    \n",
    "    Returns counts of urls, mentions, and hashtags.\n",
    "    \"\"\"\n",
    "    space_pattern = '\\s+'\n",
    "    giant_url_regex = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'\n",
    "        '[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')\n",
    "    mention_regex = '@[\\w\\-]+'\n",
    "    hashtag_regex = '#[\\w\\-]+'\n",
    "    parsed_text = re.sub(space_pattern, ' ', text_string)\n",
    "    parsed_text = re.sub(giant_url_regex, 'URLHERE', parsed_text)\n",
    "    parsed_text = re.sub(mention_regex, 'MENTIONHERE', parsed_text)\n",
    "    parsed_text = re.sub(hashtag_regex, 'HASHTAGHERE', parsed_text)\n",
    "    return(parsed_text.count('URLHERE'),parsed_text.count('MENTIONHERE'),parsed_text.count('HASHTAGHERE'))\n",
    "\n",
    "## Taken from Davidson et al.\n",
    "def other_features(tweet):\n",
    "    \"\"\"This function takes a string and returns a list of features.\n",
    "    These include Sentiment scores, Text and Readability scores,\n",
    "    as well as Twitter specific features\"\"\"\n",
    "    tweet_text = tweet[\"text\"]\n",
    "    \n",
    "    ##SENTIMENT\n",
    "    sentiment = sentiment_analyzer.polarity_scores(tweet_text)\n",
    "    \n",
    "    words = local_tokenizer.tokenize(tweet_text) #Get text only\n",
    "    \n",
    "    num_chars = sum(len(w) for w in words) #num chars in words\n",
    "    num_chars_total = len(tweet_text)\n",
    "    num_terms = len(tweet_text.split())\n",
    "    num_words = len(words)\n",
    "    num_unique_terms = len(set([x.lower() for x in words]))\n",
    "    \n",
    "    caps_count = sum([1 if x.isupper() else 0 for x in tweet_text])\n",
    "    caps_ratio = caps_count / num_chars_total\n",
    "    \n",
    "    twitter_objs = count_twitter_objs(tweet_text) #Count #, @, and http://\n",
    "    num_media = 0\n",
    "    if \"entities\" in tweet and \"media\" in tweet[\"entities\"]:\n",
    "        num_media = len(tweet[\"entities\"][\"media\"])\n",
    "    retweet = 0\n",
    "    if \"rt\" in words or \"retweeted_status\" in tweet:\n",
    "        retweet = 1\n",
    "        \n",
    "    has_place = 1 if \"coordinates\" in tweet else 0\n",
    "        \n",
    "    author = tweet[\"user\"]\n",
    "    is_verified = 1 if (\"verified\" in author and author[\"verified\"]) else 0\n",
    "    log_followers = 0\n",
    "    if \"followers_count\" in author and author[\"followers_count\"] > 0:\n",
    "         log_followers = np.log(author[\"followers_count\"])\n",
    "    log_friends = 0\n",
    "    if \"friends_count\" in author and author[\"friends_count\"] > 0:\n",
    "         log_followers = np.log(author[\"friends_count\"])\n",
    "    \n",
    "    features = [num_chars, num_chars_total, num_terms, num_words,\n",
    "                num_unique_terms, sentiment['neg'], sentiment['pos'], \n",
    "                sentiment['neu'], sentiment['compound'],\n",
    "                twitter_objs[2], twitter_objs[1],\n",
    "                twitter_objs[0], retweet, num_media,\n",
    "                is_verified, \n",
    "#                 log_followers, log_friends,\n",
    "#                 has_place,\n",
    "                caps_ratio,\n",
    "               ]\n",
    "\n",
    "    return [round(x, 4) for x in features]\n",
    "\n",
    "other_features_names = [\"num_chars\", \"num_chars_total\", \\\n",
    "                        \"num_terms\", \"num_words\", \"num_unique_words\", \"vader neg\",\"vader pos\",\n",
    "                        \"vader neu\", \"vader compound\", \\\n",
    "                        \"num_hashtags\", \"num_mentions\", \n",
    "                        \"num_urls\", \"is_retweet\", \"num_media\",\n",
    "                        \"is_verified\", \n",
    "#                         \"log_followers\", \"log_friends\",\n",
    "#                         \"has_place\",\n",
    "                        \"caps_ratio\",\n",
    "                       ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = sklearn.feature_extraction.text.TfidfVectorizer(\n",
    "    tokenizer=tokenizer_wrapper,\n",
    "    ngram_range=(1, 1),\n",
    "    stop_words=stopList, #We do better when we keep stopwords\n",
    "    use_idf=True,\n",
    "    smooth_idf=False,\n",
    "    norm=None, #Applies l2 norm smoothing\n",
    "    decode_error='replace',\n",
    "    max_features=10000,\n",
    "    min_df=4,\n",
    "    max_df=0.501\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "category_to_label = {c:i for i, c in enumerate(tweet_category_map.keys()) if c != \"Irrelevant\"}\n",
    "category_to_label[\"Irrelevant\"] = -1\n",
    "\n",
    "tweet_id_to_category = {}\n",
    "for category, tweet_ids in tweet_category_map.items():\n",
    "    if ( len(tweet_ids) < 5 ):\n",
    "        print(\"Skipping category:\", category)\n",
    "        continue\n",
    "        \n",
    "    for tweet_id in tweet_ids:\n",
    "        tweet_id_to_category[np.int64(tweet_id)] = category_to_label[category]\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tweet_pairs = [(tweet, tweet_id_to_category[tid]) \n",
    "#                for tid, tweet in tweet_id_map.items() if tid in tweet_id_to_category]\n",
    "\n",
    "tweet_pairs = [(tweet_id_map[np.int64(tweet)], category_to_label[category]) \n",
    "               for category, tweet_ids in tweet_category_map.items() for tweet in tweet_ids]\n",
    "\n",
    "tweet_texts = [tp[0][\"text\"] for tp in tweet_pairs]\n",
    "\n",
    "y_data = np.array([tp[1] for tp in tweet_pairs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Samples: 63346 63346\n"
     ]
    }
   ],
   "source": [
    "print(\"Samples:\", len(tweet_pairs), len(y_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(s):\n",
    "    \"\"\"\n",
    "    Given a text, cleans and normalizes it. Feel free to add your own stuff.\n",
    "    From: https://www.kaggle.com/mschumacher/using-fasttext-models-for-robust-embeddings\n",
    "    \"\"\"\n",
    "    s = s.lower()\n",
    "\n",
    "    # Replace numbers and symbols with language\n",
    "    s = s.replace('&', ' and ')\n",
    "    s = s.replace('@', ' at ')\n",
    "    s = s.replace('0', 'zero')\n",
    "    s = s.replace('1', 'one')\n",
    "    s = s.replace('2', 'two')\n",
    "    s = s.replace('3', 'three')\n",
    "    s = s.replace('4', 'four')\n",
    "    s = s.replace('5', 'five')\n",
    "    s = s.replace('6', 'six')\n",
    "    s = s.replace('7', 'seven')\n",
    "    s = s.replace('8', 'eight')\n",
    "    s = s.replace('9', 'nine')\n",
    "\n",
    "    return s\n",
    "\n",
    "analyzer = vectorizer.build_analyzer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_gensim = FastText.load('text_sample_2015_gensim.model')\n",
    "model_gensim = FastText.load('../models/text_sample_2013to2016_gensim_200.model')\n",
    "# model_gensim = FastText.load_fasttext_format('/scratch/olympus/models/fasttext/crawl-300d-2M-subword.bin')\n",
    "wvs = model_gensim.wv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorize(sentence):\n",
    "    tokenized = [normalize(t) for t in analyzer(sentence)]\n",
    "    \n",
    "    wv_vecs = []\n",
    "    for t in tokenized:\n",
    "\n",
    "        try:\n",
    "            v = wvs[t]\n",
    "            norm = np.linalg.norm(v)\n",
    "            normed_v = (v / norm)\n",
    "            wv_vecs.append(normed_v)\n",
    "        except:\n",
    "            continue\n",
    "    \n",
    "    m = np.array(wv_vecs)\n",
    "    normed_m = np.mean(m, axis=0)\n",
    "    \n",
    "    normed_m = np.nan_to_num(normed_m)\n",
    "\n",
    "    return normed_m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(63346, 200)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ft_features_ = [vectorize(s) for s in tweet_texts]\n",
    "ft_features = np.array([x for x in ft_features_])\n",
    "ft_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(63346, 16)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "other_ftr_data = np.array([other_features(tweet) for tweet, _ in tweet_pairs])\n",
    "other_ftr_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(63346, 98)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sources_ftrs = [tweet_sources_index.get(tweet[\"source\"], 0) for tweet, _ in tweet_pairs]\n",
    "\n",
    "source_ftr_onehot = []\n",
    "for ftr in sources_ftrs:\n",
    "    f = np.zeros(len(tweet_sources_index) + 1)\n",
    "    f[ftr] = 1.0\n",
    "    source_ftr_onehot.append(f)\n",
    "\n",
    "sources_ftr_data = np.array(source_ftr_onehot)    \n",
    "\n",
    "sources_ftr_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(63346, 314) (63346,)\n"
     ]
    }
   ],
   "source": [
    "X_data = np.concatenate([\n",
    "    ft_features, \n",
    "    other_ftr_data, \n",
    "    sources_ftr_data,\n",
    "#     pos\n",
    "], axis=1)\n",
    "\n",
    "print(X_data.shape, y_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "r_state = 1337"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_params = {\n",
    "    'n_estimators': 128, \n",
    "    \"n_jobs\": -1,\n",
    "    'random_state': r_state,\n",
    "    'class_weight': \"balanced\",\n",
    "    'criterion': 'gini',\n",
    "    'max_depth': 32,\n",
    "    'max_features': 113,\n",
    "    'min_samples_leaf': 2,\n",
    "    'min_samples_split': 54,\n",
    "}\n",
    "\n",
    "nb_params = {\n",
    "    'alpha': 0.6836531055077686,\n",
    "    'binarize': 0.027689715150536642,\n",
    "    'fit_prior': True\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAccuracy: 0.16025843050740624\n",
      "\tF1: 0.10634629511296997\n",
      "\tAccuracy: 0.13526722371117766\n",
      "\tF1: 0.10620616029586442\n",
      "\tAccuracy: 0.17959634184799747\n",
      "\tF1: 0.1404290742319845\n",
      "\tAccuracy: 0.18025548020816906\n",
      "\tF1: 0.13142838665916082\n",
      "\tAccuracy: 0.20870799810695692\n",
      "\tF1: 0.1146253533654823\n",
      "\tAccuracy: 0.17650773602778655\n",
      "\tF1: 0.14165941030443313\n",
      "\tAccuracy: 0.20717446270543616\n",
      "\tF1: 0.1456122017199815\n",
      "\tAccuracy: 0.2041106719367589\n",
      "\tF1: 0.1457744187053992\n",
      "\tAccuracy: 0.14482213438735178\n",
      "\tF1: 0.09395242167971292\n",
      "\tAccuracy: 0.12003795666613949\n",
      "\tF1: 0.08118529660768874\n",
      "Accuracy: 0.17167384361051802\n",
      "F1: 0.12072190186826774\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "f1_accum = []\n",
    "accuracy_accum = []\n",
    "\n",
    "skf = StratifiedKFold(n_splits=10, random_state=r_state)\n",
    "for train, test in skf.split(X_data, y_data):\n",
    "\n",
    "    X_train = X_data[train]\n",
    "    y_train = y_data[train]\n",
    "    \n",
    "    X_test = X_data[test]\n",
    "    y_test = y_data[test]\n",
    "\n",
    "    # train\n",
    "#     fitted_model = RandomForestClassifier(**rf_params)\n",
    "    fitted_model = BernoulliNB(**nb_params)\n",
    "    fitted_model.fit(X_train, y_train)\n",
    "\n",
    "    # Compute Precision-Recall \n",
    "    y_infer_local = fitted_model.predict(X_test)\n",
    "    local_f1 = f1_score(y_test, y_infer_local, average=\"macro\")\n",
    "    local_score = fitted_model.score(X_test, y_test)\n",
    "    print(\"\\tAccuracy:\", local_score)\n",
    "    print(\"\\tF1:\", local_f1)\n",
    "    \n",
    "    f1_accum.append(local_f1)\n",
    "    accuracy_accum.append(local_score)\n",
    "\n",
    "print(\"Accuracy:\", np.mean(accuracy_accum))\n",
    "print(\"F1:\", np.mean(f1_accum))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label: Advice\n",
      "Score: 0.8868436838948\n",
      "Label: CleanUp\n",
      "Score: 0.9952641050737221\n",
      "Label: ContextualInformation\n",
      "Score: 0.92092634104758\n",
      "Label: Discussion\n",
      "Score: 0.8648217724876077\n",
      "Label: Donations\n",
      "Score: 0.9233258611435607\n",
      "Label: EmergingThreats\n",
      "Score: 0.8985729169955483\n",
      "Label: Factoid\n",
      "Score: 0.7847377892842484\n",
      "Label: FirstPartyObservation\n",
      "Score: 0.8345909765415338\n",
      "Label: GoodsServices\n",
      "Score: 0.9784674644018565\n",
      "Label: Hashtags\n",
      "Score: 0.7759290247213715\n",
      "Label: InformationWanted\n",
      "Score: 0.9478735831781012\n",
      "Label: Location\n",
      "Score: 0.8676475231269536\n",
      "Label: MovePeople\n",
      "Score: 0.9748523979414644\n",
      "Label: MultimediaShare\n",
      "Score: 0.7734190004104442\n",
      "Label: NewSubEvent\n",
      "Score: 0.8932529283616961\n",
      "Label: News\n",
      "Score: 0.7085845988696997\n",
      "Label: Official\n",
      "Score: 0.9293404476999337\n",
      "Label: OriginalEvent\n",
      "Score: 0.8922110314779149\n",
      "Label: SearchAndRescue\n",
      "Score: 0.9894547406308212\n",
      "Label: Sentiment\n",
      "Score: 0.777112998452941\n",
      "Label: ServiceAvailable\n",
      "Score: 0.9431534745682443\n",
      "Label: ThirdPartyObservation\n",
      "Score: 0.7617844852082215\n",
      "Label: Volunteer\n",
      "Score: 0.961623464780728\n",
      "Label: Weather\n",
      "Score: 0.873125374925015\n",
      "Label: Irrelevant\n",
      "Score: 0.8115271682505604\n"
     ]
    }
   ],
   "source": [
    "category_number_list = list(category_to_label.values())\n",
    "label_to_category = {j:i for i, j in category_to_label.items()}\n",
    "for positive_category in category_number_list:\n",
    "    local_y_data = [1 if y == positive_category else 0 for y in y_data]\n",
    "    \n",
    "#     fitted_model = RandomForestClassifier(**rf_params)\n",
    "    fitted_model = BernoulliNB(**nb_params)\n",
    "    fitted_model.fit(X_data, local_y_data)\n",
    "   \n",
    "    print(\"Label:\", label_to_category[positive_category])\n",
    "    print(\"Score:\", fitted_model.score(X_data, local_y_data))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_search(X_data, y_data, clf, param_dist, n_iter_search=20, r_state=1337):\n",
    "    # run randomized search\n",
    "    random_search = RandomizedSearchCV(clf, \n",
    "                                       param_distributions=param_dist,\n",
    "                                       n_iter=n_iter_search,\n",
    "                                       cv=10,\n",
    "                                       scoring=\"f1_macro\",\n",
    "                                       random_state=r_state,\n",
    "                                       verbose=2,\n",
    "                                       n_jobs=-1,\n",
    "                                      )\n",
    "    \n",
    "    random_search.fit(X_data, y_data)\n",
    "\n",
    "    return (random_search.best_score_, random_search.best_params_)\n",
    "\n",
    "def model_eval_rf(X_data, y_data, n_iter_search=100, r_state=1337):\n",
    "\n",
    "    clf = RandomForestClassifier(\n",
    "        n_estimators=100, class_weight=\"balanced\", random_state=r_state)\n",
    "    \n",
    "    # specify parameters and distributions to sample from\n",
    "    param_dist = {\n",
    "        \"max_depth\": scipy.stats.randint(2, 8),\n",
    "        \"max_features\": scipy.stats.randint(2, min(128, X_data.shape[1])),\n",
    "        \"min_samples_split\": scipy.stats.randint(2, 512),\n",
    "        \"min_samples_leaf\": scipy.stats.randint(2, 512),\n",
    "#         \"criterion\": [\"gini\", \"entropy\"],\n",
    "    }\n",
    "    \n",
    "    return random_search(X_data, y_data, clf, param_dist, n_iter_search=n_iter_search, r_state=r_state)\n",
    "\n",
    "def model_eval_nb(X_data, y_data, n_iter_search=100, r_state=1337):\n",
    "\n",
    "    clf = BernoulliNB()\n",
    "    \n",
    "    # specify parameters and distributions to sample from\n",
    "    param_dist = {\n",
    "        \"alpha\": scipy.stats.uniform(),\n",
    "        \"binarize\": scipy.stats.uniform(),\n",
    "        \"fit_prior\": [True, False],\n",
    "    }\n",
    "    \n",
    "    return random_search(X_data, y_data, clf, param_dist, n_iter_search=n_iter_search, r_state=r_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# search_results = model_eval_nb(X_data, y_data, n_iter_search=128)\n",
    "# search_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# search_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def smapper(x):\n",
    "    if ( x < 0.75 ):\n",
    "        return 0\n",
    "    else:\n",
    "        return 1\n",
    "\n",
    "y_data_regress = np.array([smapper(tweet_priority_map[np.int64(tp[0][\"id\"])][\"score\"]) \n",
    "                           for tp in tweet_pairs])\n",
    "y_data_weights = np.array([tweet_priority_map[np.int64(tp[0][\"id\"])][\"weight\"] \n",
    "                           for tp in tweet_pairs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_priority_params = {\n",
    "    'random_state': r_state,\n",
    "    'class_weight': 'balanced',\n",
    "    'n_estimators': 128, \n",
    "    'n_jobs': -1,\n",
    "    'max_depth': 50,\n",
    "    'max_features': 14,\n",
    "    'min_samples_leaf': 33,\n",
    "    'min_samples_split': 96,\n",
    "}\n",
    "\n",
    "nb_priority_params = {\n",
    "    'alpha': 0.05134305647695325,\n",
    "    'binarize': 0.045909955637688404,\n",
    "    'fit_prior': True,\n",
    "}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tFitting...\n",
      "\t 0.9378156565656566\n",
      "\tFitting...\n",
      "\t 0.9365530303030303\n",
      "\tFitting...\n",
      "\t 0.9622730860299921\n",
      "\tFitting...\n",
      "\t 0.9539068666140489\n",
      "\tFitting...\n",
      "\t 0.9714240606251974\n",
      "\tFitting...\n",
      "\t 0.9520050520997789\n",
      "\tFitting...\n",
      "\t 0.926586675086833\n",
      "\tFitting...\n",
      "\t 0.9501105146826649\n",
      "\tFitting...\n",
      "\t 0.9398484370066309\n",
      "\tFitting...\n",
      "\t 0.939690558888538\n",
      "Score (R^2): 0.9470213937902372\n",
      "Accuracy: 0.9470213937902372\n",
      "F1: 0.8882840963921066\n"
     ]
    }
   ],
   "source": [
    "score_accum = []\n",
    "f1_accum = []\n",
    "accuracy_accum = []\n",
    "\n",
    "skf = StratifiedKFold(n_splits=10, random_state=r_state)\n",
    "for train, test in skf.split(X_data, y_data_regress):\n",
    "\n",
    "    X_train = X_data[train]\n",
    "    y_train = y_data_regress[train]\n",
    "    y_weight = y_data_weights[train]\n",
    "    \n",
    "    X_test = X_data[test]\n",
    "    y_test = y_data_regress[test]\n",
    "\n",
    "    # train\n",
    "    print(\"\\tFitting...\")\n",
    "#     fitted_model = sklearn.linear_model.LinearRegression(n_jobs=4)\n",
    "#     fitted_model = sklearn.tree.DecisionTreeRegressor(random_state=r_state, max_depth=256)\n",
    "#     fitted_model = BernoulliNB(**nb_priority_params)\n",
    "    fitted_model = RandomForestClassifier(**rf_priority_params)\n",
    "    fitted_model.fit(X_train, y_train, y_weight)\n",
    "\n",
    "    # Compute score metrics\n",
    "    r2_score = fitted_model.score(X_test, y_test)\n",
    "    score_accum.append(r2_score)\n",
    "    \n",
    "    # Compute Precision-Recall \n",
    "    y_infer_local = fitted_model.predict(X_test)\n",
    "    f1_accum.append(f1_score(y_test, y_infer_local, average=\"macro\"))\n",
    "    \n",
    "    accuracy_accum.append(fitted_model.score(X_test, y_test))\n",
    "    \n",
    "    print(\"\\t\", r2_score)\n",
    "\n",
    "print(\"Score (R^2):\", np.mean(score_accum))\n",
    "print(\"Accuracy:\", np.mean(accuracy_accum))\n",
    "print(\"F1:\", np.mean(f1_accum))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15000, 314)\n"
     ]
    }
   ],
   "source": [
    "test_tweets = []\n",
    "with open(\"../data/2019b-testing.json\", \"r\") as in_file:\n",
    "    for line in in_file:\n",
    "        tweet_top = json.loads(line)\n",
    "        tweet = json.loads(tweet_top[\"allProperties\"][\"srcjson\"])\n",
    "        test_tweets.append(tweet)\n",
    "        \n",
    "X_test_ft_ = [vectorize(s) for s in [t[\"text\"] for t in test_tweets]]\n",
    "X_test_ft_ = np.array([x for x in X_test_ft_])\n",
    "\n",
    "X_test_other = np.array([other_features(tweet) for tweet in test_tweets])\n",
    "\n",
    "\n",
    "\n",
    "sources_ftrs = [tweet_sources_index.get(tweet[\"source\"], 0) for tweet in test_tweets]\n",
    "source_ftr_onehot = []\n",
    "for ftr in sources_ftrs:\n",
    "    f = np.zeros(len(tweet_sources_index) + 1)\n",
    "    f[ftr] = 1.0\n",
    "    source_ftr_onehot.append(f)\n",
    "X_test_sources = np.array(source_ftr_onehot) \n",
    "\n",
    "X_test_data = np.concatenate([\n",
    "    X_test_ft_, \n",
    "    X_test_other, \n",
    "    X_test_sources,\n",
    "], axis=1)\n",
    "\n",
    "print(X_test_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BernoulliNB(alpha=0.6836531055077686, binarize=0.027689715150536642,\n",
       "      class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fitted_model = BernoulliNB(**nb_params)\n",
    "# fitted_model = RandomForestClassifier(**rf_params)\n",
    "fitted_model.fit(X_data, y_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_labels = fitted_model.predict(X_test_data)\n",
    "\n",
    "labeled_test_data = list(zip([t[\"id\"] for t in test_tweets], y_test_labels))\n",
    "\n",
    "id_to_cat_map = {y:x for x,y in category_to_label.items()}\n",
    "\n",
    "df = pd.DataFrame([\n",
    "    {\"tweet_id\":np.int64(tup[0]), \"label\": id_to_cat_map[tup[1]]} for tup in labeled_test_data],\n",
    ")\n",
    "\n",
    "df.to_csv(\"trec2019b_test_results_run_fasttext.csv\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Advice</th>\n",
       "      <td>389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CleanUp</th>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ContextualInformation</th>\n",
       "      <td>1029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Discussion</th>\n",
       "      <td>1045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Donations</th>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EmergingThreats</th>\n",
       "      <td>142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Factoid</th>\n",
       "      <td>2971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FirstPartyObservation</th>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GoodsServices</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hashtags</th>\n",
       "      <td>556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>InformationWanted</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Irrelevant</th>\n",
       "      <td>1215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Location</th>\n",
       "      <td>1936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MovePeople</th>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MultimediaShare</th>\n",
       "      <td>985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NewSubEvent</th>\n",
       "      <td>378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>News</th>\n",
       "      <td>1089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Official</th>\n",
       "      <td>198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>OriginalEvent</th>\n",
       "      <td>151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SearchAndRescue</th>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sentiment</th>\n",
       "      <td>1136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ServiceAvailable</th>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ThirdPartyObservation</th>\n",
       "      <td>873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Volunteer</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Weather</th>\n",
       "      <td>390</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       tweet_id\n",
       "label                          \n",
       "Advice                      389\n",
       "CleanUp                      38\n",
       "ContextualInformation      1029\n",
       "Discussion                 1045\n",
       "Donations                   137\n",
       "EmergingThreats             142\n",
       "Factoid                    2971\n",
       "FirstPartyObservation        50\n",
       "GoodsServices                 1\n",
       "Hashtags                    556\n",
       "InformationWanted            99\n",
       "Irrelevant                 1215\n",
       "Location                   1936\n",
       "MovePeople                   40\n",
       "MultimediaShare             985\n",
       "NewSubEvent                 378\n",
       "News                       1089\n",
       "Official                    198\n",
       "OriginalEvent               151\n",
       "SearchAndRescue              18\n",
       "Sentiment                  1136\n",
       "ServiceAvailable            130\n",
       "ThirdPartyObservation       873\n",
       "Volunteer                     4\n",
       "Weather                     390"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(\"label\").count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1122269086338232321    25\n",
       "1122237890539421697    23\n",
       "1122258843927445504    19\n",
       "1122224999845249025    19\n",
       "1122242433150885888    14\n",
       "1125943643754377216    12\n",
       "1126163665248350210    12\n",
       "1125908981757743105    12\n",
       "1125868443004227584    12\n",
       "1122228733144788992    11\n",
       "1125894485479702528    11\n",
       "1125861997780852738    11\n",
       "1126164160624992256    11\n",
       "1122228151361232896    10\n",
       "1122236759406981121    10\n",
       "1122229347119710208    10\n",
       "1122250898154184704    10\n",
       "1125921493337169920    10\n",
       "1126107056778252294    10\n",
       "1122227608874307585     9\n",
       "1125949590077956100     9\n",
       "1122236467651194883     9\n",
       "1122221393888735232     9\n",
       "1122237648477786112     9\n",
       "1122257063336398848     9\n",
       "1125919362785792000     9\n",
       "1122250185776746498     9\n",
       "1122244508702216192     9\n",
       "1122270464330485761     9\n",
       "1122247870378315776     8\n",
       "                       ..\n",
       "1134427602758725632     1\n",
       "1122500084720181248     1\n",
       "1134844467444191232     1\n",
       "1120301644887904257     1\n",
       "1134428183493668864     1\n",
       "1120635938219028480     1\n",
       "1134463613253513218     1\n",
       "1122237176790618119     1\n",
       "1132264726673723393     1\n",
       "1122254722981871618     1\n",
       "1122112200049664000     1\n",
       "1122923224113799168     1\n",
       "1120521480444547073     1\n",
       "1134271762441674752     1\n",
       "1126124947032514562     1\n",
       "1122272224214306817     1\n",
       "1134182128659390464     1\n",
       "1120565756830339072     1\n",
       "1131085706128973824     1\n",
       "1120321948628578304     1\n",
       "1120319462199054338     1\n",
       "1122324944883474434     1\n",
       "1120565701637394432     1\n",
       "1121744361409134593     1\n",
       "1120336941575065603     1\n",
       "1125878491076079618     1\n",
       "1120565623816278016     1\n",
       "1134498625382256640     1\n",
       "1134506731420045312     1\n",
       "1133746032058101760     1\n",
       "Name: tweet_id, Length: 13916, dtype: int64"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"tweet_id\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight='balanced',\n",
       "            criterion='gini', max_depth=50, max_features=14,\n",
       "            max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "            min_impurity_split=None, min_samples_leaf=33,\n",
       "            min_samples_split=96, min_weight_fraction_leaf=0.0,\n",
       "            n_estimators=128, n_jobs=-1, oob_score=False,\n",
       "            random_state=1337, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fitted_pri_model = RandomForestClassifier(**rf_priority_params)\n",
    "fitted_pri_model.fit(X_data, y_data_regress)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_labels = fitted_pri_model.predict(X_test_data)\n",
    "\n",
    "labeled_test_data = list(zip([t[\"id\"] for t in test_tweets], y_test_labels))\n",
    "\n",
    "df = pd.DataFrame([{\"tweet_id\":tup[0], \"priority\": tup[1]} for tup in labeled_test_data])\n",
    "\n",
    "df.to_csv(\"trec2019b_test_results_priority_run_fasttext.csv\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    14729\n",
       "1      271\n",
       "Name: priority, dtype: int64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"priority\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Multi-Label Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_pairs = []\n",
    "y_data = []\n",
    "\n",
    "indexer = {c:i for i, c in enumerate(list(category_to_label.keys()))}\n",
    "indexer_inv = {i:c for c,i in indexer.items()}\n",
    "for tweet_id, categories in category_df.groupby(\"tweet_id\"):\n",
    "    \n",
    "    tup = (\n",
    "        tweet_id_map[np.int64(tweet_id)], \n",
    "        [indexer[category] for category in categories[\"category\"] if category != \"Unknown\"]\n",
    "    )\n",
    "    tweet_pairs.append(tup)\n",
    "\n",
    "tweet_texts = [tp[0][\"text\"] for tp in tweet_pairs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_data_ = [tp[1] for tp in tweet_pairs]\n",
    "\n",
    "def one_hot_y(y_list):\n",
    "    encoded = [0] * len(indexer)\n",
    "    for y in y_list:\n",
    "        encoded[y] = 1\n",
    "    return encoded\n",
    "\n",
    "y_data = np.array([one_hot_y(y) for y in y_data_])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(26144, 200)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ft_features_ = [vectorize(s) for s in tweet_texts]\n",
    "ft_features = np.array([x for x in ft_features_])\n",
    "ft_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(26144, 16)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "other_ftr_data = np.array([other_features(tweet) for tweet, _ in tweet_pairs])\n",
    "other_ftr_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "sources_ftrs = [tweet_sources_index.get(tweet[\"source\"], 0) for tweet, _ in tweet_pairs]\n",
    "source_ftr_onehot = []\n",
    "for ftr in sources_ftrs:\n",
    "    f = np.zeros(len(tweet_sources_index) + 1)\n",
    "    f[ftr] = 1.0\n",
    "    source_ftr_onehot.append(f)\n",
    "X_test_sources = np.array(source_ftr_onehot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(26144, 314) (26144, 25)\n"
     ]
    }
   ],
   "source": [
    "X_data = np.concatenate([\n",
    "    ft_features, \n",
    "    other_ftr_data, \n",
    "    X_test_sources,\n",
    "], axis=1)\n",
    "\n",
    "print(X_data.shape, y_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "rf_params = {\n",
    "    'n_estimators': 128, \n",
    "    \"n_jobs\": -1,\n",
    "    'random_state': r_state,\n",
    "    'class_weight': \"balanced\",\n",
    "    'criterion': 'gini',\n",
    "    'max_depth': 32,\n",
    "    'max_features': 113,\n",
    "    'min_samples_leaf': 2,\n",
    "    'min_samples_split': 54,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_accum = []\n",
    "accuracy_accum = []\n",
    "\n",
    "skf = KFold(n_splits=10, random_state=r_state)\n",
    "for train, test in skf.split(X_data, y_data):\n",
    "\n",
    "    X_train = X_data[train]    \n",
    "    X_test = X_data[test]\n",
    "    \n",
    "    y_train = y_data[train]\n",
    "    y_test = y_data[test]\n",
    "\n",
    "\n",
    "    # train\n",
    "    fitted_model = RandomForestClassifier(**rf_params)\n",
    "    fitted_model.fit(X_train, y_train)\n",
    "\n",
    "    # Compute Precision-Recall \n",
    "    y_infer_local = fitted_model.predict(X_test)\n",
    "    local_f1 = f1_score(y_test, y_infer_local, average=\"weighted\")\n",
    "    local_score = fitted_model.score(X_test, y_test)\n",
    "    print(\"\\tAccuracy:\", local_score)\n",
    "    print(\"\\tF1:\", local_f1)\n",
    "    \n",
    "    f1_accum.append(local_f1)\n",
    "    accuracy_accum.append(local_score)\n",
    "\n",
    "print(\"Accuracy:\", np.mean(accuracy_accum))\n",
    "print(\"F1:\", np.mean(f1_accum))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = fitted_model.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = sklearn.multiclass.OneVsRestClassifier(RandomForestClassifier(**rf_params))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OneVsRestClassifier(estimator=RandomForestClassifier(bootstrap=True, class_weight='balanced',\n",
       "            criterion='gini', max_depth=32, max_features=113,\n",
       "            max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "            min_impurity_split=None, min_samples_leaf=2,\n",
       "            min_samples_split=54, min_weight_fraction_leaf=0.0,\n",
       "            n_estimators=128, n_jobs=-1, oob_score=False,\n",
       "            random_state=1337, verbose=0, warm_start=False),\n",
       "          n_jobs=None)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,\n",
       "       0, 0, 0])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(results[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0.5],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0.5],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ],\n",
       "       [0. ]])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sklearn.preprocessing.normalize(results[0].reshape(-1,1), norm=\"l1\", axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAccuracy: 0.047418738049713194\n",
      "\tF1: 0.32668181238566546\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAccuracy: 0.03173996175908222\n",
      "\tF1: 0.31477338315183667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAccuracy: 0.07762906309751434\n",
      "\tF1: 0.4552034182021693\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAccuracy: 0.07112810707456979\n",
      "\tF1: 0.43588296909640517\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAccuracy: 0.11055853098699311\n",
      "\tF1: 0.3308052741011941\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-63-7658c022bd89>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0;31m# train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mfitted_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmulticlass\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOneVsRestClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mRandomForestClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mrf_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m     \u001b[0mfitted_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0;31m# Compute Precision-Recall\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/multiclass.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    213\u001b[0m                 \u001b[0;34m\"not %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabel_binarizer_\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses_\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    214\u001b[0m                 self.label_binarizer_.classes_[i]])\n\u001b[0;32m--> 215\u001b[0;31m             for i, column in enumerate(columns))\n\u001b[0m\u001b[1;32m    216\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    217\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m    984\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_iterator\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    985\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 986\u001b[0;31m             \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    987\u001b[0m                 \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    988\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[0;34m(self, iterator)\u001b[0m\n\u001b[1;32m    823\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    824\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 825\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    826\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    827\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    780\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    781\u001b[0m             \u001b[0mjob_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 782\u001b[0;31m             \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    783\u001b[0m             \u001b[0;31m# A job can complete so quickly than its callback is\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    784\u001b[0m             \u001b[0;31m# called before we get here, causing self._jobs to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[0;34m(self, func, callback)\u001b[0m\n\u001b[1;32m    180\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    181\u001b[0m         \u001b[0;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 182\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    183\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    543\u001b[0m         \u001b[0;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    544\u001b[0m         \u001b[0;31m# arguments in memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 545\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    546\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    547\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    259\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 261\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    259\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 261\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/multiclass.py\u001b[0m in \u001b[0;36m_fit_binary\u001b[0;34m(estimator, X, y, classes)\u001b[0m\n\u001b[1;32m     78\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m         \u001b[0mestimator\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 80\u001b[0;31m         \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     81\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/ensemble/forest.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    333\u001b[0m                     \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrees\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    334\u001b[0m                     verbose=self.verbose, class_weight=self.class_weight)\n\u001b[0;32m--> 335\u001b[0;31m                 for i, t in enumerate(trees))\n\u001b[0m\u001b[1;32m    336\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    337\u001b[0m             \u001b[0;31m# Collect newly grown trees\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m    984\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_iterator\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    985\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 986\u001b[0;31m             \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    987\u001b[0m                 \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    988\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[0;34m(self, iterator)\u001b[0m\n\u001b[1;32m    823\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    824\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 825\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    826\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    827\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    780\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    781\u001b[0m             \u001b[0mjob_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 782\u001b[0;31m             \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    783\u001b[0m             \u001b[0;31m# A job can complete so quickly than its callback is\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    784\u001b[0m             \u001b[0;31m# called before we get here, causing self._jobs to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[0;34m(self, func, callback)\u001b[0m\n\u001b[1;32m    180\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    181\u001b[0m         \u001b[0;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 182\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    183\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    543\u001b[0m         \u001b[0;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    544\u001b[0m         \u001b[0;31m# arguments in memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 545\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    546\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    547\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    259\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 261\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    259\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 261\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/ensemble/forest.py\u001b[0m in \u001b[0;36m_parallel_build_trees\u001b[0;34m(tree, forest, X, y, sample_weight, tree_idx, n_trees, verbose, class_weight)\u001b[0m\n\u001b[1;32m    119\u001b[0m             \u001b[0mcurr_sample_weight\u001b[0m \u001b[0;34m*=\u001b[0m \u001b[0mcompute_sample_weight\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'balanced'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m         \u001b[0mtree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcurr_sample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheck_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    122\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m         \u001b[0mtree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheck_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/tree/tree.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight, check_input, X_idx_sorted)\u001b[0m\n\u001b[1;32m    797\u001b[0m             \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    798\u001b[0m             \u001b[0mcheck_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcheck_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 799\u001b[0;31m             X_idx_sorted=X_idx_sorted)\n\u001b[0m\u001b[1;32m    800\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    801\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/tree/tree.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight, check_input, X_idx_sorted)\u001b[0m\n\u001b[1;32m    363\u001b[0m                                            min_impurity_split)\n\u001b[1;32m    364\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 365\u001b[0;31m         \u001b[0mbuilder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtree_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_idx_sorted\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    366\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    367\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_outputs_\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "skf = KFold(n_splits=10, random_state=r_state)\n",
    "for train, test in skf.split(X_data, y_data):\n",
    "\n",
    "    X_train = X_data[train]    \n",
    "    X_test = X_data[test]\n",
    "    \n",
    "    y_train = y_data[train]\n",
    "    y_test = y_data[test]\n",
    "\n",
    "\n",
    "    # train\n",
    "    fitted_model = sklearn.multiclass.OneVsRestClassifier(RandomForestClassifier(**rf_params))\n",
    "    fitted_model.fit(X_train, y_train)\n",
    "\n",
    "    # Compute Precision-Recall \n",
    "    y_infer_local = fitted_model.predict(X_test)\n",
    "    local_f1 = f1_score(y_test, y_infer_local, average=\"weighted\")\n",
    "    local_score = fitted_model.score(X_test, y_test)\n",
    "    print(\"\\tAccuracy:\", local_score)\n",
    "    print(\"\\tF1:\", local_f1)\n",
    "    \n",
    "    f1_accum.append(local_f1)\n",
    "    accuracy_accum.append(local_score)\n",
    "\n",
    "print(\"Accuracy:\", np.mean(accuracy_accum))\n",
    "print(\"F1:\", np.mean(f1_accum))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prs = fitted_model.predict(X_test)\n",
    "prs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, x in enumerate(prs.sum(axis=0)):\n",
    "    print(i, indexer_inv[i], x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OneVsRestClassifier(estimator=RandomForestClassifier(bootstrap=True, class_weight='balanced',\n",
       "            criterion='gini', max_depth=32, max_features=113,\n",
       "            max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "            min_impurity_split=None, min_samples_leaf=2,\n",
       "            min_samples_split=54, min_weight_fraction_leaf=0.0,\n",
       "            n_estimators=128, n_jobs=-1, oob_score=False,\n",
       "            random_state=1337, verbose=0, warm_start=False),\n",
       "          n_jobs=None)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = sklearn.multiclass.OneVsRestClassifier(\n",
    "    RandomForestClassifier(**rf_params)\n",
    "#     BernoulliNB(**nb_params)\n",
    ")\n",
    "clf.fit(X_data, y_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_labels = clf.predict(X_test_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_rows = []\n",
    "for tid, row_labels in zip([t[\"id\"] for t in test_tweets], y_test_labels):\n",
    "    \n",
    "    row = [tid] + row_labels.tolist()\n",
    "    all_rows.append(row)\n",
    "\n",
    "\n",
    "id_to_cat_map = {y:x for x,y in category_to_label.items()}\n",
    "\n",
    "df = pd.DataFrame(all_rows, columns=[\"tweet_id\"] + [indexer_inv[i] for i in range(len(indexer_inv))])\n",
    "\n",
    "df.to_csv(\"trec2019b_test_results_run_fasttext_multi.csv\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}