01_Baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "import gzip\n",
    "import json\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy\n",
    "from scipy import interpolate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.cluster\n",
    "import sklearn.feature_extraction \n",
    "import sklearn.feature_extraction.text\n",
    "import sklearn.metrics\n",
    "import sklearn.preprocessing\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.semi_supervised import LabelPropagation\n",
    "from sklearn.semi_supervised import LabelSpreading\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/clb617/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/nltk/twitter/__init__.py:20: UserWarning: The twython library has not been installed. Some functionality from the twitter package will not be available.\n",
      "  warnings.warn(\"The twython library has not been installed. \"\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download(\"stopwords\")\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer as VS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Tweet Count: 49147\n"
     ]
    }
   ],
   "source": [
    "tweet_id_map = {}\n",
    "with open(\"../data/2019b-testing.json\", \"r\") as in_file:\n",
    "    for line in in_file:\n",
    "        tweet_top = json.loads(line)\n",
    "        tweet = json.loads(tweet_top[\"allProperties\"][\"srcjson\"])\n",
    "        tweet_id_map[np.int64(tweet[\"id\"])] = tweet\n",
    "        \n",
    "with open(\"../data/2019b-training.json\", \"r\") as in_file:\n",
    "    for line in in_file:\n",
    "        tweet = json.loads(line)\n",
    "        tweet_id_map[np.int64(tweet[\"id\"])] = tweet\n",
    "\n",
    "print(\"Total Tweet Count:\", len(tweet_id_map))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'en': 48776,\n",
       " 'es': 60,\n",
       " 'tl': 153,\n",
       " 'it': 3,\n",
       " 'fr': 13,\n",
       " 'pt': 9,\n",
       " 'und': 95,\n",
       " 'hi': 10,\n",
       " 'nl': 3,\n",
       " 'ca': 2,\n",
       " 'eu': 1,\n",
       " 'in': 6,\n",
       " 'ro': 3,\n",
       " 'et': 1,\n",
       " 'de': 1,\n",
       " 'cs': 1,\n",
       " 'ht': 1,\n",
       " 'ja': 4,\n",
       " 'th': 1,\n",
       " 'cy': 1,\n",
       " 'lt': 1,\n",
       " 'pl': 1,\n",
       " 'tr': 1}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lang_count_map = {}\n",
    "for lang in [tweet[\"lang\"] for tweet in tweet_id_map.values()]:\n",
    "    lang_count_map[lang] = lang_count_map.get(lang, 0) + 1\n",
    "lang_count_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tweets with Links: 23073\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'dlvr.it': 697,\n",
       " 'cyber4hits.com': 1,\n",
       " 'gocyber.ca': 1,\n",
       " 'athabascaadvocate.com': 3,\n",
       " 'wp.me': 167,\n",
       " 'ow.ly': 621,\n",
       " '511.alberta.ca': 3,\n",
       " 'wildfire.alberta.ca': 12,\n",
       " 'bit.ly': 2618,\n",
       " 'twitter.com': 3674,\n",
       " 'crweworld.com': 4,\n",
       " 'wfp.to': 5,\n",
       " 'trib.al': 205,\n",
       " 'instagram.com': 617,\n",
       " 'calgaryherald.com': 13,\n",
       " 'cbc.ca': 158,\n",
       " 'emergencyalert.alberta.ca': 27,\n",
       " 'gobluecircle.com': 1,\n",
       " 'rdcounty.ca': 1,\n",
       " 'podcasts.apple.com': 1,\n",
       " 'globalnews.ca': 88,\n",
       " 'firesmoke.ca': 9,\n",
       " 'nzzl.us': 3,\n",
       " 'edmonton.ctvnews.ca': 42,\n",
       " 'evnsocialnewswire.ch': 3,\n",
       " 'edmontonjournal.com': 87,\n",
       " 'ruq.us': 1,\n",
       " 'kelownanow.com': 1,\n",
       " 'spacevac.ca': 1,\n",
       " 'sapphirepureclean.co.uk': 1,\n",
       " 'alberta.ca': 10,\n",
       " 'flic.kr': 9,\n",
       " 'facebook.com': 207,\n",
       " 'wsoe.org': 9,\n",
       " 'ktvz.com': 3,\n",
       " 'huffingtonpost.ca': 8,\n",
       " 'greatfallstribune.com': 1,\n",
       " 'a.msn.com': 94,\n",
       " 'boysbygirls.co.uk': 2,\n",
       " 'todayville.com': 2,\n",
       " 'tmz.com': 3,\n",
       " 'thestar.com': 16,\n",
       " 'torstar.co': 8,\n",
       " 'go.usa.gov': 12,\n",
       " 'mingooland.com': 2,\n",
       " 'albertafirebans.ca': 2,\n",
       " 'ibc.ca': 5,\n",
       " 'theweathernetwork.com': 12,\n",
       " 'ahs.ca': 7,\n",
       " 'ctvnews.ca': 15,\n",
       " 'dailykos.com': 24,\n",
       " 'eaglenews.net': 1,\n",
       " 'follownews.com': 1,\n",
       " 'earth.nullschool.net': 1,\n",
       " 'ab.211.ca': 1,\n",
       " 'ab.nationtalk.ca': 1,\n",
       " 'macleans.ca': 7,\n",
       " 'go.wisc.edu': 2,\n",
       " 'wawanesa.com': 1,\n",
       " 'theglobeandmail.com': 7,\n",
       " 'thepostmillennial.com': 2,\n",
       " 'srd.web.alberta.ca': 5,\n",
       " 'youtu.be': 268,\n",
       " 'vice.com': 3,\n",
       " 'wildfiretoday.com': 15,\n",
       " 'lnkr.fm': 1,\n",
       " 'apple.news': 67,\n",
       " 'atmosphere.copernicus.eu': 1,\n",
       " 'cabinradio.ca': 6,\n",
       " 'airvisual.com': 1,\n",
       " 'edmonton.citynews.ca': 1,\n",
       " 'aptnnews.ca': 1,\n",
       " 'edmontonsun.com': 11,\n",
       " '660citynews.com': 6,\n",
       " 'yukon.ca': 1,\n",
       " 'ohs-pubstore.labour.alberta.ca': 1,\n",
       " 'theprogress.com': 1,\n",
       " 'medicinehatnews.com': 1,\n",
       " 'cjwe.ca': 1,\n",
       " 'ctv.news': 20,\n",
       " 'lesprom.com': 1,\n",
       " 'firefightingincanada.com': 1,\n",
       " 'gpwx.news': 9,\n",
       " 'buff.ly': 180,\n",
       " 'nnsl.com': 1,\n",
       " 'tinyurl.com': 166,\n",
       " 'flip.it': 26,\n",
       " 'evacuationpayment.alberta.ca': 2,\n",
       " 'theworldnewstoday.com': 7,\n",
       " 'highlevel.ca': 2,\n",
       " 'search.ecointernet.org': 13,\n",
       " 'albertaparks.ca': 1,\n",
       " 'treaty8.ca': 6,\n",
       " 'ecointernet.org': 5,\n",
       " 'hubs.ly': 3,\n",
       " 'alertable.ca': 4,\n",
       " 'discoverairdrie.com': 1,\n",
       " 'fields1650zy.zamsblog.com': 1,\n",
       " 'wayback.archive.org': 2,\n",
       " 'nait.li': 2,\n",
       " 'canadapost.ca': 2,\n",
       " 'wcpert.com': 1,\n",
       " 'weather.gc.ca': 6,\n",
       " 'albertahealthservices.ca': 4,\n",
       " 'washingtonpost.com': 27,\n",
       " 'blogs.discovermagazine.com': 4,\n",
       " 'dailyhive.com': 1,\n",
       " 'theguardian.com': 160,\n",
       " 'tgam.ca': 4,\n",
       " 'everythinggp.com': 4,\n",
       " 'mygrandeprairienow.com': 4,\n",
       " 'cnn.com': 354,\n",
       " 'huffp.st': 5,\n",
       " 'ph.news.yahoo.com': 7,\n",
       " 'en.wikipedia.org': 7,\n",
       " 'canadanewsmedia.ca': 1,\n",
       " 'youtube.com': 116,\n",
       " 'cwfis.cfs.nrcan.gc.ca': 3,\n",
       " 'citynews1130.com': 2,\n",
       " 'cengizadabag.com': 2,\n",
       " 'nsd61.ca': 1,\n",
       " 'pscp.tv': 40,\n",
       " 'vox.com': 11,\n",
       " 'open.alberta.ca': 1,\n",
       " 'assets.ibc.ca': 1,\n",
       " 'mymcmurray.com': 1,\n",
       " 'canada.watsupamericas.com': 1,\n",
       " 'google.ca': 5,\n",
       " 'vancouversun.com': 2,\n",
       " 'stalberttoday.ca': 1,\n",
       " 'omny.fm': 2,\n",
       " 'pnas.org': 1,\n",
       " 'strangesounds.org': 7,\n",
       " 'dvdclip.com': 1,\n",
       " 'bluebubbleworld.org': 1,\n",
       " 'calgary.ctvnews.ca': 3,\n",
       " 'player.630ched.com': 4,\n",
       " 'bridgecitynews.ca': 1,\n",
       " 'tfsnews.com': 2,\n",
       " 'foodbanksalberta.ca': 1,\n",
       " 'lnkd.in': 33,\n",
       " 'airquality.alberta.ca': 1,\n",
       " 'mix1037fm.com': 2,\n",
       " 'news4jax.com': 1,\n",
       " 'topstyletoday.com': 1,\n",
       " 'newssummedup.com': 3,\n",
       " 'rivercountry.fm': 1,\n",
       " 'firesmartcanada.ca': 1,\n",
       " 'news.deepfind.org': 6,\n",
       " 'una.ca': 1,\n",
       " 'chch.com': 1,\n",
       " 'thecruzy.com': 1,\n",
       " 'bhive.nectar.social': 1,\n",
       " 'forum.telus.com': 1,\n",
       " 'aqicn.org': 2,\n",
       " 'pres5.com': 7,\n",
       " 'albertaspca.org': 1,\n",
       " 'peaceriver.ca': 1,\n",
       " 'aos.wisc.edu': 1,\n",
       " 'moveuptogether.ca': 1,\n",
       " 'theenergymix.com': 1,\n",
       " 'capitalairshed.ca': 1,\n",
       " 'scientificamerican.com': 2,\n",
       " 'treaty8urban.ca': 1,\n",
       " 'aema.alberta.ca': 2,\n",
       " 'netnewsledger.com': 2,\n",
       " 'pb.fyi': 1,\n",
       " 'n60.nationtalk.ca': 1,\n",
       " 'esrd.maps.arcgis.com': 1,\n",
       " 'nationalpost.com': 2,\n",
       " 'linkedin.com': 3,\n",
       " 'rabble.ca': 1,\n",
       " 'folio.ca': 1,\n",
       " 'halifaxtoday.ca': 1,\n",
       " 'keyc.com': 1,\n",
       " 'beyondinsurance.ca': 1,\n",
       " 'cnn.it': 194,\n",
       " 'khq.com': 1,\n",
       " 'ctvnewsedmonton.ca': 1,\n",
       " 'our-time.ca': 1,\n",
       " 'cfweradio.ca': 1,\n",
       " 'countytimes.co.uk': 1,\n",
       " 'nbcmontana.com': 1,\n",
       " 'wxcentre.ca': 2,\n",
       " 'comixology.com': 1,\n",
       " 'hopeforwellness.ca': 1,\n",
       " 'stormchaserkyle.blogspot.com': 1,\n",
       " 'rammb-slider.cira.colostate.edu': 1,\n",
       " 'rdnewsnow.com': 1,\n",
       " 'dailyheraldtribune.com': 1,\n",
       " 'waqi.info': 1,\n",
       " 'atb.com': 1,\n",
       " 'reut.rs': 77,\n",
       " 'foxnews.com': 72,\n",
       " 'wbir.com': 4,\n",
       " 'msn.com': 27,\n",
       " 'nba101.com': 1,\n",
       " 'foxla.com': 2,\n",
       " 'link.medium.com': 2,\n",
       " 'cbsloc.al': 28,\n",
       " 'usatoday.com': 65,\n",
       " 'nbcnews.to': 65,\n",
       " 'af11.wordpress.com': 1,\n",
       " 'hill.cm': 6,\n",
       " 'nytimes.com': 127,\n",
       " 'kollmanreport.com': 6,\n",
       " 'denver.cbslocal.com': 3,\n",
       " 'abcnews.go.com': 20,\n",
       " 'on.nbc10.com': 2,\n",
       " 'thedenverchannel.com': 42,\n",
       " 'washingtontimes.com': 20,\n",
       " 'apnews.com': 20,\n",
       " 'is.gd': 72,\n",
       " 'nbcnews.com': 188,\n",
       " 'si.com': 19,\n",
       " 'dailymail.co.uk': 15,\n",
       " 'alternet.org': 7,\n",
       " 'newsclads.com': 6,\n",
       " 'gobgh.co.uk': 11,\n",
       " 'espn.com': 14,\n",
       " 'eaglenews.ph': 1,\n",
       " 'cpr.org': 4,\n",
       " 'wavy.com': 1,\n",
       " 'rssfeeds.dnj.com': 1,\n",
       " 'cbsnews.com': 65,\n",
       " 'www-m.cnn.com': 8,\n",
       " 'news.google.com': 14,\n",
       " 'click.e.usatoday.com': 2,\n",
       " 'nypost.com': 8,\n",
       " 'abcactionnews.com': 3,\n",
       " 'stripes.com': 3,\n",
       " 'theonion.com': 1,\n",
       " 'newsinfo.inquirer.net': 30,\n",
       " 'itv.com': 11,\n",
       " 'yahoo.com': 40,\n",
       " 'wbrz.com': 1,\n",
       " 'klkntv.com': 1,\n",
       " 'latimes.com': 48,\n",
       " 'fw.to': 30,\n",
       " 'news247worldpressuk.com': 1,\n",
       " 'pipr.es': 1,\n",
       " '939theeagle.com': 1,\n",
       " 'independent.co.uk': 34,\n",
       " 'on9news.tv': 11,\n",
       " 'miamiherald.com': 5,\n",
       " 'edweek.org': 2,\n",
       " 'pluralism.gr': 2,\n",
       " 'amp.cnn.com': 12,\n",
       " 'abc7chicago.com': 5,\n",
       " 'deseretnews.com': 3,\n",
       " 'yhoo.it': 61,\n",
       " 'news.sky.com': 24,\n",
       " 'local12.com': 2,\n",
       " 'fox10phoenix.com': 1,\n",
       " 'wdam.com': 1,\n",
       " 'indy100.com': 2,\n",
       " 'depauliaonline.com': 2,\n",
       " 'cbs17.com': 1,\n",
       " 'ktul.com': 2,\n",
       " 'fromthetrenchesworldreport.com': 3,\n",
       " 'go.shr.lc': 9,\n",
       " 'break.ma': 4,\n",
       " 'goo.gl': 486,\n",
       " 'one-news.net': 1,\n",
       " 'rssfeeds.theleafchronicle.com': 1,\n",
       " 'cbsn.ws': 37,\n",
       " 'ajc.com': 2,\n",
       " 'keloland.com': 2,\n",
       " 'whdh.com': 2,\n",
       " 'ind.pn': 9,\n",
       " 'reuters.com': 27,\n",
       " 'fox21news.com': 1,\n",
       " 'abcn.ws': 69,\n",
       " 'denverpost.com': 23,\n",
       " 'thecalwashingtonpost.com': 7,\n",
       " 'live5news.com': 1,\n",
       " 'nyti.ms': 254,\n",
       " 'boston-informer.com': 1,\n",
       " 'wnem.com': 1,\n",
       " 'disq.us': 7,\n",
       " 'splinternews.com': 1,\n",
       " 'commondreams.org': 2,\n",
       " 'fxn.ws': 40,\n",
       " 'es.pn': 8,\n",
       " 'ihe.art': 7,\n",
       " 'ktar.com': 2,\n",
       " 'google.com': 24,\n",
       " 'articletrunk.com': 5,\n",
       " '1310kfka.com': 1,\n",
       " 'wusa9.com': 1,\n",
       " 'fox29.com': 1,\n",
       " 'atxne.ws': 1,\n",
       " 'bradenton.com': 1,\n",
       " 'metanewshub.com': 1,\n",
       " 'thebreakingnewsheadlines.com': 1,\n",
       " 'dizzed.com': 3,\n",
       " 'whio.com': 2,\n",
       " 'nc1.tv': 1,\n",
       " 'heavy.com': 2,\n",
       " 'wfmynews2.com': 2,\n",
       " 'journalgazette.net': 1,\n",
       " '9news.com': 5,\n",
       " 'kron4.com': 2,\n",
       " 'ksby.com': 3,\n",
       " 'unfilteredradionetwork.com': 2,\n",
       " 'go.witf.org': 1,\n",
       " 'newsbreakapp.com': 15,\n",
       " 'lowellsun.com': 1,\n",
       " 'abc12.com': 1,\n",
       " 'knews2030.com': 2,\n",
       " 'en.azvision.az': 7,\n",
       " 'kltv.com': 1,\n",
       " 'westword.com': 1,\n",
       " 'relentlessschoolnurse.com': 1,\n",
       " 'bestspecialnews.com': 4,\n",
       " 'daytondailynews.com': 1,\n",
       " 'go.cbn.com': 2,\n",
       " 'giffords.me': 1,\n",
       " 'zpr.io': 5,\n",
       " 'theirworld.org': 2,\n",
       " 'witf.org': 1,\n",
       " 'theworldnewsonline.com': 2,\n",
       " 'va.newsrepublic.net': 4,\n",
       " '12newsnow.com': 4,\n",
       " 'fox19.com': 1,\n",
       " 'pbs.org': 9,\n",
       " 'mistermaxxx08.wordpress.com': 1,\n",
       " 'voiceofpeopletoday.com': 7,\n",
       " 'abc7.com': 14,\n",
       " 'journal-news.com': 1,\n",
       " 'to.pbs.org': 4,\n",
       " 'wtvy.com': 1,\n",
       " 'newzparot.com': 3,\n",
       " 'ktok.iheart.com': 2,\n",
       " 'ibexnews24.com': 1,\n",
       " 'abc3340.com': 3,\n",
       " 'wistv.com': 1,\n",
       " 'news.vice.com': 4,\n",
       " 'thehill.com': 10,\n",
       " 'abc7news.com': 19,\n",
       " 'springfieldnewssun.com': 1,\n",
       " 'ktvo.com': 2,\n",
       " 'msnbc.com': 6,\n",
       " 'politico.com': 3,\n",
       " 'kdvr.com': 6,\n",
       " 'f0rk.in': 6,\n",
       " 'forbes.com': 6,\n",
       " 'mailchi.mp': 4,\n",
       " 'wmcactionnews5.com': 1,\n",
       " 'po.st': 17,\n",
       " 'breaking.iavian.net': 5,\n",
       " 'cbs12.com': 1,\n",
       " 'wdbj7.com': 1,\n",
       " 'fox26houston.com': 2,\n",
       " 'jbs.cu.ma': 3,\n",
       " 'via.kdvr.com': 1,\n",
       " 'mirror.co.uk': 9,\n",
       " 'dpo.st': 35,\n",
       " 'kywnewsradio.radio.com': 2,\n",
       " '6abc.com': 4,\n",
       " 'mystateline.com': 1,\n",
       " 'southbendtribune.com': 1,\n",
       " 'dontspreadmywealth.com': 1,\n",
       " '11alive.com': 2,\n",
       " 'local10.com': 3,\n",
       " 'wreg.com': 1,\n",
       " 'amp.si.com': 1,\n",
       " 'cna.asia': 8,\n",
       " 'fox5ny.com': 2,\n",
       " 'espn.go.com': 1,\n",
       " 'ktts.com': 1,\n",
       " 'politi.co': 5,\n",
       " 'thequint.com': 4,\n",
       " 'charlotteobserver.com': 3,\n",
       " 'wgbh.org': 1,\n",
       " 'va.news-republic.com': 1,\n",
       " 'kget.com': 1,\n",
       " 'ground-beyond-zero.com': 1,\n",
       " 'chochilino.com': 2,\n",
       " 'newscentermaine.com': 1,\n",
       " 'vlsm.io': 1,\n",
       " 'abc27.com': 1,\n",
       " 'r29.co': 1,\n",
       " 'newyork.7a7.info': 1,\n",
       " 'nbcsandiego.com': 54,\n",
       " 'apne.ws': 53,\n",
       " 'video.foxnews.com': 9,\n",
       " 'rssfeeds.jacksonsun.com': 1,\n",
       " 'dakpeon24.com': 1,\n",
       " 'waff.com': 2,\n",
       " 'npr.org': 10,\n",
       " 'dailyhealthsecrets.com': 1,\n",
       " 'klfy.com': 1,\n",
       " 'prettifytoday.info': 1,\n",
       " 'wptv.com': 2,\n",
       " 'news.yahoo.com': 51,\n",
       " 'outlookindia.com': 1,\n",
       " 'techjollof.com': 4,\n",
       " 'newwestcity.com': 2,\n",
       " 'topbuzz.com': 1,\n",
       " 'katv.com': 1,\n",
       " 'centralillinoisproud.com': 1,\n",
       " 'wspa.com': 1,\n",
       " 'lex18.com': 1,\n",
       " 'khqa.com': 1,\n",
       " 'dribbble.com': 1,\n",
       " 'aljazeera.com': 35,\n",
       " 'mywabashvalley.com': 1,\n",
       " 'sent-trib.com': 1,\n",
       " 'talk1370.radio.com': 1,\n",
       " 'bingpedia.com': 11,\n",
       " 'start.toshiba.com': 2,\n",
       " 'timesfreepress.com': 1,\n",
       " 'tokyodailynews.com': 3,\n",
       " 'mprnews.org': 4,\n",
       " 'wchstv.com': 1,\n",
       " 'wfla.com': 1,\n",
       " 'esportnews.us': 1,\n",
       " 'cbsaustin.com': 1,\n",
       " 'wgme.com': 2,\n",
       " 'edition.cnn.com': 48,\n",
       " 'feeds.denverpost.com': 1,\n",
       " 'via.fox4kc.com': 1,\n",
       " 'macombdaily.com': 1,\n",
       " 'paper.li': 5,\n",
       " 'wowo.com': 1,\n",
       " 'kktv.com': 2,\n",
       " 'rssfeeds.knoxnews.com': 1,\n",
       " 'fox35orlando.com': 2,\n",
       " 'news.com.au': 7,\n",
       " 'wjla.com': 2,\n",
       " 'mynews4.com': 1,\n",
       " 'king5.com': 7,\n",
       " 'krdo.com': 3,\n",
       " 'via.whotv.com': 1,\n",
       " 'wqow.com': 1,\n",
       " 'wgxa.tv': 1,\n",
       " 'peptribe.info': 1,\n",
       " 'go.fox40.com': 1,\n",
       " 'wkrn.com': 1,\n",
       " 'wbko.com': 1,\n",
       " 'e-news.us': 1,\n",
       " 'rockrivertimes.com': 1,\n",
       " 'newjersey.news12.com': 1,\n",
       " 'tmblr.co': 61,\n",
       " 'ktbb.com': 1,\n",
       " 'fox32chicago.com': 1,\n",
       " 'rssfeeds.tennessean.com': 1,\n",
       " 'anews.com.tr': 2,\n",
       " 'collingwoodtoday.ca': 1,\n",
       " 'wate.com': 2,\n",
       " '2wsb.tv': 3,\n",
       " 'rssfeeds.theadvertiser.com': 1,\n",
       " 'hannaperkins.org': 1,\n",
       " 'cnycentral.com': 1,\n",
       " 'wdtn.com': 1,\n",
       " 'wgem.com': 1,\n",
       " 'rssfeeds.statesmanjournal.com': 3,\n",
       " 'stemk12.org': 1,\n",
       " 'newschannel9.com': 2,\n",
       " 'post-gazette.com': 2,\n",
       " 'rssfeeds.freep.com': 1,\n",
       " 'nbc12.com': 1,\n",
       " 'nbcchi.com': 1,\n",
       " 'newsandguts.com': 1,\n",
       " 'wwaytv3.com': 1,\n",
       " 'banglaviral.com': 1,\n",
       " 'investorshub.advfn.com': 1,\n",
       " 'vladtv.com': 3,\n",
       " 'abc11.com': 7,\n",
       " 'wtnh.com': 1,\n",
       " 'newsdecide.com': 1,\n",
       " 'fox13news.com': 1,\n",
       " 'trendolizer.com': 5,\n",
       " 'objectivenews.co': 1,\n",
       " 'standard.co.uk': 7,\n",
       " 'hrld.us': 4,\n",
       " 'via.fox6now.com': 2,\n",
       " 'trendingafrica.co.uk': 10,\n",
       " 'climatechangenews.com': 7,\n",
       " 'p.dw.com': 2,\n",
       " 'iol.co.za': 40,\n",
       " 'weather.us': 1,\n",
       " 'aspasiaeuropean.com': 3,\n",
       " 'naijanews.com': 1,\n",
       " 'mashable.com': 17,\n",
       " 'wunderground.com': 17,\n",
       " 'france24.com': 1,\n",
       " 'rocketnews.com': 10,\n",
       " 'sa-news.com': 1,\n",
       " 'poandpo.com': 16,\n",
       " 'bradblog.com': 1,\n",
       " 'axios.com': 4,\n",
       " 'bbc.in': 210,\n",
       " 'thenewhumanitarian.org': 8,\n",
       " 'afroinsider.com': 4,\n",
       " 'news.un.org': 6,\n",
       " 'global-monitoring.com': 2,\n",
       " 'news24.com': 68,\n",
       " 'plow.io': 1,\n",
       " 'shar.es': 74,\n",
       " 'enca.com': 27,\n",
       " 'businesslive.co.za': 1,\n",
       " 'ecigcanadazone.com': 2,\n",
       " 'africanews.com': 6,\n",
       " 'accuweather.com': 32,\n",
       " 'aje.io': 27,\n",
       " 'bbc.co.uk': 202,\n",
       " 'thedailyusnews.com': 2,\n",
       " 'twib.in': 26,\n",
       " 'regularnews.net': 4,\n",
       " '0news.net': 4,\n",
       " 'hurricanes.einnews.com': 14,\n",
       " 'arcg.is': 1,\n",
       " 'unosat-maps.web.cern.ch': 1,\n",
       " 'gdacs.org': 3,\n",
       " 'news.trust.org': 7,\n",
       " 'thenewsjournal.org': 4,\n",
       " 'smh.com.au': 7,\n",
       " 'thelighthouse.purephpbb.com': 5,\n",
       " 'severe-weather.eu': 1,\n",
       " 'unicef.org.uk': 2,\n",
       " 'oxf.am': 11,\n",
       " 'bloom.bg': 30,\n",
       " 'human-wrongs-watch.net': 1,\n",
       " 'nation.co.ke': 4,\n",
       " 'face2faceafrica.com': 1,\n",
       " 'hrw.org': 18,\n",
       " 'meganewstime.com': 1,\n",
       " 'time.com': 10,\n",
       " 'arkbridgeproperties.co.ke': 1,\n",
       " 'zimeye.net': 2,\n",
       " 'reliefweb.int': 11,\n",
       " 'benonicitytimes.co.za': 3,\n",
       " 'fal.cn': 2,\n",
       " 'healthtimes.co.zw': 1,\n",
       " '247newsupdate.com': 4,\n",
       " 'democracynow.org': 7,\n",
       " 'ewn.co.za': 19,\n",
       " 'wesleyan.org': 1,\n",
       " 'unicef.org': 4,\n",
       " 'newdelhitimes.com': 1,\n",
       " 'bbc.com': 150,\n",
       " 'allafrica.com': 15,\n",
       " 'news.ae': 2,\n",
       " 'letabaherald.co.za': 4,\n",
       " 'nationalgeographic.com': 1,\n",
       " 'emirates247.com': 4,\n",
       " 'thenation.com': 9,\n",
       " 'crisisrelief.un.org': 1,\n",
       " 'egu.eu': 2,\n",
       " 'afrikavuka.org': 2,\n",
       " 'blogs.nasa.gov': 3,\n",
       " 'local21news.com': 1,\n",
       " 'atcnews.org': 2,\n",
       " 'careinternational.org.uk': 1,\n",
       " 'watchers.news': 10,\n",
       " 'rootsafrikiko.com': 5,\n",
       " 'westerncapegangwatch.co.za': 4,\n",
       " 'dailymaverick.co.za': 2,\n",
       " 'hozint.com': 7,\n",
       " 'hornglobe.com': 1,\n",
       " 'unocha.org': 1,\n",
       " 'movetosydney.com': 7,\n",
       " 'guardian.ng': 2,\n",
       " 'mozambique.unfpa.org': 1,\n",
       " 'zimetro.co.zw': 2,\n",
       " 'rescue-sa.co.za': 1,\n",
       " 'ladunliadinews.com': 1,\n",
       " 'bonbonlifestylewebazine.com': 3,\n",
       " 'khalilhumam.com': 28,\n",
       " 'news.eud.adventist.org': 1,\n",
       " 'tiny.ng': 4,\n",
       " 'seattle-informer.com': 1,\n",
       " 'currenttrending.com': 2,\n",
       " 'sumo.ly': 2,\n",
       " 'bloomberg.com': 14,\n",
       " 'windy.com': 1,\n",
       " 'abc.net.au': 34,\n",
       " 'jubileedebt.org.uk': 5,\n",
       " 'clubofmozambique.com': 11,\n",
       " 'timeslive.co.za': 19,\n",
       " 'amp.theguardian.com': 1,\n",
       " 'twinklenews.com': 2,\n",
       " 'worldvision.com.au': 3,\n",
       " 'climateactionaustralia.wordpress.com': 2,\n",
       " 'hvper.com': 2,\n",
       " 'zimbabwestar.com': 1,\n",
       " 'iom.int': 1,\n",
       " 'digitaljournal.com': 4,\n",
       " 'annahar.com': 1,\n",
       " 'rss.cnn.com': 10,\n",
       " 'wxch.nl': 2,\n",
       " 'dailynewsappraisal.com': 1,\n",
       " 'goatysnews.wordpress.com': 1,\n",
       " 'kenyan-digest.com': 3,\n",
       " 'explore-cape-verde.com': 1,\n",
       " 'tribune.com.pk': 1,\n",
       " 'truenews.ng': 1,\n",
       " 'canada.pres5.com': 1,\n",
       " 'care.ca': 2,\n",
       " 'newsinn.org': 2,\n",
       " 'tearfund.org.nz': 2,\n",
       " 'iharare.com': 2,\n",
       " 'undispatch.com': 3,\n",
       " 'hastingsobserver.co.uk': 1,\n",
       " 'm.accuweather.com': 7,\n",
       " 'amnesty.org': 2,\n",
       " '…turalhazardsanddisasters.blogspot.com': 1,\n",
       " 'qoo.ly': 2,\n",
       " 'mobile.reuters.com': 1,\n",
       " 'solutionsu.solutionsjournalism.org': 2,\n",
       " 'eurekalert.org': 3,\n",
       " 'kenn.tt': 1,\n",
       " 'biztechhub.co.zw': 1,\n",
       " 'news.bahai.org': 2,\n",
       " 'u.afp.com': 11,\n",
       " 'n.pr': 22,\n",
       " 'newsheadlines.com.ng': 1,\n",
       " 'sg.news.yahoo.com': 3,\n",
       " 'uncensoredopinion.co.za': 1,\n",
       " 'oddcrimes.com': 5,\n",
       " 'english.alarabiya.net': 5,\n",
       " 'the-herald.zw': 1,\n",
       " 'newsdzezimbabwe.co.uk': 1,\n",
       " 'phys.org': 3,\n",
       " 'uk.news.yahoo.com': 5,\n",
       " 'awrld.at': 1,\n",
       " 'oxfam.org.au': 1,\n",
       " 'public.wmo.int': 1,\n",
       " 'unrefugees.org.au': 1,\n",
       " 'worldvision.org': 2,\n",
       " 'france.timesofnews.com': 1,\n",
       " 'fai.informazione.it': 1,\n",
       " 'islamic-relief.org': 1,\n",
       " 'earthobservatory.nasa.gov': 4,\n",
       " 'feeds.feedblitz.com': 1,\n",
       " 'africatimes.com': 2,\n",
       " '7news.com.au': 1,\n",
       " 'newsatw.com': 2,\n",
       " 'sakai-hk.com': 1,\n",
       " 'unwebtv.live': 1,\n",
       " 'ino.to': 2,\n",
       " 'save.tc': 2,\n",
       " 'go.nasa.gov': 9,\n",
       " 'telegraph.co.uk': 16,\n",
       " 'africazine.com': 3,\n",
       " 'japantimes.co.jp': 4,\n",
       " 'sandiegoinformer.com': 4,\n",
       " 'dw.com': 4,\n",
       " 'voanews.com': 8,\n",
       " 'fastcompany.com': 1,\n",
       " 'gu.com': 111,\n",
       " 'thewire.in': 1,\n",
       " 'africa.timesofnews.com': 1,\n",
       " 'modernghana.com': 3,\n",
       " 'inewsguyana.com': 1,\n",
       " 'jojosnewsblog.com': 1,\n",
       " 'irisglobal.org': 1,\n",
       " 'emergency.copernicus.eu': 1,\n",
       " 'europa.eu': 1,\n",
       " 'climatecrocks.com': 1,\n",
       " 'links.cfr.mkt5175.com': 2,\n",
       " 'en.banglatribune.com': 2,\n",
       " 'thepeninsulaqatar.com': 5,\n",
       " 'nst.com.my': 1,\n",
       " 'furtherafrica.com': 1,\n",
       " 'weather.com': 15,\n",
       " 'herald.co.zw': 1,\n",
       " 'thesouthafrican.com': 15,\n",
       " 'watsupafrica.com': 1,\n",
       " 'directrelief.org': 7,\n",
       " 'care.org': 3,\n",
       " 'uni.cf': 9,\n",
       " 'on.wkyc.com': 5,\n",
       " 'tropical.atmos.colostate.edu': 1,\n",
       " 'mefmi.org': 1,\n",
       " 'theage.com.au': 2,\n",
       " 'enfoquenoticias.com.mx': 1,\n",
       " 'radionz.co.nz': 2,\n",
       " 'za.trendwiki.co': 11,\n",
       " 'thedailystar.net': 2,\n",
       " 'hisz.rsoe.hu': 5,\n",
       " 'news360.com': 11,\n",
       " 'bayareane.ws': 1,\n",
       " 'oxfamireland.org': 1,\n",
       " 'all4women.co.za': 5,\n",
       " 'tropic.ssec.wisc.edu': 1,\n",
       " 'unhcr.org': 1,\n",
       " 'topstyletodaycom.com': 1,\n",
       " 'trendynewstime.com': 1,\n",
       " 'newscul.blogspot.com': 1,\n",
       " 'wapo.st': 20,\n",
       " 'tropicaltidbits.com': 1,\n",
       " 'newsok.com': 2,\n",
       " 'robertscribbler.com': 1,\n",
       " 'brandpowerng.com': 1,\n",
       " 'tinyw.in': 2,\n",
       " 'unicefusa.org': 9,\n",
       " 'ledger-enquirer.com': 1,\n",
       " 'trtworld.com': 3,\n",
       " 'compassionservices.org': 1,\n",
       " 'storms.pw': 2,\n",
       " 'tmsnrt.rs': 6,\n",
       " 'aiwa.press': 1,\n",
       " 'dpaq.de': 1,\n",
       " 'nnuya.com': 3,\n",
       " 'insight.wfp.org': 1,\n",
       " 'talkradio.co.uk': 1,\n",
       " 'worldcentralkitchen.org': 1,\n",
       " 'startribune.com': 1,\n",
       " 'zwnews.com': 1,\n",
       " 'jpost.com': 10,\n",
       " 'gulf-times.com': 1,\n",
       " 'gov.uk': 2,\n",
       " 'uphindia.com': 4,\n",
       " 'newsday.co.zw': 1,\n",
       " 'africanharvesters.com': 1,\n",
       " 'iono.fm': 8,\n",
       " 'nairobinews.nation.co.ke': 1,\n",
       " 'uk.reuters.com': 5,\n",
       " 'media.ifrc.org': 3,\n",
       " 'latestcommentary.com': 2,\n",
       " 'ift.tt': 375,\n",
       " 'daily-sun.com': 2,\n",
       " '1848.nl': 1,\n",
       " 'taipeitimes.com': 2,\n",
       " 'rtlnieuws.nl': 1,\n",
       " 'drk.de': 1,\n",
       " 'pic.twitter.com': 70,\n",
       " 'nemafoundation.org': 1,\n",
       " 'zimtimes.com': 1,\n",
       " 'ssd.noaa.gov': 6,\n",
       " 'anarchimedia.com': 1,\n",
       " 'peoplesdispatch.org': 1,\n",
       " 'crowdrise.com': 9,\n",
       " 'zoomzimbabwe.com': 2,\n",
       " 'africa.cgtn.com': 2,\n",
       " 'thisisafrica.me': 1,\n",
       " 'developmentarena.blogspot.com': 1,\n",
       " 'standardmedia.co.ke': 2,\n",
       " '350.org': 2,\n",
       " 'news.mb.com.ph': 7,\n",
       " 'e.africa': 2,\n",
       " 'news.pindula.co.zw': 4,\n",
       " 'endtimeheadlines.org': 2,\n",
       " 'timesofearth.com': 1,\n",
       " 'gadgetsnow.com': 1,\n",
       " 'news2.fr': 3,\n",
       " 'sarcsapp.co.za': 2,\n",
       " 'ippf.org': 1,\n",
       " 'ncbi.nlm.nih.gov': 1,\n",
       " 'tasks.hotosm.org': 8,\n",
       " 'isitjust.us': 1,\n",
       " 'f7td5.app.goo.gl': 3,\n",
       " 'mailtribune.com': 1,\n",
       " 'rappler.com': 58,\n",
       " 'wxcharts.com': 1,\n",
       " 'earther.gizmodo.com': 4,\n",
       " 'transafricaradio.net': 1,\n",
       " 'ptvng.com': 2,\n",
       " 'an24.net': 1,\n",
       " 'globaljusticeonline.org': 1,\n",
       " 'the-star.co.ke': 2,\n",
       " 'eturbonews.com': 1,\n",
       " 'stateofpress.com': 1,\n",
       " 'nnn.com.ng': 1,\n",
       " 'viralupdatenews.com': 2,\n",
       " 'dailysign.al': 2,\n",
       " 'upi.com': 7,\n",
       " 'tropicalstormrisk.com': 2,\n",
       " 'newscientist.com': 2,\n",
       " 'tweetedtimes.com': 17,\n",
       " 'globaldispatchespodcast.com': 1,\n",
       " 'af.reuters.com': 3,\n",
       " 'kansascity.com': 1,\n",
       " 'ghanaguardian.com': 4,\n",
       " 'click1.crm.foreignpolicy.com': 1,\n",
       " 'travis.af.mil': 1,\n",
       " 'visionviral.com': 1,\n",
       " 'libya360.wordpress.com': 1,\n",
       " 'skepticalscience.com': 1,\n",
       " 'reut.tv': 1,\n",
       " 'alhuria.com': 2,\n",
       " 'gofundme.com': 18,\n",
       " 'islamic-relief.org.uk': 1,\n",
       " 'wetteronline.de': 1,\n",
       " 'sfgate.com': 3,\n",
       " 'kxnet.com': 1,\n",
       " 'feeds.reuters.com': 9,\n",
       " 'gf.me': 1,\n",
       " 'donate.actionaidusa.org': 1,\n",
       " 'hddsexplorer.usgs.gov': 1,\n",
       " 'independent.ng': 1,\n",
       " 'ecowatch.com': 7,\n",
       " 'blog.gogreenr12.org': 2,\n",
       " 'trendingpress.com': 2,\n",
       " 'm.youtube.com': 11,\n",
       " 'nationalgeographic.org': 1,\n",
       " 'africanexponent.com': 1,\n",
       " 'wionews.com': 1,\n",
       " 'c.bank': 1,\n",
       " 'news3lv.com': 1,\n",
       " 'rte.ie': 5,\n",
       " 'kijani.info': 1,\n",
       " 'eblaradio.com': 2,\n",
       " 'tuko.co.ke': 1,\n",
       " 'support.wfpusa.org': 1,\n",
       " 'primeviewtv.com': 1,\n",
       " 'londonglossy.com': 1,\n",
       " 'others.direct': 1,\n",
       " 'thebigwobble.org': 1,\n",
       " 'travel.state.gov': 2,\n",
       " 'indiandiplomacy.org': 2,\n",
       " 'sampur.se': 1,\n",
       " 'sunnewsonline.com': 2,\n",
       " 'environewsnigeria.com': 2,\n",
       " 'translatorswithoutborders.org': 1,\n",
       " 'alternativeafrica.com': 1,\n",
       " 'erccportal.jrc.ec.europa.eu': 1,\n",
       " 'citizen.co.za': 5,\n",
       " 'wepen.in': 1,\n",
       " 'qz.com': 14,\n",
       " 'trivis.de': 1,\n",
       " 'dld.bz': 9,\n",
       " 'internationalpeoplesearch.com': 2,\n",
       " 'm.news24.com': 10,\n",
       " 'livingstrongtv.com': 1,\n",
       " 'chrisonet.com': 1,\n",
       " 'sabcnews.com': 13,\n",
       " 'thetimes.co.uk': 1,\n",
       " 're.ssec.wisc.edu': 4,\n",
       " 'heykayjones.com.ng': 1,\n",
       " 'act.gp': 1,\n",
       " 'msf.org': 1,\n",
       " 'newsnow.co.uk': 1,\n",
       " 'wired.co.uk': 1,\n",
       " 'emsc-csem.org': 18,\n",
       " 'ripplesdaily.com': 2,\n",
       " 'intelli.news': 2,\n",
       " 'somsirsa.wordpress.com': 4,\n",
       " 'earthquake.phivolcs.dost.gov.ph': 51,\n",
       " 'volcanodiscovery.com': 12,\n",
       " '1bataan.com': 1,\n",
       " 'channelnewsasia.com': 6,\n",
       " 'google1.org': 1,\n",
       " 'www3.nhk.or.jp': 1,\n",
       " 'news.lalate.com': 2,\n",
       " 'simplenews.co.uk': 4,\n",
       " 'pna.gov.ph': 19,\n",
       " 'americanmilitarynews.com': 2,\n",
       " 'newsbitsph.com': 2,\n",
       " 'thenational.ae': 3,\n",
       " 'automaticblogging.com': 1,\n",
       " 'ayalafoundation.org': 2,\n",
       " 'sortiwa.com': 2,\n",
       " 'earthquake.usgs.gov': 26,\n",
       " 'reddit.com': 5,\n",
       " 'mol.im': 1,\n",
       " 'thehindu.com': 4,\n",
       " 'philstar.com': 13,\n",
       " 'gizmodo.com': 7,\n",
       " 'ready.gov': 1,\n",
       " 'cnnphilippines.com': 11,\n",
       " 'st.news': 2,\n",
       " 'on.rt.com': 29,\n",
       " 'earthquaketrack.com': 4,\n",
       " 'map.lu': 4,\n",
       " 'headlines.taipeinewsnetwork.com': 1,\n",
       " 'besttennews.com': 1,\n",
       " 'news.abs-cbn.com': 3,\n",
       " 'enjeuxenergies.wordpress.com': 2,\n",
       " 'reports.unocha.org': 1,\n",
       " 'article.worldnews.com': 2,\n",
       " 'onemediaph.com': 2,\n",
       " 'abc7.la': 3,\n",
       " 'j.mp': 141,\n",
       " 'panahon.tv': 2,\n",
       " '24fun.me': 3,\n",
       " 'cnn.ph': 5,\n",
       " 'ospreyflightsolutions.com': 1,\n",
       " 'varsitarian.net': 1,\n",
       " 'thesummitexpress.com': 3,\n",
       " 'weatheralertonline.com': 2,\n",
       " 'northluzon.politics.com.ph': 1,\n",
       " 'spinics.net': 2,\n",
       " 'play.google.com': 5,\n",
       " 'coconuts.co': 2,\n",
       " 'conceptnewscentral.com': 2,\n",
       " 'web.facebook.com': 4,\n",
       " 'on.doi.gov': 34,\n",
       " 'cms.insureteck.com': 1,\n",
       " 'space.trendolizer.com': 1,\n",
       " 'cebupacificair.com': 1,\n",
       " 'iloilometropolitantimes.com': 1,\n",
       " 'newsinflight.com': 1,\n",
       " 'theleaderassumpta.com': 2,\n",
       " 'artikuloblogazine.com': 1,\n",
       " 'iamdouzone.com': 1,\n",
       " 't.co': 10,\n",
       " 'timebulletin.com': 1,\n",
       " 'clips.twitch.tv': 1,\n",
       " 'manilainformer.com': 9,\n",
       " 'thesun.co.uk': 1,\n",
       " 'english.alahednews.com.lb': 1,\n",
       " 'theusposts.com': 4,\n",
       " 'ptwc.weather.gov': 3,\n",
       " 'devdiscourse.com': 2,\n",
       " 'lat.ms': 19,\n",
       " 'gizmo.do': 1,\n",
       " 'indiatvnews.com': 1,\n",
       " 'patriotify.com': 1,\n",
       " 'garda.page.link': 3,\n",
       " 'khaleejtimes.com': 2,\n",
       " 'curiouscat.me': 2,\n",
       " 'mgadgetsworld.wordpress.com': 1,\n",
       " 'conandaily.com': 1,\n",
       " 'enfieldindependent.co.uk': 1,\n",
       " 'gamerix.net': 1,\n",
       " 'thrillblender.com': 1,\n",
       " 'earthquake-report.com': 4,\n",
       " 'newsnation.in': 2,\n",
       " 'marieblog.com': 1,\n",
       " 'sptnkne.ws': 5,\n",
       " 'tradeforprofit.net': 2,\n",
       " 'wa-news.com': 3,\n",
       " 'thisisinsider.com': 1,\n",
       " 'droolindog.net': 3,\n",
       " 'sunstar.com.ph': 2,\n",
       " 'expatreaders.com': 1,\n",
       " 'gmanetwork.com': 5,\n",
       " 'gcfrng.com': 4,\n",
       " 'inquirer.net': 1,\n",
       " 'eqgr.gr': 3,\n",
       " 'firstcoastnews.com': 2,\n",
       " 'str.sg': 16,\n",
       " 'sputniknews.com': 3,\n",
       " 'quakeapp.com': 2,\n",
       " 'earthcentral.org': 1,\n",
       " 'philippines.liveuamap.com': 1,\n",
       " 'on.digg.com': 1,\n",
       " 'ndtv.com': 9,\n",
       " 'phivolcs.dost.gov.ph': 1,\n",
       " 'boredpanda.com': 1,\n",
       " 'jerichosplace.com': 1,\n",
       " 'electroverse.net': 1,\n",
       " 'philnewstoday.com': 1,\n",
       " 'unravelmalta.com': 1,\n",
       " 'cebudailynews.inquirer.net': 2,\n",
       " 'tvnz.co.nz': 1,\n",
       " 'eqbot.com': 2,\n",
       " 'eos.org': 1,\n",
       " 'earthquakealert.blogspot.com': 4,\n",
       " 'meteotube.net': 1,\n",
       " 'scmp.com': 2,\n",
       " '…ben-earthquake-terremoto.blogspot.com': 1,\n",
       " 'ph72.org': 1,\n",
       " 'cbmin.org': 1,\n",
       " 'earthquak.es': 3,\n",
       " '7d.news': 1,\n",
       " '7news.link': 2,\n",
       " 'm.facebook.com': 9,\n",
       " 'timesnownews.com': 2,\n",
       " 'earthquakealert3.blogspot.com': 1,\n",
       " 'tribune.net.ph': 3,\n",
       " 'indiatimespost.com': 1,\n",
       " 'earthquakenewstoday.com': 3,\n",
       " 'shr.gs': 1,\n",
       " 'pia.gov.ph': 1,\n",
       " 'twitch.tv': 7,\n",
       " 'mag.time.com': 1,\n",
       " 'share.snippetmedia.com': 1,\n",
       " 'subic.morefun.ph': 1,\n",
       " 'business.inquirer.net': 1,\n",
       " 'googleweblight.com': 1,\n",
       " '7ny.tv': 6,\n",
       " 'philnews.ph': 1,\n",
       " 'wsvn.com': 2,\n",
       " 'aviation24.be': 1,\n",
       " 'haaretz.com': 4,\n",
       " 'myonde.com': 1,\n",
       " 'fsbuq.com': 1,\n",
       " '10daily.com.au': 1,\n",
       " 'wthr.com': 2,\n",
       " 'welovetheearth.org': 1,\n",
       " 'spoti.fi': 1,\n",
       " 'sandiegouniontribune.com': 27,\n",
       " 'citinews.co.uk': 1,\n",
       " 'interspacereporter.com': 3,\n",
       " 'businessmirror.com.ph': 1,\n",
       " 'insder.co': 1,\n",
       " 'via.fox43.com': 2,\n",
       " 'bible.com': 1,\n",
       " 'balikas.net': 1,\n",
       " 'unofficialnetworks.com': 1,\n",
       " 'swift1.phivolcs.dost.gov.ph': 1,\n",
       " 'worldandeverything.org': 1,\n",
       " 'thecontext.net': 2,\n",
       " 'heraldscotland.com': 1,\n",
       " 'asianews.network': 1,\n",
       " ...}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "link_count = 0\n",
    "domain_count_map = {}\n",
    "for entity in [tweet[\"entities\"] for tweet in tweet_id_map.values() if \"entities\" in tweet]:\n",
    "\n",
    "    if ( len(entity[\"urls\"]) > 0 ):\n",
    "        link_count += 1\n",
    "        \n",
    "        for url in entity[\"urls\"]:\n",
    "            domain = url[\"display_url\"].lower().partition(\"/\")[0]\n",
    "            domain_count_map[domain] = domain_count_map.get(domain, 0) + 1\n",
    "print(\"Tweets with Links:\", link_count)\n",
    "domain_count_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tweets with Images: 6149\n"
     ]
    }
   ],
   "source": [
    "img_count = 0\n",
    "for entity in [tweet[\"entities\"] for tweet in tweet_id_map.values() if \"entities\" in tweet]:\n",
    "\n",
    "    if ( \"media\" in entity ):\n",
    "#         print(entity[\"media\"])\n",
    "        img_count += 1\n",
    "print(\"Tweets with Images:\", img_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'symbols': [],\n",
       " 'urls': [{'expanded_url': 'http://CyG-NewsAgent.net/NewsAd.php?url=http://bit.ly/VPxmU3',\n",
       "   'indices': [65, 87],\n",
       "   'display_url': 'CyG-NewsAgent.net/NewsAd.php?urlâ\\x80Š',\n",
       "   'url': 'http://t.co/9ccakAJjE3'}],\n",
       " 'hashtags': [{'text': 'NewsAd', 'indices': [52, 59]}],\n",
       " 'user_mentions': []}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sources: 1011\n",
      "Mapped Sources: 97\n"
     ]
    }
   ],
   "source": [
    "tweet_sources_map = {}\n",
    "\n",
    "for tweet in tweet_id_map.values():\n",
    "    tweet_sources_map[tweet[\"source\"]] = tweet_sources_map.get(tweet[\"source\"], 0) + 1\n",
    "\n",
    "print(\"Sources:\", len(tweet_sources_map))\n",
    "tweet_sources_index = {x:i+1 for i, x in enumerate([x for x in tweet_sources_map.keys()if tweet_sources_map[x] > 10])}\n",
    "\n",
    "print(\"Mapped Sources:\", len(tweet_sources_index))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_priority_map = {}\n",
    "priority_df = pd.read_csv(\"tweet_to_priority.csv\", dtype={\"tweet_id\": np.int64})\n",
    "for row in priority_df.itertuples():\n",
    "    tweet_id = row.tweet_id\n",
    "    \n",
    "    tweet_priority_map[tweet_id] = {\n",
    "        \"score\": row.score_mean,\n",
    "        \"weight\": 1.0 - row.score_std\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_category_map = {}\n",
    "category_df = pd.read_csv(\"tweet_to_category.csv\")\n",
    "\n",
    "cat_update_map = {\n",
    "    \"ContinuingNews\": \"News\",\n",
    "    \"PastNews\": \"ContextualInformation\",\n",
    "    \"KnownAlready\": \"OriginalEvent\",\n",
    "    \"SignificantEventChange\": \"NewSubEvent\",\n",
    "}\n",
    "\n",
    "category_df[\"category\"] = category_df[\"category\"].apply(lambda x: cat_update_map.get(x, x))\n",
    "\n",
    "for category, tweets in category_df.groupby(\"category\"):\n",
    "    tweet_category_map[category] = list(tweets[\"tweet_id\"])\n",
    "    \n",
    "# Deleted in 2019\n",
    "del(tweet_category_map[\"Unknown\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Category: Advice\n",
      "\tTweet Count: 1706 Retrieved Fraction: 1.0\n",
      "\t {'en': 1705, 'und': 1}\n",
      "Category: CleanUp\n",
      "\tTweet Count: 135 Retrieved Fraction: 1.0\n",
      "\t {'en': 135}\n",
      "Category: ContextualInformation\n",
      "\tTweet Count: 1610 Retrieved Fraction: 1.0\n",
      "\t {'en': 1561, 'hi': 1, 'es': 39, 'pt': 2, 'it': 2, 'fr': 2, 'und': 2, 'tl': 1}\n",
      "Category: Discussion\n",
      "\tTweet Count: 2921 Retrieved Fraction: 1.0\n",
      "\t {'en': 2878, 'und': 5, 'hi': 1, 'es': 27, 'pt': 3, 'it': 1, 'fr': 2, 'tl': 2, 'nl': 2}\n",
      "Category: Donations\n",
      "\tTweet Count: 894 Retrieved Fraction: 1.0\n",
      "\t {'en': 894}\n",
      "Category: EmergingThreats\n",
      "\tTweet Count: 1192 Retrieved Fraction: 1.0\n",
      "\t {'en': 1191, 'fr': 1}\n",
      "Category: Factoid\n",
      "\tTweet Count: 3525 Retrieved Fraction: 1.0\n",
      "\t {'en': 3511, 'tl': 8, 'und': 2, 'ca': 1, 'es': 3}\n",
      "Category: FirstPartyObservation\n",
      "\tTweet Count: 4216 Retrieved Fraction: 1.0\n",
      "\t {'en': 4211, 'hi': 2, 'und': 2, 'tl': 1}\n",
      "Category: GoodsServices\n",
      "\tTweet Count: 146 Retrieved Fraction: 1.0\n",
      "\t {'en': 146}\n",
      "Category: Hashtags\n",
      "\tTweet Count: 7159 Retrieved Fraction: 1.0\n",
      "\t {'en': 7043, 'es': 12, 'it': 1, 'fr': 2, 'und': 36, 'ca': 1, 'eu': 1, 'pt': 1, 'hi': 2, 'tl': 50, 'cy': 2, 'in': 4, 'pl': 1, 'lt': 1, 'ja': 1, 'tr': 1}\n",
      "Category: InformationWanted\n",
      "\tTweet Count: 259 Retrieved Fraction: 1.0\n",
      "\t {'en': 259}\n",
      "Category: Irrelevant\n",
      "\tTweet Count: 5294 Retrieved Fraction: 1.0\n",
      "\t {'en': 5223, 'und': 22, 'pt': 3, 'in': 1, 'es': 6, 'tl': 32, 'fr': 1, 'nl': 1, 'hi': 3, 'pl': 1, 'tr': 1}\n",
      "Category: Location\n",
      "\tTweet Count: 1506 Retrieved Fraction: 1.0\n",
      "\t {'en': 1506}\n",
      "Category: MovePeople\n",
      "\tTweet Count: 157 Retrieved Fraction: 1.0\n",
      "\t {'en': 157}\n",
      "Category: MultimediaShare\n",
      "\tTweet Count: 5696 Retrieved Fraction: 1.0\n",
      "\t {'en': 5642, 'tl': 3, 'und': 9, 'hi': 1, 'es': 33, 'it': 2, 'fr': 2, 'pt': 1, 'pl': 1, 'ja': 1, 'cy': 1}\n",
      "Category: NewSubEvent\n",
      "\tTweet Count: 591 Retrieved Fraction: 1.0\n",
      "\t {'en': 585, 'tl': 4, 'et': 2}\n",
      "Category: News\n",
      "\tTweet Count: 6837 Retrieved Fraction: 1.0\n",
      "\t {'en': 6819, 'ja': 2, 'tl': 7, 'und': 4, 'hi': 1, 'fr': 1, 'es': 1, 'pt': 1, 'cy': 1}\n",
      "Category: Official\n",
      "\tTweet Count: 705 Retrieved Fraction: 1.0\n",
      "\t {'en': 702, 'fr': 1, 'pt': 1, 'es': 1}\n",
      "Category: OriginalEvent\n",
      "\tTweet Count: 2111 Retrieved Fraction: 1.0\n",
      "\t {'en': 2054, 'ja': 2, 'hi': 1, 'es': 30, 'it': 1, 'pt': 1, 'tl': 9, 'fr': 2, 'und': 9, 'cy': 1, 'lt': 1}\n",
      "Category: SearchAndRescue\n",
      "\tTweet Count: 307 Retrieved Fraction: 1.0\n",
      "\t {'en': 307}\n",
      "Category: Sentiment\n",
      "\tTweet Count: 8806 Retrieved Fraction: 1.0\n",
      "\t {'en': 8694, 'tl': 49, 'und': 50, 'hi': 4, 'es': 1, 'eu': 1, 'in': 4, 'pl': 1, 'lt': 1, 'tr': 1}\n",
      "Category: ServiceAvailable\n",
      "\tTweet Count: 1285 Retrieved Fraction: 1.0\n",
      "\t {'en': 1281, 'hi': 2, 'ro': 2}\n",
      "Category: ThirdPartyObservation\n",
      "\tTweet Count: 4520 Retrieved Fraction: 1.0\n",
      "\t {'en': 4517, 'ca': 1, 'tl': 1, 'pt': 1}\n",
      "Category: Volunteer\n",
      "\tTweet Count: 145 Retrieved Fraction: 1.0\n",
      "\t {'en': 145}\n",
      "Category: Weather\n",
      "\tTweet Count: 1623 Retrieved Fraction: 1.0\n",
      "\t {'en': 1621, 'tl': 2}\n"
     ]
    }
   ],
   "source": [
    "for category, tweet_ids in tweet_category_map.items():\n",
    "    retrieved_count = sum([1 if np.int64(tid) in tweet_id_map else 0 in tweet_id_map for tid in tweet_ids])\n",
    "    print(\"Category:\", category)\n",
    "    print(\"\\tTweet Count:\", len(tweet_ids), \"Retrieved Fraction:\", retrieved_count/len(tweet_ids))\n",
    "    \n",
    "    lang_count_map = {}\n",
    "    for lang in [tweet_id_map[np.int64(tid)][\"lang\"] for tid in tweet_ids if int(tid) in tweet_id_map]:\n",
    "        lang_count_map[lang] = lang_count_map.get(lang, 0) + 1\n",
    "    print(\"\\t\", str(lang_count_map))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# But first, read in stopwrods\n",
    "enStop = stopwords.words('english')\n",
    "\n",
    "# Skip stop words, retweet signs, @ symbols, and URL headers\n",
    "stopList = enStop +\\\n",
    "    [\"http\", \"https\", \"rt\", \"@\", \":\", \"t.co\", \"co\", \"amp\", \"&amp;\", \"...\", \"\\n\", \"\\r\"]\n",
    "stopList.extend(string.punctuation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def tokenizer_wrapper(text):\n",
    "#     return [t.lemma_ for t in nlp(text)]\n",
    "\n",
    "local_tokenizer = TweetTokenizer()\n",
    "def tokenizer_wrapper(text):\n",
    "    return local_tokenizer.tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate Additional Features\n",
    "sentiment_analyzer = VS()\n",
    "\n",
    "def count_twitter_objs(text_string):\n",
    "    \"\"\"\n",
    "    Accepts a text string and replaces:\n",
    "    1) urls with URLHERE\n",
    "    2) lots of whitespace with one instance\n",
    "    3) mentions with MENTIONHERE\n",
    "    4) hashtags with HASHTAGHERE\n",
    "\n",
    "    This allows us to get standardized counts of urls and mentions\n",
    "    Without caring about specific people mentioned.\n",
    "    \n",
    "    Returns counts of urls, mentions, and hashtags.\n",
    "    \"\"\"\n",
    "    space_pattern = '\\s+'\n",
    "    giant_url_regex = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'\n",
    "        '[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')\n",
    "    mention_regex = '@[\\w\\-]+'\n",
    "    hashtag_regex = '#[\\w\\-]+'\n",
    "    parsed_text = re.sub(space_pattern, ' ', text_string)\n",
    "    parsed_text = re.sub(giant_url_regex, 'URLHERE', parsed_text)\n",
    "    parsed_text = re.sub(mention_regex, 'MENTIONHERE', parsed_text)\n",
    "    parsed_text = re.sub(hashtag_regex, 'HASHTAGHERE', parsed_text)\n",
    "    return(parsed_text.count('URLHERE'),parsed_text.count('MENTIONHERE'),parsed_text.count('HASHTAGHERE'))\n",
    "\n",
    "## Taken from Davidson et al.\n",
    "def other_features(tweet):\n",
    "    \"\"\"This function takes a string and returns a list of features.\n",
    "    These include Sentiment scores, Text and Readability scores,\n",
    "    as well as Twitter specific features\"\"\"\n",
    "    tweet_text = tweet[\"text\"]\n",
    "    \n",
    "    ##SENTIMENT\n",
    "    sentiment = sentiment_analyzer.polarity_scores(tweet_text)\n",
    "    \n",
    "    words = local_tokenizer.tokenize(tweet_text) #Get text only\n",
    "    \n",
    "    num_chars = sum(len(w) for w in words) #num chars in words\n",
    "    num_chars_total = len(tweet_text)\n",
    "    num_terms = len(tweet_text.split())\n",
    "    num_words = len(words)\n",
    "    num_unique_terms = len(set([x.lower() for x in words]))\n",
    "    \n",
    "    caps_count = sum([1 if x.isupper() else 0 for x in tweet_text])\n",
    "    caps_ratio = caps_count / num_chars_total\n",
    "    \n",
    "    twitter_objs = count_twitter_objs(tweet_text) #Count #, @, and http://\n",
    "    num_media = 0\n",
    "    if \"entities\" in tweet and \"media\" in tweet[\"entities\"]:\n",
    "        num_media = len(tweet[\"entities\"][\"media\"])\n",
    "    retweet = 0\n",
    "    if \"rt\" in words or \"retweeted_status\" in tweet:\n",
    "        retweet = 1\n",
    "        \n",
    "    has_place = 1 if \"coordinates\" in tweet else 0\n",
    "        \n",
    "    author = tweet[\"user\"]\n",
    "    is_verified = 1 if (\"verified\" in author and author[\"verified\"]) else 0\n",
    "    log_followers = 0\n",
    "    if \"followers_count\" in author and author[\"followers_count\"] > 0:\n",
    "         log_followers = np.log(author[\"followers_count\"])\n",
    "    log_friends = 0\n",
    "    if \"friends_count\" in author and author[\"friends_count\"] > 0:\n",
    "         log_followers = np.log(author[\"friends_count\"])\n",
    "    \n",
    "    features = [num_chars, num_chars_total, num_terms, num_words,\n",
    "                num_unique_terms, sentiment['neg'], sentiment['pos'], \n",
    "                sentiment['neu'], sentiment['compound'],\n",
    "                twitter_objs[2], twitter_objs[1],\n",
    "                twitter_objs[0], retweet, num_media,\n",
    "                is_verified, \n",
    "#                 log_followers, log_friends,\n",
    "#                 has_place,\n",
    "                caps_ratio,\n",
    "               ]\n",
    "\n",
    "    return [round(x, 4) for x in features]\n",
    "\n",
    "other_features_names = [\"num_chars\", \"num_chars_total\", \\\n",
    "                        \"num_terms\", \"num_words\", \"num_unique_words\", \"vader neg\",\"vader pos\",\n",
    "                        \"vader neu\", \"vader compound\", \\\n",
    "                        \"num_hashtags\", \"num_mentions\", \n",
    "                        \"num_urls\", \"is_retweet\", \"num_media\",\n",
    "                        \"is_verified\", \n",
    "#                         \"log_followers\", \"log_friends\",\n",
    "#                         \"has_place\",\n",
    "                        \"caps_ratio\",\n",
    "                       ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "category_to_label = {c:i+1 for i, c in enumerate(tweet_category_map.keys()) if c != \"Irrelevant\"}\n",
    "category_to_label[\"Irrelevant\"] = 0\n",
    "\n",
    "tweet_id_to_category = {}\n",
    "for category, tweet_ids in tweet_category_map.items():\n",
    "    if ( len(tweet_ids) < 5 ):\n",
    "        print(\"Skipping category:\", category)\n",
    "        continue\n",
    "        \n",
    "    for tweet_id in tweet_ids:\n",
    "        tweet_id_to_category[np.int64(tweet_id)] = category_to_label[category]\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tweet_pairs = [(tweet, tweet_id_to_category[tid]) \n",
    "#                for tid, tweet in tweet_id_map.items() if tid in tweet_id_to_category]\n",
    "\n",
    "tweet_pairs = [(tweet_id_map[np.int64(tweet)], category_to_label[category]) \n",
    "               for category, tweet_ids in tweet_category_map.items() for tweet in tweet_ids]\n",
    "\n",
    "tweet_texts = [tp[0][\"text\"] for tp in tweet_pairs]\n",
    "\n",
    "y_data = np.array([tp[1] for tp in tweet_pairs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tweets with Categories: 63346\n"
     ]
    }
   ],
   "source": [
    "print(\"Tweets with Categories:\", y_data.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/base.py:251: UserWarning: Trying to unpickle estimator TfidfTransformer from version 0.19.2 when using version 0.20.0. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n",
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/base.py:251: UserWarning: Trying to unpickle estimator TfidfVectorizer from version 0.19.2 when using version 0.20.0. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n"
     ]
    }
   ],
   "source": [
    "#Construct tfidf matrix and get relevant scores\n",
    "vectorizer = joblib.load(\"../models/2013to2016_tfidf_vectorizer_20190109.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf = vectorizer.transform(tweet_texts).toarray()\n",
    "vocab = {v:i for i, v in enumerate(vectorizer.get_feature_names())}\n",
    "idf_vals = vectorizer.idf_\n",
    "idf_dict = {i:idf_vals[i] for i in vocab.values()} #keys are indices; values are IDF scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "other_ftr_data = np.array([other_features(tweet) for tweet, _ in tweet_pairs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(63346, 98)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sources_ftrs = [tweet_sources_index.get(tweet[\"source\"], 0) for tweet, _ in tweet_pairs]\n",
    "\n",
    "source_ftr_onehot = []\n",
    "for ftr in sources_ftrs:\n",
    "    f = np.zeros(len(tweet_sources_index) + 1)\n",
    "    f[ftr] = 1.0\n",
    "    source_ftr_onehot.append(f)\n",
    "\n",
    "sources_ftr_data = np.array(source_ftr_onehot)    \n",
    "\n",
    "sources_ftr_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(63346, 10114) (63346,)\n"
     ]
    }
   ],
   "source": [
    "X_data = np.concatenate([\n",
    "    tfidf, \n",
    "    other_ftr_data, \n",
    "    sources_ftr_data,\n",
    "], axis=1)\n",
    "\n",
    "print(X_data.shape, y_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "ftr_names_ = np.concatenate([\n",
    "    np.array([x for x in vocab]), \n",
    "    other_features_names, \n",
    "    [\"Unknown\"] + list(tweet_sources_index.keys()),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary Word Count: 10000\n",
      "['🚨', '🚨 🚨', '🚫', '🚮', '🚶', '🤑', '🤔', '🤔 🤔', '🤗', '🤘']\n"
     ]
    }
   ],
   "source": [
    "print(\"Dictionary Word Count:\", len(vocab))\n",
    "print([x[0] for x in vocab.items()][-10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "r_state = 1337"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.04170722897885452\n",
      "F1: 0.03286180635130048\n"
     ]
    }
   ],
   "source": [
    "category_number_list = list(category_to_label.values())\n",
    "d = len(category_number_list)\n",
    "\n",
    "f1_accum = []\n",
    "accuracy_accum = []\n",
    "\n",
    "skf = StratifiedKFold(n_splits=5, random_state=r_state)\n",
    "for train, test in skf.split(X_data, y_data):\n",
    "\n",
    "    # Get actual labels\n",
    "    y_test = y_data[test]\n",
    "\n",
    "    # Compute Precision-Recall \n",
    "    y_infer_local = [category_number_list[np.random.randint(0, d)] for smaple in y_test]\n",
    "    f1_accum.append(f1_score(y_test, y_infer_local, average=\"macro\"))\n",
    "    \n",
    "    accuracy_accum.append(sklearn.metrics.accuracy_score(y_test, y_infer_local))\n",
    "\n",
    "print(\"Accuracy:\", np.mean(accuracy_accum))\n",
    "print(\"F1:\", np.mean(f1_accum))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_params = {\n",
    "    'n_estimators': 128, \n",
    "    \"n_jobs\": -1,\n",
    "    'random_state': r_state,\n",
    "    'class_weight': \"balanced\",\n",
    "    'criterion': 'gini',\n",
    "    'max_depth': 32,\n",
    "    'max_features': 113,\n",
    "    'min_samples_leaf': 2,\n",
    "    'min_samples_split': 54,\n",
    "}\n",
    "\n",
    "nb_params = {\n",
    "    'alpha': 0.01579145221181444,\n",
    "    'binarize': 0.7316900686676242,\n",
    "    'fit_prior': False\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAccuracy: 0.14450047273873307\n",
      "\tF1: 0.1267691676147095\n",
      "\tAccuracy: 0.14992905565189973\n",
      "\tF1: 0.1305630019851192\n",
      "\tAccuracy: 0.15894039735099338\n",
      "\tF1: 0.150074935113354\n",
      "\tAccuracy: 0.15092256741838828\n",
      "\tF1: 0.1328749929130375\n",
      "\tAccuracy: 0.19608771099542516\n",
      "\tF1: 0.14836075142113697\n",
      "\tAccuracy: 0.1670350489422166\n",
      "\tF1: 0.16925834448066449\n",
      "\tAccuracy: 0.16671934260429835\n",
      "\tF1: 0.14916061480819287\n",
      "\tAccuracy: 0.16553359683794466\n",
      "\tF1: 0.15552492118332595\n",
      "\tAccuracy: 0.1075098814229249\n",
      "\tF1: 0.11007156023681368\n",
      "\tAccuracy: 0.13348094259054247\n",
      "\tF1: 0.10624347308560496\n",
      "Accuracy: 0.15406590165533668\n",
      "F1: 0.13789017628419592\n"
     ]
    }
   ],
   "source": [
    "f1_accum = []\n",
    "accuracy_accum = []\n",
    "\n",
    "skf = StratifiedKFold(n_splits=10, random_state=r_state)\n",
    "for train, test in skf.split(X_data, y_data):\n",
    "\n",
    "    X_train = X_data[train]\n",
    "    y_train = y_data[train]\n",
    "    \n",
    "    X_test = X_data[test]\n",
    "    y_test = y_data[test]\n",
    "\n",
    "    # train\n",
    "#     fitted_model = RandomForestClassifier(**rf_params)\n",
    "    fitted_model = BernoulliNB(**nb_params)\n",
    "    fitted_model.fit(X_train, y_train)\n",
    "\n",
    "    # Compute Precision-Recall \n",
    "    y_infer_local = fitted_model.predict(X_test)\n",
    "    local_f1 = f1_score(y_test, y_infer_local, average=\"macro\")\n",
    "    local_score = fitted_model.score(X_test, y_test)\n",
    "    print(\"\\tAccuracy:\", local_score)\n",
    "    print(\"\\tF1:\", local_f1)\n",
    "    \n",
    "    f1_accum.append(local_f1)\n",
    "    accuracy_accum.append(local_score)\n",
    "\n",
    "print(\"Accuracy:\", np.mean(accuracy_accum))\n",
    "print(\"F1:\", np.mean(f1_accum))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label: Advice\n",
      "Score: 0.8429577242446248\n",
      "\t vader neg 0.03439873564865353\n",
      "\t vader compound 0.03320672930789363\n",
      "\t vader pos 0.025977430221543932\n",
      "\t num_words 0.022666039368761142\n",
      "\t num_chars 0.022366630841430427\n",
      "\t num_unique_words 0.021612348301868933\n",
      "\t num_chars_total 0.021036273390178054\n",
      "\t num_terms 0.019911911583429472\n",
      "\t vader neu 0.017514663870457827\n",
      "\t philippines 0.01712507842688468\n",
      "Label: CleanUp\n",
      "Score: 0.9944590029362549\n",
      "\t num_chars 0.03409871488662725\n",
      "\t num_chars_total 0.03244046335624888\n",
      "\t vader neu 0.02667578655181617\n",
      "\t num_terms 0.023532252062691043\n",
      "\t begins 0.023398190446503903\n",
      "\t caps_ratio 0.02232172371782861\n",
      "\t num_unique_words 0.021907634121341904\n",
      "\t vader compound 0.02174514004076544\n",
      "\t num_words 0.020815610852640897\n",
      "\t damage 0.01991325163528457\n",
      "Label: ContextualInformation\n",
      "Score: 0.9395384081078522\n",
      "\t school 0.08053748491842129\n",
      "\t num_hashtags 0.06523070137524266\n",
      "\t shooting 0.0643357336462351\n",
      "\t florida 0.04907706841145317\n",
      "\t num_chars_total 0.03612528163496518\n",
      "\t cruz 0.02656091237300016\n",
      "\t num_chars 0.02580390532851647\n",
      "\t num_terms 0.025751869593128522\n",
      "\t boston 0.02496159392442538\n",
      "\t num_unique_words 0.0244938510681632\n",
      "Label: Discussion\n",
      "Score: 0.8723044864711268\n",
      "\t school 0.04010701930732089\n",
      "\t num_hashtags 0.03749142188065888\n",
      "\t florida 0.029764255305686396\n",
      "\t is_retweet 0.02616159871787724\n",
      "\t caps_ratio 0.02583903139352929\n",
      "\t num_chars_total 0.023304596199886608\n",
      "\t shooting 0.020170329370902008\n",
      "\t num_chars 0.019664969326843074\n",
      "\t philippines 0.018973303549553212\n",
      "\t cruz 0.018870862778236092\n",
      "Label: Donations\n",
      "Score: 0.9418432103053074\n",
      "\t donate 0.06691759134212917\n",
      "\t vader compound 0.05528081687033038\n",
      "\t help 0.0417016872049983\n",
      "\t vader pos 0.03772698912504459\n",
      "\t vader neg 0.030795732065972716\n",
      "\t relief 0.02531582962686345\n",
      "\t caps_ratio 0.019235898891772886\n",
      "\t victims 0.019103910452592823\n",
      "\t vader neu 0.017697310950778627\n",
      "\t num_urls 0.015500020975012994\n",
      "Label: EmergingThreats\n",
      "Score: 0.8889116913459414\n",
      "\t vader pos 0.03668141650801249\n",
      "\t vader compound 0.03263458281534839\n",
      "\t num_chars 0.027145445302722143\n",
      "\t vader neu 0.02467997478124257\n",
      "\t num_terms 0.023947553502714807\n",
      "\t num_chars_total 0.023063006985635383\n",
      "\t num_words 0.022519227112855512\n",
      "\t num_unique_words 0.021837986847255318\n",
      "\t philippines 0.018395298226610957\n",
      "\t caps_ratio 0.016848725866510875\n",
      "Label: Factoid\n",
      "Score: 0.8075963754617498\n",
      "\t vader compound 0.053658957512190526\n",
      "\t vader pos 0.04434856959979368\n",
      "\t earthquake 0.02654115510072828\n",
      "\t dallas 0.02433077399857379\n",
      "\t vader neg 0.024064444641777503\n",
      "\t num_chars 0.02132807396928721\n",
      "\t death 0.019822794048126174\n",
      "\t dead 0.017276504492550884\n",
      "\t num_chars_total 0.017014460264023545\n",
      "\t vader neu 0.01434953812033111\n",
      "Label: FirstPartyObservation\n",
      "Score: 0.7083951630726486\n",
      "\t shooting 0.05253504969935619\n",
      "\t dallas 0.03245291122941435\n",
      "\t num_chars 0.0289923892692037\n",
      "\t vader compound 0.028068490136932783\n",
      "\t num_chars_total 0.02270983978692498\n",
      "\t vader neg 0.022041212436383358\n",
      "\t vader pos 0.021666693830226257\n",
      "\t school 0.018120883820267965\n",
      "\t num_hashtags 0.017572238035234576\n",
      "\t caps_ratio 0.014821774955384809\n",
      "Label: GoodsServices\n",
      "Score: 0.9916174659804882\n",
      "\t food 0.0395495108749394\n",
      "\t need 0.031938879807021835\n",
      "\t num_unique_words 0.030385224540142756\n",
      "\t num_mentions 0.029581379997048456\n",
      "\t vader compound 0.02876046370064534\n",
      "\t help 0.028650117003367084\n",
      "\t num_chars 0.02541484783979291\n",
      "\t num_words 0.02494429428700432\n",
      "\t num_chars_total 0.022524768080911226\n",
      "\t caps_ratio 0.021396880783830608\n",
      "Label: Hashtags\n",
      "Score: 0.643592334164746\n",
      "\t num_hashtags 0.24550408634989992\n",
      "\t num_urls 0.0442500998872848\n",
      "\t philippines 0.025941741567917908\n",
      "\t caps_ratio 0.022203240957798385\n",
      "\t <a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a> 0.01794721508136203\n",
      "\t num_chars 0.017545302731590533\n",
      "\t <a href=\"http://twitterfeed.com\" rel=\"nofollow\">twitterfeed</a> 0.01702331675298828\n",
      "\t num_unique_words 0.01491521446388475\n",
      "\t num_words 0.014341922833722674\n",
      "\t num_terms 0.01280637112893871\n",
      "Label: InformationWanted\n",
      "Score: 0.9860606826003221\n",
      "\t num_urls 0.05044410107070168\n",
      "\t vader compound 0.036332302301317544\n",
      "\t vader neg 0.03546758908713117\n",
      "\t caps_ratio 0.03510856434635179\n",
      "\t vader neu 0.029767955425822137\n",
      "\t num_chars 0.02083571452821964\n",
      "\t num_terms 0.019934856560884683\n",
      "\t vader pos 0.01955478935388537\n",
      "\t num_words 0.018738695748876013\n",
      "\t anyone 0.017890008203821042\n",
      "Label: Location\n",
      "Score: 0.8878540081457392\n",
      "\t num_chars 0.05939345165339428\n",
      "\t num_chars_total 0.05822304221311559\n",
      "\t dallas 0.036931788356258044\n",
      "\t num_terms 0.03676908122879938\n",
      "\t num_words 0.03543552788711724\n",
      "\t num_unique_words 0.030858692339862305\n",
      "\t <a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a> 0.029244471541427057\n",
      "\t vader neu 0.02547949726016315\n",
      "\t philippines 0.020043918942714335\n",
      "\t num_hashtags 0.01865064016786913\n",
      "Label: MovePeople\n",
      "Score: 0.9857765289047453\n",
      "\t num_hashtags 0.04947455312655751\n",
      "\t vader neg 0.03678867182488121\n",
      "\t vader neu 0.03632691210427952\n",
      "\t vader compound 0.03560418400189448\n",
      "\t num_unique_words 0.03074250555145588\n",
      "\t num_terms 0.029948120832929477\n",
      "\t num_chars 0.027326333682295433\n",
      "\t num_words 0.026817302694929223\n",
      "\t num_chars_total 0.02513357161250585\n",
      "\t vader pos 0.022848878008910646\n",
      "Label: MultimediaShare\n",
      "Score: 0.6437344110125344\n",
      "\t num_urls 0.2440300186600243\n",
      "\t num_media 0.0713195046645973\n",
      "\t caps_ratio 0.05129987830063529\n",
      "\t num_chars 0.03493021089975781\n",
      "\t num_chars_total 0.022408460858233916\n",
      "\t num_terms 0.021966531968212535\n",
      "\t num_words 0.016735147768944646\n",
      "\t num_unique_words 0.016651344032644556\n",
      "\t vader pos 0.01298937475193233\n",
      "\t vader neu 0.012114206508009992\n",
      "Label: NewSubEvent\n",
      "Score: 0.9448741830581252\n",
      "\t vader compound 0.04045314409344163\n",
      "\t vader pos 0.03878828538201573\n",
      "\t paris 0.028454757613785957\n",
      "\t num_terms 0.02188091332835844\n",
      "\t num_chars_total 0.021345064057048703\n",
      "\t caps_ratio 0.021249292612770695\n",
      "\t vader neu 0.020790515785249732\n",
      "\t num_chars 0.018135289318547913\n",
      "\t num_unique_words 0.01757212703411098\n",
      "\t num_words 0.017487735466465485\n",
      "Label: News\n",
      "Score: 0.6683610646291794\n",
      "\t num_urls 0.04900994894826495\n",
      "\t caps_ratio 0.028097515580086416\n",
      "\t vader compound 0.027261707477276426\n",
      "\t num_chars 0.026458830280108248\n",
      "\t vader pos 0.025806106336028038\n",
      "\t is_retweet 0.020797570890523857\n",
      "\t num_chars_total 0.019275810910082382\n",
      "\t num_terms 0.01907812008852304\n",
      "\t vader neu 0.018716441220932842\n",
      "\t num_words 0.018226603237194904\n",
      "Label: Official\n",
      "Score: 0.9135857039118492\n",
      "\t vader neu 0.03189243329213767\n",
      "\t is_retweet 0.03177429579518101\n",
      "\t num_terms 0.025790556971263746\n",
      "\t num_chars 0.02569687752772596\n",
      "\t num_chars_total 0.024865685055617013\n",
      "\t num_unique_words 0.02460730915936109\n",
      "\t num_words 0.024464972274209257\n",
      "\t caps_ratio 0.023703946347066858\n",
      "\t shooting 0.022739034879905487\n",
      "\t vader pos 0.022006594536471507\n",
      "Label: OriginalEvent\n",
      "Score: 0.8803712941622202\n",
      "\t earthquake 0.031146087724362156\n",
      "\t num_chars 0.027637645545159656\n",
      "\t num_chars_total 0.026698597764753834\n",
      "\t costa 0.026674401894609744\n",
      "\t vader compound 0.025964629144871586\n",
      "\t num_terms 0.02167433569795526\n",
      "\t vader neg 0.02046353485593347\n",
      "\t num_words 0.01950601662336006\n",
      "\t num_unique_words 0.019095461316983312\n",
      "\t hits 0.017916679998655125\n",
      "Label: SearchAndRescue\n",
      "Score: 0.993132952356897\n",
      "\t missing 0.08366860374851881\n",
      "\t vader neu 0.025652896768573\n",
      "\t vader compound 0.02462696027891622\n",
      "\t caps_ratio 0.02440809348346979\n",
      "\t help 0.02430778959552611\n",
      "\t st 0.02238114159318348\n",
      "\t num_chars 0.021231455423621728\n",
      "\t num_unique_words 0.020726391727541253\n",
      "\t pls 0.01945528294560245\n",
      "\t num_terms 0.018892602714837232\n",
      "Label: Sentiment\n",
      "Score: 0.778833706942822\n",
      "\t num_urls 0.07348525259633583\n",
      "\t vader neu 0.04496284452732154\n",
      "\t num_chars 0.04290637412148797\n",
      "\t vader pos 0.04135784112690718\n",
      "\t vader compound 0.03657652448786107\n",
      "\t num_chars_total 0.036186133013560606\n",
      "\t prayers 0.0316273053130334\n",
      "\t caps_ratio 0.023005316249899294\n",
      "\t num_terms 0.02071309252846479\n",
      "\t num_unique_words 0.020155903042934558\n",
      "Label: ServiceAvailable\n",
      "Score: 0.9050926656773909\n",
      "\t vader neg 0.04742351713798102\n",
      "\t vader compound 0.034310345321160285\n",
      "\t vader pos 0.02628044748083282\n",
      "\t shooting 0.024301139092595948\n",
      "\t caps_ratio 0.018309690457624234\n",
      "\t aid 0.01722355321295595\n",
      "\t num_chars 0.017165250803320667\n",
      "\t help 0.016444456445329915\n",
      "\t philippines 0.01554632373220808\n",
      "\t num_chars_total 0.01494477150926975\n",
      "Label: ThirdPartyObservation\n",
      "Score: 0.7532914469737632\n",
      "\t philippines 0.03531390998198513\n",
      "\t num_chars 0.030980291000062162\n",
      "\t paris 0.028920809398029095\n",
      "\t num_urls 0.026550366321261606\n",
      "\t <a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a> 0.026440512103121795\n",
      "\t caps_ratio 0.025910839066314657\n",
      "\t vader pos 0.023242253841652762\n",
      "\t num_chars_total 0.022403545001875365\n",
      "\t shooting 0.021973147980225437\n",
      "\t vader compound 0.020509378656053925\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label: Volunteer\n",
      "Score: 0.9926120039150065\n",
      "\t help 0.04692925407193221\n",
      "\t vader compound 0.045326814266867375\n",
      "\t vader neg 0.03789196375679748\n",
      "\t vader pos 0.031499479729352096\n",
      "\t need 0.026886772603254484\n",
      "\t caps_ratio 0.02647019835533224\n",
      "\t num_mentions 0.024162957675057443\n",
      "\t num_chars 0.023917752287824313\n",
      "\t num_chars_total 0.02141937111909905\n",
      "\t num_unique_words 0.021395286154220896\n",
      "Label: Weather\n",
      "Score: 0.872272913838285\n",
      "\t philippines 0.0676530813277522\n",
      "\t vader neu 0.03360040963110236\n",
      "\t vader pos 0.0241171448968388\n",
      "\t winds 0.021843531334293228\n",
      "\t num_urls 0.02182602128537618\n",
      "\t vader neg 0.0196630954726181\n",
      "\t is_retweet 0.019174489534670117\n",
      "\t caps_ratio 0.01906215572903833\n",
      "\t vader compound 0.016818294503683655\n",
      "\t shooting 0.016318234905852918\n",
      "Label: Irrelevant\n",
      "Score: 0.8392952988349699\n",
      "\t num_hashtags 0.0591957174204993\n",
      "\t num_urls 0.05590062571065893\n",
      "\t dallas 0.04735509927492537\n",
      "\t hurricane 0.0414714808980506\n",
      "\t caps_ratio 0.028577519073931038\n",
      "\t shooting 0.021828082476056863\n",
      "\t is_verified 0.019141092965914076\n",
      "\t num_chars 0.016600376366854734\n",
      "\t num_terms 0.016508313798497466\n",
      "\t philippines 0.013853088784520363\n"
     ]
    }
   ],
   "source": [
    "label_to_category = {j:i for i, j in category_to_label.items()}\n",
    "for positive_category in category_number_list:\n",
    "    local_y_data = [1 if y == positive_category else 0 for y in y_data]\n",
    "    \n",
    "    fitted_model = RandomForestClassifier(**rf_params)\n",
    "    fitted_model.fit(X_data, local_y_data)\n",
    "    \n",
    "    weights = [(ftr_names_[idx], coef) \n",
    "               for idx, coef in enumerate(fitted_model.feature_importances_)]\n",
    "\n",
    "    tops = sorted(weights, key=lambda x: x[1], reverse=True)[:10]\n",
    "    \n",
    "    print(\"Label:\", label_to_category[positive_category])\n",
    "    print(\"Score:\", fitted_model.score(X_data, local_y_data))\n",
    "    for token, weight in tops:\n",
    "        print(\"\\t\", token, weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def smapper(x):\n",
    "    if ( x < 0.75 ):\n",
    "        return 0\n",
    "    else:\n",
    "        return 1\n",
    "\n",
    "y_data_regress = np.array([smapper(tweet_priority_map[np.int64(tp[0][\"id\"])][\"score\"]) \n",
    "                           for tp in tweet_pairs])\n",
    "y_data_weights = np.array([tweet_priority_map[np.int64(tp[0][\"id\"])][\"weight\"] \n",
    "                           for tp in tweet_pairs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tFitting...\n",
      "\t 0.8737373737373737\n",
      "\tFitting...\n",
      "\t 0.8442234848484849\n",
      "\tFitting...\n",
      "\t 0.8803472770323599\n",
      "\tFitting...\n",
      "\t 0.8871349644830308\n",
      "\tFitting...\n",
      "\t 0.8976949794758446\n",
      "\tFitting...\n",
      "\t 0.8425955162614461\n",
      "\tFitting...\n",
      "\t 0.803283864856331\n",
      "\tFitting...\n",
      "\t 0.8916956109883171\n",
      "\tFitting...\n",
      "\t 0.9051152510262077\n",
      "\tFitting...\n",
      "\t 0.8369119040101042\n",
      "Score (R^2): 0.8662740226719501\n",
      "Accuracy: 0.8662740226719501\n",
      "F1: 0.7321632620234546\n"
     ]
    }
   ],
   "source": [
    "score_accum = []\n",
    "f1_accum = []\n",
    "accuracy_accum = []\n",
    "\n",
    "rf_priority_params = {\n",
    "    'random_state': r_state,\n",
    "    'class_weight': 'balanced',\n",
    "    'n_estimators': 128, \n",
    "    'n_jobs': -1,\n",
    "    'max_depth': 50,\n",
    "    'max_features': 14,\n",
    "    'min_samples_leaf': 33,\n",
    "    'min_samples_split': 96,\n",
    "}\n",
    "\n",
    "nb_priority_params = {\n",
    "    'alpha': 0.05134305647695325,\n",
    "    'binarize': 0.045909955637688404,\n",
    "    'fit_prior': True,\n",
    "}\n",
    "\n",
    "skf = StratifiedKFold(n_splits=10, random_state=r_state)\n",
    "for train, test in skf.split(X_data, y_data_regress):\n",
    "\n",
    "    X_train = X_data[train]\n",
    "    y_train = y_data_regress[train]\n",
    "    y_weight = y_data_weights[train]\n",
    "    \n",
    "    X_test = X_data[test]\n",
    "    y_test = y_data_regress[test]\n",
    "\n",
    "    # train\n",
    "    print(\"\\tFitting...\")\n",
    "#     fitted_model = sklearn.linear_model.LinearRegression(n_jobs=4)\n",
    "#     fitted_model = sklearn.tree.DecisionTreeRegressor(random_state=r_state, max_depth=256)\n",
    "    fitted_model = BernoulliNB(**nb_priority_params)\n",
    "#     fitted_model = RandomForestClassifier(**rf_priority_params)\n",
    "    fitted_model.fit(X_train, y_train, y_weight)\n",
    "\n",
    "    # Compute score metrics\n",
    "    r2_score = fitted_model.score(X_test, y_test)\n",
    "    score_accum.append(r2_score)\n",
    "    \n",
    "    # Compute Precision-Recall \n",
    "    y_infer_local = fitted_model.predict(X_test)\n",
    "    f1_accum.append(f1_score(y_test, y_infer_local, average=\"macro\"))\n",
    "    \n",
    "    accuracy_accum.append(fitted_model.score(X_test, y_test))\n",
    "    \n",
    "    print(\"\\t\", r2_score)\n",
    "\n",
    "print(\"Score (R^2):\", np.mean(score_accum))\n",
    "print(\"Accuracy:\", np.mean(accuracy_accum))\n",
    "print(\"F1:\", np.mean(f1_accum))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_search(X_data, y_data, clf, param_dist, n_iter_search=20, r_state=1337):\n",
    "    # run randomized search\n",
    "    random_search = RandomizedSearchCV(clf, \n",
    "                                       param_distributions=param_dist,\n",
    "                                       n_iter=n_iter_search,\n",
    "                                       cv=10,\n",
    "                                       scoring=\"f1_macro\",\n",
    "                                       random_state=r_state,\n",
    "                                       verbose=2,\n",
    "                                       n_jobs=16,\n",
    "                                      )\n",
    "    \n",
    "    random_search.fit(X_data, y_data)\n",
    "\n",
    "    return (random_search.best_score_, random_search.best_params_)\n",
    "\n",
    "def model_eval_rf(X_data, y_data, n_iter_search=100, r_state=1337):\n",
    "\n",
    "    clf = RandomForestClassifier(\n",
    "        n_estimators=100, class_weight=\"balanced\", random_state=r_state)\n",
    "    \n",
    "    # specify parameters and distributions to sample from\n",
    "    param_dist = {\n",
    "        \"max_depth\": [2, 4, 8, 16, 32, 64, 128, None],\n",
    "        \"max_features\": scipy.stats.randint(1, 512),\n",
    "        \"min_samples_split\": scipy.stats.randint(2, 512),\n",
    "        \"min_samples_leaf\": scipy.stats.randint(2, 512),\n",
    "        \"criterion\": [\"gini\", \"entropy\"],\n",
    "    }\n",
    "    \n",
    "    return random_search(X_data, y_data, clf, param_dist, n_iter_search=n_iter_search, r_state=r_state)\n",
    "\n",
    "def model_eval_nb(X_data, y_data, n_iter_search=100, r_state=1337):\n",
    "\n",
    "    clf = BernoulliNB()\n",
    "    \n",
    "    # specify parameters and distributions to sample from\n",
    "    param_dist = {\n",
    "        \"alpha\": scipy.stats.uniform(),\n",
    "        \"binarize\": scipy.stats.uniform(),\n",
    "        \"fit_prior\": [True, False],\n",
    "    }\n",
    "    \n",
    "    return random_search(X_data, y_data, clf, param_dist, n_iter_search=n_iter_search, r_state=r_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# search_results = model_eval_nb(X_data, y_data_regress, n_iter_search=128)\n",
    "# search_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# search_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15000, 10114)\n"
     ]
    }
   ],
   "source": [
    "test_tweets = []\n",
    "with open(\"../data/2019b-testing.json\", \"r\") as in_file:\n",
    "    for line in in_file:\n",
    "        tweet_top = json.loads(line)\n",
    "        tweet = json.loads(tweet_top[\"allProperties\"][\"srcjson\"])\n",
    "        test_tweets.append(tweet)\n",
    "        \n",
    "X_test_tfidf = vectorizer.transform([t[\"text\"] for t in test_tweets]).toarray()\n",
    "X_test_other = np.array([other_features(tweet) for tweet in test_tweets])\n",
    "\n",
    "sources_ftrs = [tweet_sources_index.get(tweet[\"source\"], 0) for tweet in test_tweets]\n",
    "source_ftr_onehot = []\n",
    "for ftr in sources_ftrs:\n",
    "    f = np.zeros(len(tweet_sources_index) + 1)\n",
    "    f[ftr] = 1.0\n",
    "    source_ftr_onehot.append(f)\n",
    "X_test_sources = np.array(source_ftr_onehot)    \n",
    "\n",
    "X_test_data = np.concatenate([\n",
    "    X_test_tfidf, \n",
    "    X_test_other, \n",
    "    X_test_sources,\n",
    "], axis=1)\n",
    "\n",
    "print(X_test_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BernoulliNB(alpha=0.01579145221181444, binarize=0.7316900686676242,\n",
       "      class_prior=None, fit_prior=False)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fitted_model = BernoulliNB(**nb_params)\n",
    "fitted_model.fit(X_data, y_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_labels = fitted_model.predict(X_test_data)\n",
    "\n",
    "labeled_test_data = list(zip([t[\"id\"] for t in test_tweets], y_test_labels))\n",
    "\n",
    "id_to_cat_map = {y:x for x,y in category_to_label.items()}\n",
    "\n",
    "df = pd.DataFrame([{\"tweet_id\":tup[0], \"label\": id_to_cat_map[tup[1]]} for tup in labeled_test_data])\n",
    "\n",
    "df.to_csv(\"trec2019b_test_results_run_baseline.csv\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Advice</th>\n",
       "      <td>363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CleanUp</th>\n",
       "      <td>257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ContextualInformation</th>\n",
       "      <td>1913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Discussion</th>\n",
       "      <td>802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Donations</th>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EmergingThreats</th>\n",
       "      <td>443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Factoid</th>\n",
       "      <td>1603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FirstPartyObservation</th>\n",
       "      <td>431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GoodsServices</th>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hashtags</th>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>InformationWanted</th>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Irrelevant</th>\n",
       "      <td>1165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Location</th>\n",
       "      <td>482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MovePeople</th>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MultimediaShare</th>\n",
       "      <td>648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NewSubEvent</th>\n",
       "      <td>196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>News</th>\n",
       "      <td>743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Official</th>\n",
       "      <td>418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>OriginalEvent</th>\n",
       "      <td>1642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SearchAndRescue</th>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sentiment</th>\n",
       "      <td>1360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ServiceAvailable</th>\n",
       "      <td>184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ThirdPartyObservation</th>\n",
       "      <td>520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Volunteer</th>\n",
       "      <td>92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Weather</th>\n",
       "      <td>710</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       tweet_id\n",
       "label                          \n",
       "Advice                      363\n",
       "CleanUp                     257\n",
       "ContextualInformation      1913\n",
       "Discussion                  802\n",
       "Donations                   137\n",
       "EmergingThreats             443\n",
       "Factoid                    1603\n",
       "FirstPartyObservation       431\n",
       "GoodsServices                70\n",
       "Hashtags                    554\n",
       "InformationWanted           106\n",
       "Irrelevant                 1165\n",
       "Location                    482\n",
       "MovePeople                  128\n",
       "MultimediaShare             648\n",
       "NewSubEvent                 196\n",
       "News                        743\n",
       "Official                    418\n",
       "OriginalEvent              1642\n",
       "SearchAndRescue              33\n",
       "Sentiment                  1360\n",
       "ServiceAvailable            184\n",
       "ThirdPartyObservation       520\n",
       "Volunteer                    92\n",
       "Weather                     710"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df.groupby(\"label\").count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1122269086338232321    25\n",
       "1122237890539421697    23\n",
       "1122258843927445504    19\n",
       "1122224999845249025    19\n",
       "1122242433150885888    14\n",
       "1125943643754377216    12\n",
       "1126163665248350210    12\n",
       "1125908981757743105    12\n",
       "1125868443004227584    12\n",
       "1122228733144788992    11\n",
       "1125894485479702528    11\n",
       "1125861997780852738    11\n",
       "1126164160624992256    11\n",
       "1122228151361232896    10\n",
       "1122236759406981121    10\n",
       "1122229347119710208    10\n",
       "1122250898154184704    10\n",
       "1125921493337169920    10\n",
       "1126107056778252294    10\n",
       "1122227608874307585     9\n",
       "1125949590077956100     9\n",
       "1122236467651194883     9\n",
       "1122221393888735232     9\n",
       "1122237648477786112     9\n",
       "1122257063336398848     9\n",
       "1125919362785792000     9\n",
       "1122250185776746498     9\n",
       "1122244508702216192     9\n",
       "1122270464330485761     9\n",
       "1122247870378315776     8\n",
       "                       ..\n",
       "1134427602758725632     1\n",
       "1122500084720181248     1\n",
       "1134844467444191232     1\n",
       "1120301644887904257     1\n",
       "1134428183493668864     1\n",
       "1120635938219028480     1\n",
       "1134463613253513218     1\n",
       "1122237176790618119     1\n",
       "1132264726673723393     1\n",
       "1122254722981871618     1\n",
       "1122112200049664000     1\n",
       "1122923224113799168     1\n",
       "1120521480444547073     1\n",
       "1134271762441674752     1\n",
       "1126124947032514562     1\n",
       "1122272224214306817     1\n",
       "1134182128659390464     1\n",
       "1120565756830339072     1\n",
       "1131085706128973824     1\n",
       "1120321948628578304     1\n",
       "1120319462199054338     1\n",
       "1122324944883474434     1\n",
       "1120565701637394432     1\n",
       "1121744361409134593     1\n",
       "1120336941575065603     1\n",
       "1125878491076079618     1\n",
       "1120565623816278016     1\n",
       "1134498625382256640     1\n",
       "1134506731420045312     1\n",
       "1133746032058101760     1\n",
       "Name: tweet_id, Length: 13916, dtype: int64"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"tweet_id\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight='balanced',\n",
       "            criterion='gini', max_depth=50, max_features=14,\n",
       "            max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "            min_impurity_split=None, min_samples_leaf=33,\n",
       "            min_samples_split=96, min_weight_fraction_leaf=0.0,\n",
       "            n_estimators=128, n_jobs=-1, oob_score=False,\n",
       "            random_state=1337, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fitted_pri_model = BernoulliNB(**nb_params)\n",
    "fitted_pri_model = RandomForestClassifier(**rf_priority_params)\n",
    "fitted_pri_model.fit(X_data, y_data_regress)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_labels = fitted_pri_model.predict(X_test_data)\n",
    "\n",
    "labeled_test_data = list(zip([t[\"id\"] for t in test_tweets], y_test_labels))\n",
    "\n",
    "df = pd.DataFrame([{\"tweet_id\":tup[0], \"priority\": tup[1]} for tup in labeled_test_data])\n",
    "\n",
    "df.to_csv(\"trec2019b_test_results_priority_run_baseline.csv\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    11718\n",
       "1     3282\n",
       "Name: priority, dtype: int64"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"priority\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test with Multiple Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_pairs = []\n",
    "y_data = []\n",
    "\n",
    "indexer = {c:i for i, c in enumerate(list(category_to_label.keys()))}\n",
    "indexer_inv = {i:c for c,i in indexer.items()}\n",
    "for tweet_id, categories in category_df.groupby(\"tweet_id\"):\n",
    "    \n",
    "    tup = (\n",
    "        tweet_id_map[np.int64(tweet_id)], \n",
    "        [indexer[category] for category in categories[\"category\"] if category != 'Unknown']\n",
    "    )\n",
    "    tweet_pairs.append(tup)\n",
    "\n",
    "tweet_texts = [tp[0][\"text\"] for tp in tweet_pairs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_data_ = [tp[1] for tp in tweet_pairs]\n",
    "\n",
    "def one_hot_y(y_list):\n",
    "    encoded = [0] * len(indexer)\n",
    "    for y in y_list:\n",
    "        encoded[y] = 1\n",
    "    return encoded\n",
    "\n",
    "y_data = np.array([one_hot_y(y) for y in y_data_])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf = vectorizer.transform(tweet_texts).toarray()\n",
    "vocab = {v:i for i, v in enumerate(vectorizer.get_feature_names())}\n",
    "idf_vals = vectorizer.idf_\n",
    "idf_dict = {i:idf_vals[i] for i in vocab.values()} #keys are indices; values are IDF scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(26144, 16)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "other_ftr_data = np.array([other_features(tweet) for tweet, _ in tweet_pairs])\n",
    "other_ftr_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "sources_ftrs = [tweet_sources_index.get(tweet[\"source\"], 0) for tweet, _ in tweet_pairs]\n",
    "source_ftr_onehot = []\n",
    "for ftr in sources_ftrs:\n",
    "    f = np.zeros(len(tweet_sources_index) + 1)\n",
    "    f[ftr] = 1.0\n",
    "    source_ftr_onehot.append(f)\n",
    "X_test_sources = np.array(source_ftr_onehot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(26144, 10114) (26144, 25)\n"
     ]
    }
   ],
   "source": [
    "X_data = np.concatenate([\n",
    "    tfidf, \n",
    "    other_ftr_data, \n",
    "    X_test_sources,\n",
    "], axis=1)\n",
    "\n",
    "ftr_names_ = np.concatenate([\n",
    "    np.array([x for x in vocab]), \n",
    "    other_features_names, \n",
    "    [\"Unknown\"] + list(tweet_sources_index.keys()),\n",
    "])\n",
    "\n",
    "print(X_data.shape, y_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAccuracy: 0.015296367112810707\n",
      "\tF1: 0.38791710192607903\n",
      "\tAccuracy: 0.0130019120458891\n",
      "\tF1: 0.39642833057968724\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAccuracy: 0.012619502868068833\n",
      "\tF1: 0.5142703431992867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAccuracy: 0.021797323135755258\n",
      "\tF1: 0.5433356525055141\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAccuracy: 0.010711553175210406\n",
      "\tF1: 0.3919908923692285\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAccuracy: 0.029839326702371844\n",
      "\tF1: 0.40033223606947715\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAccuracy: 0.05547054322876817\n",
      "\tF1: 0.44991348915080664\n",
      "\tAccuracy: 0.01874521805661821\n",
      "\tF1: 0.46942190245055354\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clb617/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tAccuracy: 0.0279265493496557\n",
      "\tF1: 0.4548733602034812\n",
      "\tAccuracy: 0.027161438408569244\n",
      "\tF1: 0.3634248923787077\n",
      "Accuracy: 0.3043774280138184\n",
      "F1: 0.5344177990912915\n"
     ]
    }
   ],
   "source": [
    "skf = KFold(n_splits=10, random_state=r_state)\n",
    "for train, test in skf.split(X_data, y_data):\n",
    "\n",
    "    X_train = X_data[train]    \n",
    "    X_test = X_data[test]\n",
    "    \n",
    "    y_train = y_data[train]\n",
    "    y_test = y_data[test]\n",
    "\n",
    "\n",
    "    # train\n",
    "    fitted_model = sklearn.multiclass.OneVsRestClassifier(\n",
    "#         RandomForestClassifier(**rf_params)\n",
    "        BernoulliNB(**nb_params)\n",
    "    )\n",
    "    fitted_model.fit(X_train, y_train)\n",
    "\n",
    "    # Compute Precision-Recall \n",
    "    y_infer_local = fitted_model.predict(X_test)\n",
    "    local_f1 = f1_score(y_test, y_infer_local, average=\"weighted\")\n",
    "    local_score = fitted_model.score(X_test, y_test)\n",
    "    print(\"\\tAccuracy:\", local_score)\n",
    "    print(\"\\tF1:\", local_f1)\n",
    "    \n",
    "    f1_accum.append(local_f1)\n",
    "    accuracy_accum.append(local_score)\n",
    "\n",
    "print(\"Accuracy:\", np.mean(accuracy_accum))\n",
    "print(\"F1:\", np.mean(f1_accum))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 1, ..., 0, 0, 0],\n",
       "       [0, 0, 1, ..., 0, 0, 0],\n",
       "       [0, 0, 1, ..., 0, 0, 1],\n",
       "       ...,\n",
       "       [1, 0, 0, ..., 0, 0, 0],\n",
       "       [1, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 1]])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prs = fitted_model.predict(X_test)\n",
    "prs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Advice 695\n",
      "1 CleanUp 161\n",
      "2 ContextualInformation 266\n",
      "3 Discussion 416\n",
      "4 Donations 178\n",
      "5 EmergingThreats 373\n",
      "6 Factoid 346\n",
      "7 FirstPartyObservation 1066\n",
      "8 GoodsServices 139\n",
      "9 Hashtags 995\n",
      "10 InformationWanted 256\n",
      "11 Location 178\n",
      "12 MovePeople 201\n",
      "13 MultimediaShare 467\n",
      "14 NewSubEvent 261\n",
      "15 News 724\n",
      "16 Official 284\n",
      "17 OriginalEvent 196\n",
      "18 SearchAndRescue 95\n",
      "19 Sentiment 925\n",
      "20 ServiceAvailable 219\n",
      "21 ThirdPartyObservation 481\n",
      "22 Volunteer 156\n",
      "23 Weather 344\n",
      "24 Irrelevant 848\n"
     ]
    }
   ],
   "source": [
    "for i, x in enumerate(prs.sum(axis=0)):\n",
    "    print(i, indexer_inv[i], x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = sklearn.multiclass.OneVsRestClassifier(\n",
    "#         RandomForestClassifier(**rf_params)\n",
    "    BernoulliNB(**nb_params)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OneVsRestClassifier(estimator=BernoulliNB(alpha=0.01579145221181444, binarize=0.7316900686676242,\n",
       "      class_prior=None, fit_prior=False),\n",
       "          n_jobs=None)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_data, y_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_labels = clf.predict(X_test_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_rows = []\n",
    "for tid, row_labels in zip([t[\"id\"] for t in test_tweets], y_test_labels):\n",
    "    \n",
    "    row = [tid] + row_labels.tolist()\n",
    "    all_rows.append(row)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "id_to_cat_map = {y:x for x,y in category_to_label.items()}\n",
    "\n",
    "df = pd.DataFrame(all_rows, columns=[\"tweet_id\"] + [indexer_inv[i] for i in range(len(indexer_inv))])\n",
    "\n",
    "df.to_csv(\"trec2019b_test_results_run_baseline_multi.csv\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}