Classical Models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T03:54:41.621995Z",
     "start_time": "2020-10-11T03:54:39.381509Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import nltk\n",
    "import sklearn\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm_notebook\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T03:54:42.677010Z",
     "start_time": "2020-10-11T03:54:42.181601Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>One of the other reviewers has mentioned that ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>A wonderful little production. &lt;br /&gt;&lt;br /&gt;The...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>I thought this was a wonderful way to spend ti...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Basically there's a family where a little boy ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Petter Mattei's \"Love in the Time of Money\" is...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>Probably my all-time favorite movie, a story o...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>I sure would like to see a resurrection of a u...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>This show was an amazing, fresh &amp; innovative i...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>Encouraged by the positive comments about this...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>If you like original gut wrenching laughter yo...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review sentiment\n",
       "0  One of the other reviewers has mentioned that ...  positive\n",
       "1  A wonderful little production. <br /><br />The...  positive\n",
       "2  I thought this was a wonderful way to spend ti...  positive\n",
       "3  Basically there's a family where a little boy ...  negative\n",
       "4  Petter Mattei's \"Love in the Time of Money\" is...  positive\n",
       "5  Probably my all-time favorite movie, a story o...  positive\n",
       "6  I sure would like to see a resurrection of a u...  positive\n",
       "7  This show was an amazing, fresh & innovative i...  negative\n",
       "8  Encouraged by the positive comments about this...  negative\n",
       "9  If you like original gut wrenching laughter yo...  positive"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"data/IMDB Dataset.csv\")\n",
    "print (data.shape)\n",
    "data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T03:54:42.716125Z",
     "start_time": "2020-10-11T03:54:42.702979Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "review       0\n",
       "sentiment    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T03:54:43.339863Z",
     "start_time": "2020-10-11T03:54:43.330405Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "negative    25000\n",
       "positive    25000\n",
       "Name: sentiment, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.sentiment.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T03:54:43.927132Z",
     "start_time": "2020-10-11T03:54:43.920752Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def remove_html(text):\n",
    "    bs = BeautifulSoup(text, \"html.parser\")\n",
    "    return ' ' + bs.get_text() + ' '\n",
    "\n",
    "def keep_only_letters(text):\n",
    "    text=re.sub(r'[^a-zA-Z\\s]','',text)\n",
    "    return text\n",
    "\n",
    "def convert_to_lowercase(text):\n",
    "    return text.lower()\n",
    "\n",
    "def clean_reviews(text):\n",
    "    text = remove_html(text)\n",
    "    text = keep_only_letters(text)\n",
    "    text = convert_to_lowercase(text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T03:54:52.100857Z",
     "start_time": "2020-10-11T03:54:44.439463Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data['review'] = data['review'].apply(lambda review: clean_reviews(review))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T03:54:52.515390Z",
     "start_time": "2020-10-11T03:54:52.506276Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "179\n",
      "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his']\n"
     ]
    }
   ],
   "source": [
    "english_stop_words = nltk.corpus.stopwords.words('english')\n",
    "print(len(english_stop_words))\n",
    "print (english_stop_words[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T03:55:05.705351Z",
     "start_time": "2020-10-11T03:54:53.250824Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def remove_stop_words(text):\n",
    "    for stopword in english_stop_words:\n",
    "        stopword = ' ' + stopword + ' '\n",
    "        text = text.replace(stopword, ' ')\n",
    "    return text\n",
    "\n",
    "data['review'] = data['review'].apply(remove_stop_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T03:56:47.411285Z",
     "start_time": "2020-10-11T03:55:05.706672Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def text_stemming(text):\n",
    "    stemmer = nltk.porter.PorterStemmer()\n",
    "    stemmed = ' '.join([stemmer.stem(token) for token in text.split()])\n",
    "    return stemmed\n",
    "\n",
    "data['review'] = data['review'].apply(text_stemming)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T03:56:47.422554Z",
     "start_time": "2020-10-11T03:56:47.412911Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>one review mention watch oz episod youll hook ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>wonder littl product film techniqu unassum old...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>thought wonder way spend time hot summer weeke...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>basic there famili littl boy jake think there ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>petter mattei love time money visual stun film...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>probabl alltim favorit movi stori selfless sac...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>sure would like see resurrect date seahunt ser...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>show amaz fresh innov idea first air first yea...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>encourag posit comment film look forward watch...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>like origin gut wrench laughter like movi youn...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review sentiment\n",
       "0  one review mention watch oz episod youll hook ...  positive\n",
       "1  wonder littl product film techniqu unassum old...  positive\n",
       "2  thought wonder way spend time hot summer weeke...  positive\n",
       "3  basic there famili littl boy jake think there ...  negative\n",
       "4  petter mattei love time money visual stun film...  positive\n",
       "5  probabl alltim favorit movi stori selfless sac...  positive\n",
       "6  sure would like see resurrect date seahunt ser...  positive\n",
       "7  show amaz fresh innov idea first air first yea...  negative\n",
       "8  encourag posit comment film look forward watch...  negative\n",
       "9  like origin gut wrench laughter like movi youn...  positive"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Splitting into Training and Test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T03:56:47.430683Z",
     "start_time": "2020-10-11T03:56:47.425854Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "imdb_train = data[:40000]\n",
    "imdb_test = data[40000:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Term Frequency Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T03:56:51.675412Z",
     "start_time": "2020-10-11T03:56:47.433162Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000, 150374) (10000, 150374)\n"
     ]
    }
   ],
   "source": [
    "vectorizer = sklearn.feature_extraction.text.CountVectorizer(binary=False,ngram_range=(1,1))\n",
    "tf_features_train = vectorizer.fit_transform(imdb_train['review'])\n",
    "tf_features_test = vectorizer.transform(imdb_test['review'])\n",
    "print (tf_features_train.shape, tf_features_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T03:56:51.685042Z",
     "start_time": "2020-10-11T03:56:51.676513Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000 10000\n"
     ]
    }
   ],
   "source": [
    "train_labels = [1 if sentiment=='positive' else 0 for sentiment in imdb_train['sentiment']]\n",
    "test_labels = [1 if sentiment=='positive' else 0 for sentiment in imdb_test['sentiment']]\n",
    "print (len(train_labels), len(test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T03:57:01.298186Z",
     "start_time": "2020-10-11T03:56:51.686313Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/k15/anaconda/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
      "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
      "                   random_state=None, solver='warn', tol=0.0001, verbose=0,\n",
      "                   warm_start=False)\n"
     ]
    }
   ],
   "source": [
    "clf = sklearn.linear_model.LogisticRegression()\n",
    "clf.fit(tf_features_train, train_labels)\n",
    "print (clf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-09T11:08:35.265441Z",
     "start_time": "2020-10-09T11:08:35.234232Z"
    },
    "collapsed": true
   },
   "source": [
    "## Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-10T14:07:12.496404Z",
     "start_time": "2020-10-10T14:07:12.448807Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "    Negative       0.88      0.88      0.88      4993\n",
      "    Positive       0.88      0.88      0.88      5007\n",
      "\n",
      "    accuracy                           0.88     10000\n",
      "   macro avg       0.88      0.88      0.88     10000\n",
      "weighted avg       0.88      0.88      0.88     10000\n",
      "\n",
      "[[4398  595]\n",
      " [ 581 4426]]\n"
     ]
    }
   ],
   "source": [
    "predictions = clf.predict(tf_features_test)\n",
    "print(sklearn.metrics.classification_report(test_labels, predictions, target_names=['Negative', 'Positive']))\n",
    "print(sklearn.metrics.confusion_matrix(test_labels, predictions, labels=[0, 1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bigram features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-10T15:53:40.251033Z",
     "start_time": "2020-10-10T15:52:42.451135Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000, 2494028) (10000, 2494028)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/k15/anaconda/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "    Negative       0.90      0.89      0.90      4993\n",
      "    Positive       0.89      0.90      0.90      5007\n",
      "\n",
      "    accuracy                           0.90     10000\n",
      "   macro avg       0.90      0.90      0.90     10000\n",
      "weighted avg       0.90      0.90      0.90     10000\n",
      "\n",
      "[[4457  536]\n",
      " [ 496 4511]]\n"
     ]
    }
   ],
   "source": [
    "vectorizer = sklearn.feature_extraction.text.CountVectorizer(binary=False,ngram_range=(1,2))\n",
    "tf_features_train = vectorizer.fit_transform(imdb_train['review'])\n",
    "tf_features_test = vectorizer.transform(imdb_test['review'])\n",
    "print (tf_features_train.shape, tf_features_test.shape)\n",
    "\n",
    "clf = sklearn.linear_model.LogisticRegression()\n",
    "clf.fit(tf_features_train, train_labels)\n",
    "\n",
    "predictions = clf.predict(tf_features_test)\n",
    "print(sklearn.metrics.classification_report(test_labels, predictions, target_names=['Negative', 'Positive']))\n",
    "print(sklearn.metrics.confusion_matrix(test_labels, predictions, labels=[0, 1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-10T15:56:58.982204Z",
     "start_time": "2020-10-10T15:55:07.408422Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000, 6802553) (10000, 6802553)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/k15/anaconda/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "    Negative       0.90      0.89      0.90      4993\n",
      "    Positive       0.89      0.90      0.90      5007\n",
      "\n",
      "    accuracy                           0.90     10000\n",
      "   macro avg       0.90      0.90      0.90     10000\n",
      "weighted avg       0.90      0.90      0.90     10000\n",
      "\n",
      "[[4452  541]\n",
      " [ 500 4507]]\n"
     ]
    }
   ],
   "source": [
    "vectorizer = sklearn.feature_extraction.text.CountVectorizer(binary=False,ngram_range=(1,3))\n",
    "tf_features_train = vectorizer.fit_transform(imdb_train['review'])\n",
    "tf_features_test = vectorizer.transform(imdb_test['review'])\n",
    "print (tf_features_train.shape, tf_features_test.shape)\n",
    "\n",
    "clf = sklearn.linear_model.LogisticRegression()\n",
    "clf.fit(tf_features_train, train_labels)\n",
    "\n",
    "predictions = clf.predict(tf_features_test)\n",
    "print(sklearn.metrics.classification_report(test_labels, predictions, target_names=['Negative', 'Positive']))\n",
    "print(sklearn.metrics.confusion_matrix(test_labels, predictions, labels=[0, 1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MultiNomial Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T04:01:53.720192Z",
     "start_time": "2020-10-11T04:01:53.715349Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T04:09:13.361565Z",
     "start_time": "2020-10-11T04:08:20.262114Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000, 6802553) (10000, 6802553)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    Negative       0.88      0.89      0.89      4993\n",
      "    Positive       0.89      0.88      0.88      5007\n",
      "\n",
      "    accuracy                           0.88     10000\n",
      "   macro avg       0.88      0.88      0.88     10000\n",
      "weighted avg       0.88      0.88      0.88     10000\n",
      "\n",
      "[[4456  537]\n",
      " [ 614 4393]]\n"
     ]
    }
   ],
   "source": [
    "vectorizer = sklearn.feature_extraction.text.CountVectorizer(binary=False,ngram_range=(1,3))\n",
    "tf_features_train = vectorizer.fit_transform(imdb_train['review'])\n",
    "tf_features_test = vectorizer.transform(imdb_test['review'])\n",
    "print (tf_features_train.shape, tf_features_test.shape)\n",
    "\n",
    "clf = MultinomialNB()\n",
    "clf.fit(tf_features_train, train_labels)\n",
    "\n",
    "predictions = clf.predict(tf_features_test)\n",
    "print(sklearn.metrics.classification_report(test_labels, predictions, target_names=['Negative', 'Positive']))\n",
    "print(sklearn.metrics.confusion_matrix(test_labels, predictions, labels=[0, 1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-10T15:56:58.987273Z",
     "start_time": "2020-10-10T15:56:58.983617Z"
    }
   },
   "source": [
    "# LSVM "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T04:14:31.561141Z",
     "start_time": "2020-10-11T04:12:45.787162Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000, 6802553) (10000, 6802553)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    Negative       0.90      0.89      0.89      4993\n",
      "    Positive       0.89      0.90      0.90      5007\n",
      "\n",
      "    accuracy                           0.90     10000\n",
      "   macro avg       0.90      0.90      0.90     10000\n",
      "weighted avg       0.90      0.90      0.90     10000\n",
      "\n",
      "[[4459  534]\n",
      " [ 513 4494]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/k15/anaconda/lib/python3.6/site-packages/sklearn/svm/base.py:929: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "vectorizer = sklearn.feature_extraction.text.CountVectorizer(binary=False,ngram_range=(1,3))\n",
    "tf_features_train = vectorizer.fit_transform(imdb_train['review'])\n",
    "tf_features_test = vectorizer.transform(imdb_test['review'])\n",
    "print (tf_features_train.shape, tf_features_test.shape)\n",
    "\n",
    "clf = sklearn.svm.LinearSVC()\n",
    "clf.fit(tf_features_train, train_labels)\n",
    "\n",
    "predictions = clf.predict(tf_features_test)\n",
    "print(sklearn.metrics.classification_report(test_labels, predictions, target_names=['Negative', 'Positive']))\n",
    "print(sklearn.metrics.confusion_matrix(test_labels, predictions, labels=[0, 1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TF-IDF feature Creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T06:17:05.052075Z",
     "start_time": "2020-10-11T06:17:00.494695Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000, 150374) (10000, 150374)\n"
     ]
    }
   ],
   "source": [
    "vectorizer = sklearn.feature_extraction.text.TfidfVectorizer(use_idf=True,ngram_range=(1,1))\n",
    "tfidf_features_train = vectorizer.fit_transform(imdb_train['review'])\n",
    "tfidf_features_test = vectorizer.transform(imdb_test['review'])\n",
    "print (tfidf_features_train.shape, tfidf_features_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# On TFIDF features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T06:23:56.999984Z",
     "start_time": "2020-10-11T06:23:01.813206Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000, 6652179) (10000, 6652179)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/k15/anaconda/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "    Negative       0.86      0.85      0.86      4993\n",
      "    Positive       0.85      0.87      0.86      5007\n",
      "\n",
      "    accuracy                           0.86     10000\n",
      "   macro avg       0.86      0.86      0.86     10000\n",
      "weighted avg       0.86      0.86      0.86     10000\n",
      "\n",
      "[[4247  746]\n",
      " [ 666 4341]]\n"
     ]
    }
   ],
   "source": [
    "#Create features\n",
    "vectorizer = sklearn.feature_extraction.text.TfidfVectorizer(use_idf=True,ngram_range=(2,3))\n",
    "tfidf_features_train = vectorizer.fit_transform(imdb_train['review'])\n",
    "tfidf_features_test = vectorizer.transform(imdb_test['review'])\n",
    "print (tfidf_features_train.shape, tfidf_features_test.shape)\n",
    "\n",
    "#train model\n",
    "clf = sklearn.linear_model.LogisticRegression()\n",
    "clf.fit(tfidf_features_train, train_labels)\n",
    "\n",
    "#evaluation\n",
    "predictions = clf.predict(tfidf_features_test)\n",
    "print(sklearn.metrics.classification_report(test_labels, predictions, target_names=['Negative', 'Positive']))\n",
    "print(sklearn.metrics.confusion_matrix(test_labels, predictions, labels=[0, 1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T06:28:05.145071Z",
     "start_time": "2020-10-11T06:27:02.745214Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000, 6802553) (10000, 6802553)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    Negative       0.90      0.89      0.90      4993\n",
      "    Positive       0.89      0.91      0.90      5007\n",
      "\n",
      "    accuracy                           0.90     10000\n",
      "   macro avg       0.90      0.90      0.90     10000\n",
      "weighted avg       0.90      0.90      0.90     10000\n",
      "\n",
      "[[4444  549]\n",
      " [ 468 4539]]\n"
     ]
    }
   ],
   "source": [
    "#Create features\n",
    "vectorizer = sklearn.feature_extraction.text.TfidfVectorizer(use_idf=True,ngram_range=(1,3))\n",
    "tfidf_features_train = vectorizer.fit_transform(imdb_train['review'])\n",
    "tfidf_features_test = vectorizer.transform(imdb_test['review'])\n",
    "print (tfidf_features_train.shape, tfidf_features_test.shape)\n",
    "\n",
    "#train model\n",
    "clf = sklearn.svm.LinearSVC()\n",
    "clf.fit(tfidf_features_train, train_labels)\n",
    "\n",
    "#evaluation\n",
    "predictions = clf.predict(tfidf_features_test)\n",
    "print(sklearn.metrics.classification_report(test_labels, predictions, target_names=['Negative', 'Positive']))\n",
    "print(sklearn.metrics.confusion_matrix(test_labels, predictions, labels=[0, 1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-11T06:33:27.393551Z",
     "start_time": "2020-10-11T06:32:29.098463Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000, 6802553) (10000, 6802553)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    Negative       0.88      0.89      0.89      4993\n",
      "    Positive       0.89      0.88      0.89      5007\n",
      "\n",
      "    accuracy                           0.89     10000\n",
      "   macro avg       0.89      0.89      0.89     10000\n",
      "weighted avg       0.89      0.89      0.89     10000\n",
      "\n",
      "[[4464  529]\n",
      " [ 604 4403]]\n"
     ]
    }
   ],
   "source": [
    "#Create features\n",
    "vectorizer = sklearn.feature_extraction.text.TfidfVectorizer(use_idf=True,ngram_range=(1,3))\n",
    "tfidf_features_train = vectorizer.fit_transform(imdb_train['review'])\n",
    "tfidf_features_test = vectorizer.transform(imdb_test['review'])\n",
    "print (tfidf_features_train.shape, tfidf_features_test.shape)\n",
    "\n",
    "#train model\n",
    "clf = sklearn.naive_bayes.MultinomialNB()\n",
    "clf.fit(tfidf_features_train, train_labels)\n",
    "\n",
    "#evaluation\n",
    "predictions = clf.predict(tfidf_features_test)\n",
    "print(sklearn.metrics.classification_report(test_labels, predictions, target_names=['Negative', 'Positive']))\n",
    "print(sklearn.metrics.confusion_matrix(test_labels, predictions, labels=[0, 1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}