Text_Classification(7_3).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Importing the Libraries\n",
    "import pandas as pd\n",
    "dataset=pd.read_csv(\"E:/final_dataset_basicmlmodel.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>@user when a father is dysfunctional and is s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>@user @user thanks for #lyft credit i can't us...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>bihday your majesty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>#model   i love u take with u all the time in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>factsguide: society now    #motivation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  label                                              tweet\n",
       "0   1      0   @user when a father is dysfunctional and is s...\n",
       "1   2      0  @user @user thanks for #lyft credit i can't us...\n",
       "2   3      0                                bihday your majesty\n",
       "3   4      0  #model   i love u take with u all the time in ...\n",
       "4   5      0             factsguide: society now    #motivation"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 .  â #ireland consumer price index (mom) climbed from previous 0.2% to 0.5% in may   #blog #silver #gold #forex\n",
      "2 . we are so selfish. #orlando #standwithorlando #pulseshooting #orlandoshooting #biggerproblems #selfish #heabreaking   #values #love #\n",
      "3 . i get to see my daddy today!!   #80days #gettingfed\n",
      "4 . ouch...junior is angryð#got7 #junior #yugyoem   #omg \n",
      "5 . i am thankful for having a paner. #thankful #positive     \n"
     ]
    }
   ],
   "source": [
    "#Understanding the tweets\n",
    "\n",
    "for index,tweet in enumerate(dataset[\"tweet\"][10:15]):\n",
    "    print(index+1,\".\",tweet)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note :- Noise present in Tweets\n",
    "\n",
    "We can see that there are many hashtags present in the tweets of the form # symbol followed by text. We particularly don't need the # symbol so we will clean it out.\n",
    "Also, there are strange symbols like â and ð in tweet 4. This is actually unicode characters that is present in our dataset that we need to get rid of because they don't particularly add anything meaningful.\n",
    "There are also numerals and percentages "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def clean_text(text):\n",
    "    \n",
    "    text=re.sub(r'[^a-zA-Z\\']', ' ', text)\n",
    "    \n",
    "    #Remove Unicode characters\n",
    "    text = re.sub(r'[^\\x00-\\x7F]+', '', text)\n",
    "    \n",
    "    #Convert to lowercase to maintain consistency\n",
    "    text = text.lower()\n",
    "       \n",
    "    return text\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset['clean_text']=dataset.tweet.apply(lambda x: clean_text(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'with', 'been', \"didn't\", 'same', \"you've\", 'r', 'because', 'few', 'was', 'herself', 'against', \"aren't\", \"doesn't\", 'it', 'some', 'from', 'at', 'them', \"weren't\", 'did', 'he', 'as', 'once', 'other', 'is', \"we'll\", 'your', 'itself', 'http', 'out', 'themselves', 'myself', \"what's\", 'himself', 'ourselves', 'nor', 'should', 'how', 'under', 'ours', 'after', 'would', 'his', \"here's\", 'which', 'by', 'her', 'any', 'about', \"they've\", \"isn't\", 'between', \"couldn't\", \"they'd\", \"they're\", \"i've\", \"won't\", \"wouldn't\", 'not', 'she', \"it's\", 'for', \"don't\", 'above', 'during', 'such', 'too', 'had', \"you'll\", \"they'll\", \"who's\", \"hadn't\", 'also', 'only', 'what', \"how's\", 'when', 'that', 'com', 'an', \"he'd\", \"shan't\", 'so', \"i'd\", 'else', 'yours', 'these', 'very', \"we're\", \"wasn't\", 'my', 'their', 'just', \"he'll\", 'over', 'those', 'they', 'both', 'to', 'be', 'ought', 'cannot', 'like', \"why's\", 'all', 'down', 'where', 'this', 'on', \"she's\", 'below', \"when's\", 'again', 'have', 'our', 'you', 'however', \"i'm\", \"shouldn't\", 'further', \"hasn't\", 'off', 'a', \"we'd\", 'more', \"that's\", 'are', 'theirs', \"we've\", \"you're\", 'can', 'am', 'since', 'doing', 'and', 'otherwise', 'being', 'why', \"mustn't\", \"he's\", 'him', 'whom', \"let's\", 'in', 'yourself', 'into', 'me', 'while', 'k', 'shall', 'through', 'yourselves', \"haven't\", 'we', 'i', 'no', 'up', 'each', 'do', 'get', 'if', 'has', \"i'll\", 'of', 'most', \"there's\", 'but', \"can't\", 'ever', 'own', 'there', 'were', 'before', \"you'd\", \"she'll\", 'does', 'hers', 'or', 'here', 'its', 'than', \"where's\", \"she'd\", 'could', 'having', 'the', 'then', 'www', 'who', 'until'}\n"
     ]
    }
   ],
   "source": [
    "from wordcloud import STOPWORDS\n",
    "\n",
    "print(STOPWORDS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Generating word frequency\n",
    "\n",
    "def gen_freq(text):\n",
    "    #Create a list\n",
    "    word_list=[]\n",
    "    \n",
    "    for tw_words in text.split():\n",
    "        word_list.extend(tw_words)\n",
    "        \n",
    "    #Creating word frequency\n",
    "    word_freq=pd.Series(word_list).value_counts()\n",
    "    \n",
    "    return word_freq\n",
    "    \n",
    "#To check if a negetive word is present\n",
    "def any_neg(words):\n",
    "    for word in words:\n",
    "        if word in ['n', 'no', 'non', 'not'] or re.search(r\"\\wn't\", word):\n",
    "            return 1\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "#To check if 100 rare words are present\n",
    "    \n",
    "def any_rare(words,rare_100):\n",
    "    for word in words:\n",
    "        if word in rare_100:\n",
    "            return 1\n",
    "    else:\n",
    "            return 0\n",
    "#Check whether prompt words are present\n",
    "def is_question(words):\n",
    "    for word in words:\n",
    "        if word in ['when', 'what', 'how', 'why', 'who']:\n",
    "            return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_freq = gen_freq(dataset.clean_text.str)\n",
    "#100 most rare words in the dataset\n",
    "rare_100 = word_freq[-100:]\n",
    "#Number of words in a tweet\n",
    "dataset['word_count'] = dataset.clean_text.str.split().apply(lambda x: len(x))\n",
    "#Negation present or not\n",
    "dataset['any_neg'] = dataset.clean_text.str.split().apply(lambda x: any_neg(x))\n",
    "#Prompt present or not\n",
    "dataset['is_question'] = dataset.clean_text.str.split().apply(lambda x: is_question(x))\n",
    "#Any of the most 100 rare words present or not\n",
    "dataset['any_rare'] = dataset.clean_text.str.split().apply(lambda x: any_rare(x, rare_100))\n",
    "#Character count of the tweet\n",
    "dataset['char_count'] = dataset.clean_text.apply(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user    3351\n",
       "the     1880\n",
       "to      1497\n",
       "a       1232\n",
       "you      949\n",
       "in       899\n",
       "of       893\n",
       "is       853\n",
       "and      821\n",
       "i        805\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Top 10 common words are\n",
    "gen_freq(dataset.clean_text.str)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>tweet</th>\n",
       "      <th>clean_text</th>\n",
       "      <th>word_count</th>\n",
       "      <th>any_neg</th>\n",
       "      <th>is_question</th>\n",
       "      <th>any_rare</th>\n",
       "      <th>char_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>@user when a father is dysfunctional and is s...</td>\n",
       "      <td>user when a father is dysfunctional and is s...</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>@user @user thanks for #lyft credit i can't us...</td>\n",
       "      <td>user  user thanks for  lyft credit i can't us...</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>bihday your majesty</td>\n",
       "      <td>bihday your majesty</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>#model   i love u take with u all the time in ...</td>\n",
       "      <td>model   i love u take with u all the time in ...</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>factsguide: society now    #motivation</td>\n",
       "      <td>factsguide  society now     motivation</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  label                                              tweet  \\\n",
       "0   1      0   @user when a father is dysfunctional and is s...   \n",
       "1   2      0  @user @user thanks for #lyft credit i can't us...   \n",
       "2   3      0                                bihday your majesty   \n",
       "3   4      0  #model   i love u take with u all the time in ...   \n",
       "4   5      0             factsguide: society now    #motivation   \n",
       "\n",
       "                                          clean_text  word_count  any_neg  \\\n",
       "0    user when a father is dysfunctional and is s...          18        0   \n",
       "1   user  user thanks for  lyft credit i can't us...          19        1   \n",
       "2                                bihday your majesty           3        0   \n",
       "3   model   i love u take with u all the time in ...          12        0   \n",
       "4             factsguide  society now     motivation           4        0   \n",
       "\n",
       "   is_question  any_rare  char_count  \n",
       "0            1         0         102  \n",
       "1            0         0         122  \n",
       "2            0         0          21  \n",
       "3            0         0          86  \n",
       "4            0         0          39  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Splitting the dataset into Train-Test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = dataset[['word_count', 'any_neg', 'any_rare', 'char_count', 'is_question']]\n",
    "y = dataset.label\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.linear_model import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LinearRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "coefficient of determination: 0.0171252008919\n"
     ]
    }
   ],
   "source": [
    "r_sq = model.score(X_train, y_train)\n",
    "print('coefficient of determination:', r_sq)\n",
    "pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Naive Bayes classifier from sklearn\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "#Initialize GaussianNB classifier\n",
    "model = GaussianNB()\n",
    "#Fit the model on the train dataset\n",
    "model = model.fit(X_train, y_train)\n",
    "#Make predictions on the test dataset\n",
    "pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 58.2326764145 %\n"
     ]
    }
   ],
   "source": [
    "#Evaluate the ML model\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "print(\"Accuracy:\", accuracy_score(y_test, pred)*100, \"%\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}