16-Topic_Modelling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\duong\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers\\punkt.zip.\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\duong\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping corpora\\stopwords.zip.\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\duong\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data] Downloading package omw-1.4 to\n",
      "[nltk_data]     C:\\Users\\duong\\AppData\\Roaming\\nltk_data...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Article</th>\n",
       "      <th>Title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Data analysis is the process of inspecting and...</td>\n",
       "      <td>Best Books to Learn Data Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The performance of a machine learning algorith...</td>\n",
       "      <td>Assumptions of Machine Learning Algorithms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>You must have seen the news divided into categ...</td>\n",
       "      <td>News Classification with Machine Learning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>When there are only two classes in a classific...</td>\n",
       "      <td>Multiclass Classification Algorithms in Machin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The Multinomial Naive Bayes is one of the vari...</td>\n",
       "      <td>Multinomial Naive Bayes in Machine Learning</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Article  \\\n",
       "0  Data analysis is the process of inspecting and...   \n",
       "1  The performance of a machine learning algorith...   \n",
       "2  You must have seen the news divided into categ...   \n",
       "3  When there are only two classes in a classific...   \n",
       "4  The Multinomial Naive Bayes is one of the vari...   \n",
       "\n",
       "                                               Title  \n",
       "0                  Best Books to Learn Data Analysis  \n",
       "1         Assumptions of Machine Learning Algorithms  \n",
       "2          News Classification with Machine Learning  \n",
       "3  Multiclass Classification Algorithms in Machin...  \n",
       "4        Multinomial Naive Bayes in Machine Learning  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "import string\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('omw-1.4')\n",
    "\n",
    "data = pd.read_csv(\"Data/articles.csv\", encoding='latin1')\n",
    "data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_text(text):\n",
    "    # Convert text to lowercase\n",
    "    text = text.lower()\n",
    "    # Remove punctuation\n",
    "    text = text.translate(str.maketrans('', '', string.punctuation))\n",
    "    # Tokenize text\n",
    "    tokens = nltk.word_tokenize(text)\n",
    "    # Remove stopwords\n",
    "    stop_words = set(stopwords.words(\"english\"))\n",
    "    tokens = [word for word in tokens if word not in stop_words]\n",
    "    # Lemmatize tokens\n",
    "    lemma = WordNetLemmatizer()\n",
    "    tokens = [lemma.lemmatize(word) for word in tokens]\n",
    "    # Join tokens to form prepocessed text\n",
    "    preprocessed_text = ' '.join(tokens)\n",
    "    return preprocessed_text\n",
    "\n",
    "data['Article'] = data['Article'].apply(preprocess_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer()\n",
    "x = vectorizer.fit_transform(data['Article'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "lda = LatentDirichletAllocation(n_components=5, random_state=42)\n",
    "lda.fit(x)\n",
    "\n",
    "topic_modelling = lda.transform(x)\n",
    "\n",
    "topic_labels = np.argmax(topic_modelling, axis=1)\n",
    "data['topic_labels'] = topic_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Article</th>\n",
       "      <th>Title</th>\n",
       "      <th>topic_labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>data analysis process inspecting exploring dat...</td>\n",
       "      <td>Best Books to Learn Data Analysis</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>performance machine learning algorithm particu...</td>\n",
       "      <td>Assumptions of Machine Learning Algorithms</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>must seen news divided category go news websit...</td>\n",
       "      <td>News Classification with Machine Learning</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>two class classification problem problem binar...</td>\n",
       "      <td>Multiclass Classification Algorithms in Machin...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>multinomial naive bayes one variant naive baye...</td>\n",
       "      <td>Multinomial Naive Bayes in Machine Learning</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>must seen news divided category go news websit...</td>\n",
       "      <td>News Classification with Machine Learning</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>natural language processing nlp subfield artif...</td>\n",
       "      <td>Best Books to Learn NLP</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>using thirdparty application api manage functi...</td>\n",
       "      <td>Send Instagram Messages using Python</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>twitter one popular social medium apps people ...</td>\n",
       "      <td>Pfizer Vaccine Sentiment Analysis using Python</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>squid game currently one trending show netflix...</td>\n",
       "      <td>Squid Game Sentiment Analysis using Python</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Article  \\\n",
       "0  data analysis process inspecting exploring dat...   \n",
       "1  performance machine learning algorithm particu...   \n",
       "2  must seen news divided category go news websit...   \n",
       "3  two class classification problem problem binar...   \n",
       "4  multinomial naive bayes one variant naive baye...   \n",
       "5  must seen news divided category go news websit...   \n",
       "6  natural language processing nlp subfield artif...   \n",
       "7  using thirdparty application api manage functi...   \n",
       "8  twitter one popular social medium apps people ...   \n",
       "9  squid game currently one trending show netflix...   \n",
       "\n",
       "                                               Title  topic_labels  \n",
       "0                  Best Books to Learn Data Analysis             2  \n",
       "1         Assumptions of Machine Learning Algorithms             3  \n",
       "2          News Classification with Machine Learning             1  \n",
       "3  Multiclass Classification Algorithms in Machin...             3  \n",
       "4        Multinomial Naive Bayes in Machine Learning             1  \n",
       "5          News Classification with Machine Learning             1  \n",
       "6                            Best Books to Learn NLP             2  \n",
       "7               Send Instagram Messages using Python             4  \n",
       "8     Pfizer Vaccine Sentiment Analysis using Python             0  \n",
       "9         Squid Game Sentiment Analysis using Python             2  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.1"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}