Fin-bert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "de7dff69",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import BertTokenizer, BertModel, BertForSequenceClassification\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "40c8cd70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9566eb535b8483b8c08d119b487d26b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model.bin:   0%|          | 0.00/438M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\saite\\anaconda3\\Lib\\site-packages\\huggingface_hub\\file_download.py:148: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\saite\\.cache\\huggingface\\hub\\models--ProsusAI--finbert. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    }
   ],
   "source": [
    "# Load ProsusAI/finbert for embeddings\n",
    "emb_model_name = 'ProsusAI/finbert'\n",
    "emb_tokenizer = BertTokenizer.from_pretrained(emb_model_name)\n",
    "emb_model = BertModel.from_pretrained(emb_model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5b77223a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f333f706b0de44808176d9747a726be5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt:   0%|          | 0.00/226k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\saite\\anaconda3\\Lib\\site-packages\\huggingface_hub\\file_download.py:148: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\saite\\.cache\\huggingface\\hub\\models--yiyanghkust--finbert-tone. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21b83bbf347f424a96660fdd1da77170",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/533 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49f1d4dcada642358b08cbbade37efa4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model.bin:   0%|          | 0.00/439M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Load yiyanghkust/finbert-tone for sentiment analysis\n",
    "sent_model_name = 'yiyanghkust/finbert-tone'\n",
    "sent_tokenizer = BertTokenizer.from_pretrained(sent_model_name)\n",
    "sent_model = BertForSequenceClassification.from_pretrained(sent_model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b7f06d32",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"news_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "df7091aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "090f3edd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 8148/8148 [52:22<00:00,  2.59it/s]\n"
     ]
    }
   ],
   "source": [
    "def get_embeddings(texts):\n",
    "    encoded_input = emb_tokenizer(texts, padding=True, truncation=True, return_tensors='pt', max_length=512)\n",
    "    with torch.no_grad():\n",
    "        outputs = emb_model(**encoded_input)\n",
    "    embeddings = outputs.last_hidden_state[:, 0, :].numpy()  # Extract embeddings of the [CLS] token\n",
    "    return embeddings\n",
    "\n",
    "def get_sentiment(texts):\n",
    "    encoded_input = sent_tokenizer(texts, padding=True, truncation=True, return_tensors='pt', max_length=512)\n",
    "    with torch.no_grad():\n",
    "        outputs = sent_model(**encoded_input)\n",
    "    probabilities = torch.nn.functional.softmax(outputs.logits, dim=-1).numpy()\n",
    "    return probabilities\n",
    "\n",
    "def process_texts_in_batches(texts, batch_size=10):\n",
    "    all_embeddings = []\n",
    "    all_sentiments = []\n",
    "    for i in tqdm(range(0, len(texts), batch_size)):\n",
    "        batch_texts = texts[i:i + batch_size]\n",
    "        embeddings = get_embeddings(batch_texts)\n",
    "        sentiments = get_sentiment(batch_texts)\n",
    "        all_embeddings.extend(embeddings)\n",
    "        all_sentiments.extend(sentiments)\n",
    "    return all_embeddings, all_sentiments\n",
    "\n",
    "# Applying batch processing\n",
    "embeddings, sentiments = process_texts_in_batches(df['summary'].tolist(), batch_size=3)\n",
    "df['embeddings'] = embeddings\n",
    "df['sentiment_scores'] = sentiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "81cb977a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>time_published</th>\n",
       "      <th>authors</th>\n",
       "      <th>summary</th>\n",
       "      <th>banner_image</th>\n",
       "      <th>source</th>\n",
       "      <th>category_within_source</th>\n",
       "      <th>source_domain</th>\n",
       "      <th>topics</th>\n",
       "      <th>overall_sentiment_score</th>\n",
       "      <th>overall_sentiment_label</th>\n",
       "      <th>ticker_sentiment</th>\n",
       "      <th>date</th>\n",
       "      <th>embeddings</th>\n",
       "      <th>sentiment_scores</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Apple, AMC, Meta, And How Elon Musk Is Reactin...</td>\n",
       "      <td>https://www.benzinga.com/news/large-cap/22/12/...</td>\n",
       "      <td>20221231T190310</td>\n",
       "      <td>['Michael Cohen']</td>\n",
       "      <td>Benzinga examined the prospects for many inves...</td>\n",
       "      <td>https://cdn.benzinga.com/files/images/story/20...</td>\n",
       "      <td>Benzinga</td>\n",
       "      <td>Trading</td>\n",
       "      <td>www.benzinga.com</td>\n",
       "      <td>[{'topic': 'Financial Markets', 'relevance_sco...</td>\n",
       "      <td>-0.079733</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>[{'ticker': 'MSTR', 'relevance_score': '0.2478...</td>\n",
       "      <td>20221231</td>\n",
       "      <td>[0.12574883, -0.24758251, 0.44414037, -0.37607...</td>\n",
       "      <td>[0.049918726, 0.00011439536, 0.94996685]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Bulls In A Bear Market: These 10 Stocks Clocke...</td>\n",
       "      <td>https://www.benzinga.com/analyst-ratings/analy...</td>\n",
       "      <td>20221231T161114</td>\n",
       "      <td>['Shanthi Rexaline']</td>\n",
       "      <td>2022 would go down as one of the worst years f...</td>\n",
       "      <td>https://cdn.benzinga.com/files/images/story/20...</td>\n",
       "      <td>Benzinga</td>\n",
       "      <td>General</td>\n",
       "      <td>www.benzinga.com</td>\n",
       "      <td>[{'topic': 'Life Sciences', 'relevance_score':...</td>\n",
       "      <td>0.064077</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>[{'ticker': 'AAPL', 'relevance_score': '0.1137...</td>\n",
       "      <td>20221231</td>\n",
       "      <td>[0.055901304, 0.17285924, -0.28222713, -0.7317...</td>\n",
       "      <td>[4.4178055e-06, 2.3380646e-05, 0.9999722]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Where Will Unity Software Stock Be in 3 Years?</td>\n",
       "      <td>https://www.fool.com/investing/2022/12/31/wher...</td>\n",
       "      <td>20221231T154500</td>\n",
       "      <td>['Leo Sun']</td>\n",
       "      <td>The game-engine developer still has a lot to p...</td>\n",
       "      <td>https://g.foolcdn.com/image/?url=https%3A%2F%2...</td>\n",
       "      <td>Motley Fool</td>\n",
       "      <td>NaN</td>\n",
       "      <td>www.fool.com</td>\n",
       "      <td>[{'topic': 'IPO', 'relevance_score': '0.769861...</td>\n",
       "      <td>0.202065</td>\n",
       "      <td>Somewhat-Bullish</td>\n",
       "      <td>[{'ticker': 'U', 'relevance_score': '0.095517'...</td>\n",
       "      <td>20221231</td>\n",
       "      <td>[0.49274713, 0.46850753, -0.35436904, 0.238316...</td>\n",
       "      <td>[0.86392456, 0.006136369, 0.12993914]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3 Unstoppable Growth Stocks to Buy After a Sto...</td>\n",
       "      <td>https://www.fool.com/investing/2022/12/31/3-un...</td>\n",
       "      <td>20221231T140000</td>\n",
       "      <td>['Dani Cook']</td>\n",
       "      <td>These companies have grown by triple-digit per...</td>\n",
       "      <td>https://g.foolcdn.com/image/?url=https%3A%2F%2...</td>\n",
       "      <td>Motley Fool</td>\n",
       "      <td>NaN</td>\n",
       "      <td>www.fool.com</td>\n",
       "      <td>[{'topic': 'Financial Markets', 'relevance_sco...</td>\n",
       "      <td>0.152163</td>\n",
       "      <td>Somewhat-Bullish</td>\n",
       "      <td>[{'ticker': 'AMD', 'relevance_score': '0.22324...</td>\n",
       "      <td>20221231</td>\n",
       "      <td>[0.8152653, -0.14137518, -0.18449828, -0.27026...</td>\n",
       "      <td>[1.3719941e-06, 0.99999845, 1.043882e-07]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Even in an Advertising Slowdown, These 3 Stock...</td>\n",
       "      <td>https://www.fool.com/investing/2022/12/31/even...</td>\n",
       "      <td>20221231T120000</td>\n",
       "      <td>['Adam Levy']</td>\n",
       "      <td>Ad sales growth will slow in 2023, but these s...</td>\n",
       "      <td>https://g.foolcdn.com/image/?url=https%3A%2F%2...</td>\n",
       "      <td>Motley Fool</td>\n",
       "      <td>NaN</td>\n",
       "      <td>www.fool.com</td>\n",
       "      <td>[{'topic': 'Earnings', 'relevance_score': '0.9...</td>\n",
       "      <td>0.151938</td>\n",
       "      <td>Somewhat-Bullish</td>\n",
       "      <td>[{'ticker': 'GOOG', 'relevance_score': '0.2433...</td>\n",
       "      <td>20221231</td>\n",
       "      <td>[0.19915596, -0.1503704, 0.5523808, 0.14805436...</td>\n",
       "      <td>[0.0020524256, 0.011877623, 0.98607004]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24437</th>\n",
       "      <td>Nvidia's High-End Chips Ended Up In Chinese Ha...</td>\n",
       "      <td>https://www.benzinga.com/markets/asia/24/04/38...</td>\n",
       "      <td>20240423T050111</td>\n",
       "      <td>['Benzinga Neuro']</td>\n",
       "      <td>Chinese entities have managed to acquire high-...</td>\n",
       "      <td>https://cdn.benzinga.com/files/images/story/20...</td>\n",
       "      <td>Benzinga</td>\n",
       "      <td>Markets</td>\n",
       "      <td>www.benzinga.com</td>\n",
       "      <td>[{'topic': 'Mergers &amp; Acquisitions', 'relevanc...</td>\n",
       "      <td>0.095783</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>[{'ticker': 'NVDA', 'relevance_score': '0.4274...</td>\n",
       "      <td>20240423</td>\n",
       "      <td>[0.27100983, 0.5836564, -1.0146141, -0.0952048...</td>\n",
       "      <td>[0.9999578, 1.1006024e-05, 3.1285294e-05]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24438</th>\n",
       "      <td>Broadcom Inc.  ( AVGO )  Gains But Lags Market...</td>\n",
       "      <td>https://www.zacks.com/stock/news/2259620/broad...</td>\n",
       "      <td>20240422T214516</td>\n",
       "      <td>['Zacks Equity Research']</td>\n",
       "      <td>In the most recent trading session, Broadcom I...</td>\n",
       "      <td>https://staticx-tuner.zacks.com/images/default...</td>\n",
       "      <td>Zacks Commentary</td>\n",
       "      <td>NaN</td>\n",
       "      <td>www.zacks.com</td>\n",
       "      <td>[{'topic': 'Earnings', 'relevance_score': '0.9...</td>\n",
       "      <td>0.14976</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>[{'ticker': 'AVGO', 'relevance_score': '0.5372...</td>\n",
       "      <td>20240422</td>\n",
       "      <td>[0.31473163, -1.2273293, 0.6177586, -0.3789892...</td>\n",
       "      <td>[0.99998486, 4.1800554e-06, 1.0952555e-05]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24439</th>\n",
       "      <td>Data Center Interconnect Market size worth $ 3...</td>\n",
       "      <td>https://www.benzinga.com/pressreleases/24/04/g...</td>\n",
       "      <td>20240422T140000</td>\n",
       "      <td>['Globe Newswire']</td>\n",
       "      <td>Jersey City, New Jersey, April 22, 2024 ( GLOB...</td>\n",
       "      <td>https://www.benzinga.com/next-assets/images/sc...</td>\n",
       "      <td>Benzinga</td>\n",
       "      <td>General</td>\n",
       "      <td>www.benzinga.com</td>\n",
       "      <td>[{'topic': 'Financial Markets', 'relevance_sco...</td>\n",
       "      <td>0.252395</td>\n",
       "      <td>Somewhat-Bullish</td>\n",
       "      <td>[{'ticker': 'CIEN', 'relevance_score': '0.0282...</td>\n",
       "      <td>20240422</td>\n",
       "      <td>[0.6741358, -0.5193285, 0.25507122, -0.5724481...</td>\n",
       "      <td>[0.0041528903, 0.9958448, 2.3902537e-06]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24440</th>\n",
       "      <td>Is Nvidia Getting Ready to Disrupt Another Com...</td>\n",
       "      <td>https://www.fool.com/investing/2024/04/22/is-n...</td>\n",
       "      <td>20240422T123900</td>\n",
       "      <td>['Nicholas Rossolillo']</td>\n",
       "      <td>Some investors were second-guessing the streng...</td>\n",
       "      <td>https://g.foolcdn.com/editorial/images/773500/...</td>\n",
       "      <td>Motley Fool</td>\n",
       "      <td>NaN</td>\n",
       "      <td>www.fool.com</td>\n",
       "      <td>[{'topic': 'Financial Markets', 'relevance_sco...</td>\n",
       "      <td>0.200374</td>\n",
       "      <td>Somewhat-Bullish</td>\n",
       "      <td>[{'ticker': 'MSFT', 'relevance_score': '0.0584...</td>\n",
       "      <td>20240422</td>\n",
       "      <td>[0.5494284, -0.32981983, -0.42738038, -0.37707...</td>\n",
       "      <td>[0.0037541273, 0.9959959, 0.00024995624]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24441</th>\n",
       "      <td>Biden Cabinet Member Isn't Impressed With Huaw...</td>\n",
       "      <td>https://www.benzinga.com/news/24/04/38350312/b...</td>\n",
       "      <td>20240422T040220</td>\n",
       "      <td>['Ananya Gairola']</td>\n",
       "      <td>U.S. Commerce Secretary Gina Raimondo dismisse...</td>\n",
       "      <td>https://cdn.benzinga.com/files/images/story/20...</td>\n",
       "      <td>Benzinga</td>\n",
       "      <td>News</td>\n",
       "      <td>www.benzinga.com</td>\n",
       "      <td>[{'topic': 'Manufacturing', 'relevance_score':...</td>\n",
       "      <td>0.167207</td>\n",
       "      <td>Somewhat-Bullish</td>\n",
       "      <td>[{'ticker': 'SPGI', 'relevance_score': '0.0962...</td>\n",
       "      <td>20240422</td>\n",
       "      <td>[-0.18458675, -0.15688337, -0.121610455, 0.161...</td>\n",
       "      <td>[0.007273258, 0.00011163251, 0.9926151]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24442 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title  \\\n",
       "0      Apple, AMC, Meta, And How Elon Musk Is Reactin...   \n",
       "1      Bulls In A Bear Market: These 10 Stocks Clocke...   \n",
       "2         Where Will Unity Software Stock Be in 3 Years?   \n",
       "3      3 Unstoppable Growth Stocks to Buy After a Sto...   \n",
       "4      Even in an Advertising Slowdown, These 3 Stock...   \n",
       "...                                                  ...   \n",
       "24437  Nvidia's High-End Chips Ended Up In Chinese Ha...   \n",
       "24438  Broadcom Inc.  ( AVGO )  Gains But Lags Market...   \n",
       "24439  Data Center Interconnect Market size worth $ 3...   \n",
       "24440  Is Nvidia Getting Ready to Disrupt Another Com...   \n",
       "24441  Biden Cabinet Member Isn't Impressed With Huaw...   \n",
       "\n",
       "                                                     url   time_published  \\\n",
       "0      https://www.benzinga.com/news/large-cap/22/12/...  20221231T190310   \n",
       "1      https://www.benzinga.com/analyst-ratings/analy...  20221231T161114   \n",
       "2      https://www.fool.com/investing/2022/12/31/wher...  20221231T154500   \n",
       "3      https://www.fool.com/investing/2022/12/31/3-un...  20221231T140000   \n",
       "4      https://www.fool.com/investing/2022/12/31/even...  20221231T120000   \n",
       "...                                                  ...              ...   \n",
       "24437  https://www.benzinga.com/markets/asia/24/04/38...  20240423T050111   \n",
       "24438  https://www.zacks.com/stock/news/2259620/broad...  20240422T214516   \n",
       "24439  https://www.benzinga.com/pressreleases/24/04/g...  20240422T140000   \n",
       "24440  https://www.fool.com/investing/2024/04/22/is-n...  20240422T123900   \n",
       "24441  https://www.benzinga.com/news/24/04/38350312/b...  20240422T040220   \n",
       "\n",
       "                         authors  \\\n",
       "0              ['Michael Cohen']   \n",
       "1           ['Shanthi Rexaline']   \n",
       "2                    ['Leo Sun']   \n",
       "3                  ['Dani Cook']   \n",
       "4                  ['Adam Levy']   \n",
       "...                          ...   \n",
       "24437         ['Benzinga Neuro']   \n",
       "24438  ['Zacks Equity Research']   \n",
       "24439         ['Globe Newswire']   \n",
       "24440    ['Nicholas Rossolillo']   \n",
       "24441         ['Ananya Gairola']   \n",
       "\n",
       "                                                 summary  \\\n",
       "0      Benzinga examined the prospects for many inves...   \n",
       "1      2022 would go down as one of the worst years f...   \n",
       "2      The game-engine developer still has a lot to p...   \n",
       "3      These companies have grown by triple-digit per...   \n",
       "4      Ad sales growth will slow in 2023, but these s...   \n",
       "...                                                  ...   \n",
       "24437  Chinese entities have managed to acquire high-...   \n",
       "24438  In the most recent trading session, Broadcom I...   \n",
       "24439  Jersey City, New Jersey, April 22, 2024 ( GLOB...   \n",
       "24440  Some investors were second-guessing the streng...   \n",
       "24441  U.S. Commerce Secretary Gina Raimondo dismisse...   \n",
       "\n",
       "                                            banner_image            source  \\\n",
       "0      https://cdn.benzinga.com/files/images/story/20...          Benzinga   \n",
       "1      https://cdn.benzinga.com/files/images/story/20...          Benzinga   \n",
       "2      https://g.foolcdn.com/image/?url=https%3A%2F%2...       Motley Fool   \n",
       "3      https://g.foolcdn.com/image/?url=https%3A%2F%2...       Motley Fool   \n",
       "4      https://g.foolcdn.com/image/?url=https%3A%2F%2...       Motley Fool   \n",
       "...                                                  ...               ...   \n",
       "24437  https://cdn.benzinga.com/files/images/story/20...          Benzinga   \n",
       "24438  https://staticx-tuner.zacks.com/images/default...  Zacks Commentary   \n",
       "24439  https://www.benzinga.com/next-assets/images/sc...          Benzinga   \n",
       "24440  https://g.foolcdn.com/editorial/images/773500/...       Motley Fool   \n",
       "24441  https://cdn.benzinga.com/files/images/story/20...          Benzinga   \n",
       "\n",
       "      category_within_source     source_domain  \\\n",
       "0                    Trading  www.benzinga.com   \n",
       "1                    General  www.benzinga.com   \n",
       "2                        NaN      www.fool.com   \n",
       "3                        NaN      www.fool.com   \n",
       "4                        NaN      www.fool.com   \n",
       "...                      ...               ...   \n",
       "24437                Markets  www.benzinga.com   \n",
       "24438                    NaN     www.zacks.com   \n",
       "24439                General  www.benzinga.com   \n",
       "24440                    NaN      www.fool.com   \n",
       "24441                   News  www.benzinga.com   \n",
       "\n",
       "                                                  topics  \\\n",
       "0      [{'topic': 'Financial Markets', 'relevance_sco...   \n",
       "1      [{'topic': 'Life Sciences', 'relevance_score':...   \n",
       "2      [{'topic': 'IPO', 'relevance_score': '0.769861...   \n",
       "3      [{'topic': 'Financial Markets', 'relevance_sco...   \n",
       "4      [{'topic': 'Earnings', 'relevance_score': '0.9...   \n",
       "...                                                  ...   \n",
       "24437  [{'topic': 'Mergers & Acquisitions', 'relevanc...   \n",
       "24438  [{'topic': 'Earnings', 'relevance_score': '0.9...   \n",
       "24439  [{'topic': 'Financial Markets', 'relevance_sco...   \n",
       "24440  [{'topic': 'Financial Markets', 'relevance_sco...   \n",
       "24441  [{'topic': 'Manufacturing', 'relevance_score':...   \n",
       "\n",
       "      overall_sentiment_score overall_sentiment_label  \\\n",
       "0                   -0.079733                 Neutral   \n",
       "1                    0.064077                 Neutral   \n",
       "2                    0.202065        Somewhat-Bullish   \n",
       "3                    0.152163        Somewhat-Bullish   \n",
       "4                    0.151938        Somewhat-Bullish   \n",
       "...                       ...                     ...   \n",
       "24437                0.095783                 Neutral   \n",
       "24438                 0.14976                 Neutral   \n",
       "24439                0.252395        Somewhat-Bullish   \n",
       "24440                0.200374        Somewhat-Bullish   \n",
       "24441                0.167207        Somewhat-Bullish   \n",
       "\n",
       "                                        ticker_sentiment      date  \\\n",
       "0      [{'ticker': 'MSTR', 'relevance_score': '0.2478...  20221231   \n",
       "1      [{'ticker': 'AAPL', 'relevance_score': '0.1137...  20221231   \n",
       "2      [{'ticker': 'U', 'relevance_score': '0.095517'...  20221231   \n",
       "3      [{'ticker': 'AMD', 'relevance_score': '0.22324...  20221231   \n",
       "4      [{'ticker': 'GOOG', 'relevance_score': '0.2433...  20221231   \n",
       "...                                                  ...       ...   \n",
       "24437  [{'ticker': 'NVDA', 'relevance_score': '0.4274...  20240423   \n",
       "24438  [{'ticker': 'AVGO', 'relevance_score': '0.5372...  20240422   \n",
       "24439  [{'ticker': 'CIEN', 'relevance_score': '0.0282...  20240422   \n",
       "24440  [{'ticker': 'MSFT', 'relevance_score': '0.0584...  20240422   \n",
       "24441  [{'ticker': 'SPGI', 'relevance_score': '0.0962...  20240422   \n",
       "\n",
       "                                              embeddings  \\\n",
       "0      [0.12574883, -0.24758251, 0.44414037, -0.37607...   \n",
       "1      [0.055901304, 0.17285924, -0.28222713, -0.7317...   \n",
       "2      [0.49274713, 0.46850753, -0.35436904, 0.238316...   \n",
       "3      [0.8152653, -0.14137518, -0.18449828, -0.27026...   \n",
       "4      [0.19915596, -0.1503704, 0.5523808, 0.14805436...   \n",
       "...                                                  ...   \n",
       "24437  [0.27100983, 0.5836564, -1.0146141, -0.0952048...   \n",
       "24438  [0.31473163, -1.2273293, 0.6177586, -0.3789892...   \n",
       "24439  [0.6741358, -0.5193285, 0.25507122, -0.5724481...   \n",
       "24440  [0.5494284, -0.32981983, -0.42738038, -0.37707...   \n",
       "24441  [-0.18458675, -0.15688337, -0.121610455, 0.161...   \n",
       "\n",
       "                                 sentiment_scores  \n",
       "0        [0.049918726, 0.00011439536, 0.94996685]  \n",
       "1       [4.4178055e-06, 2.3380646e-05, 0.9999722]  \n",
       "2           [0.86392456, 0.006136369, 0.12993914]  \n",
       "3       [1.3719941e-06, 0.99999845, 1.043882e-07]  \n",
       "4         [0.0020524256, 0.011877623, 0.98607004]  \n",
       "...                                           ...  \n",
       "24437   [0.9999578, 1.1006024e-05, 3.1285294e-05]  \n",
       "24438  [0.99998486, 4.1800554e-06, 1.0952555e-05]  \n",
       "24439    [0.0041528903, 0.9958448, 2.3902537e-06]  \n",
       "24440    [0.0037541273, 0.9959959, 0.00024995624]  \n",
       "24441     [0.007273258, 0.00011163251, 0.9926151]  \n",
       "\n",
       "[24442 rows x 16 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a7b91c5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"[{'topic': 'Financial Markets', 'relevance_score': '0.316726'}\\n {'topic': 'Manufacturing', 'relevance_score': '0.333333'}\\n {'topic': 'Energy & Transportation', 'relevance_score': '0.333333'}\\n {'topic': 'Technology', 'relevance_score': '0.333333'}]\""
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['topics'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a245dbc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "\n",
    "def insert_commas(string):\n",
    "    # Insert a comma between '}' and '{'\n",
    "    corrected_string = string.replace('}\\n {', '},\\n {')\n",
    "    return corrected_string\n",
    "def parse_string(row):\n",
    "    try:\n",
    "        # Insert commas to correct the syntax\n",
    "        corrected_row = insert_commas(row)\n",
    "        # Convert the string representation of list-dict into actual list-dict\n",
    "        row_parsed = ast.literal_eval(corrected_row)\n",
    "        return row_parsed\n",
    "    except Exception as e:\n",
    "        print(f\"Error parsing row: {row} - {e}\")\n",
    "        return None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ecb19083",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error parsing row: topics - malformed node or string on line 1: <ast.Name object at 0x000001A898A5AA40>\n"
     ]
    }
   ],
   "source": [
    "# Apply this function to your DataFrame\n",
    "df['parsed_topics'] = df['topics'].apply(parse_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "41dfce0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        [{'topic': 'Financial Markets', 'relevance_sco...\n",
       "1        [{'topic': 'Life Sciences', 'relevance_score':...\n",
       "2        [{'topic': 'IPO', 'relevance_score': '0.769861...\n",
       "3        [{'topic': 'Financial Markets', 'relevance_sco...\n",
       "4        [{'topic': 'Earnings', 'relevance_score': '0.9...\n",
       "                               ...                        \n",
       "24437    [{'topic': 'Mergers & Acquisitions', 'relevanc...\n",
       "24438    [{'topic': 'Earnings', 'relevance_score': '0.9...\n",
       "24439    [{'topic': 'Financial Markets', 'relevance_sco...\n",
       "24440    [{'topic': 'Financial Markets', 'relevance_sco...\n",
       "24441    [{'topic': 'Manufacturing', 'relevance_score':...\n",
       "Name: parsed_topics, Length: 24442, dtype: object"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['parsed_topics']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2e2fef21",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'Technology': 17890,\n",
       "         'Financial Markets': 17165,\n",
       "         'Manufacturing': 13427,\n",
       "         'Earnings': 11387,\n",
       "         'Finance': 7142,\n",
       "         'Retail & Wholesale': 6229,\n",
       "         'Economy - Monetary': 4773,\n",
       "         'Life Sciences': 4032,\n",
       "         'Energy & Transportation': 2139,\n",
       "         'Real Estate & Construction': 1334,\n",
       "         'Blockchain': 1175,\n",
       "         'IPO': 1026,\n",
       "         'Mergers & Acquisitions': 749,\n",
       "         'Economy - Fiscal': 512,\n",
       "         'Economy - Macro': 302})"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "# Flatten the list of dictionaries to extract topics\n",
    "topics = [d['topic'] for sublist in df['parsed_topics'] if sublist is not None for d in sublist if 'topic' in d]\n",
    "\n",
    "# Count occurrences of each topic\n",
    "topic_counts = Counter(topics)\n",
    "topic_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "394d3af9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>time_published</th>\n",
       "      <th>authors</th>\n",
       "      <th>summary</th>\n",
       "      <th>banner_image</th>\n",
       "      <th>source</th>\n",
       "      <th>category_within_source</th>\n",
       "      <th>source_domain</th>\n",
       "      <th>topics</th>\n",
       "      <th>...</th>\n",
       "      <th>Retail &amp; Wholesale</th>\n",
       "      <th>Economy - Monetary</th>\n",
       "      <th>Life Sciences</th>\n",
       "      <th>Energy &amp; Transportation</th>\n",
       "      <th>Real Estate &amp; Construction</th>\n",
       "      <th>Blockchain</th>\n",
       "      <th>IPO</th>\n",
       "      <th>Mergers &amp; Acquisitions</th>\n",
       "      <th>Economy - Fiscal</th>\n",
       "      <th>Economy - Macro</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Apple, AMC, Meta, And How Elon Musk Is Reactin...</td>\n",
       "      <td>https://www.benzinga.com/news/large-cap/22/12/...</td>\n",
       "      <td>20221231T190310</td>\n",
       "      <td>['Michael Cohen']</td>\n",
       "      <td>Benzinga examined the prospects for many inves...</td>\n",
       "      <td>https://cdn.benzinga.com/files/images/story/20...</td>\n",
       "      <td>Benzinga</td>\n",
       "      <td>Trading</td>\n",
       "      <td>www.benzinga.com</td>\n",
       "      <td>[{'topic': 'Financial Markets', 'relevance_sco...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Bulls In A Bear Market: These 10 Stocks Clocke...</td>\n",
       "      <td>https://www.benzinga.com/analyst-ratings/analy...</td>\n",
       "      <td>20221231T161114</td>\n",
       "      <td>['Shanthi Rexaline']</td>\n",
       "      <td>2022 would go down as one of the worst years f...</td>\n",
       "      <td>https://cdn.benzinga.com/files/images/story/20...</td>\n",
       "      <td>Benzinga</td>\n",
       "      <td>General</td>\n",
       "      <td>www.benzinga.com</td>\n",
       "      <td>[{'topic': 'Life Sciences', 'relevance_score':...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.158519</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Where Will Unity Software Stock Be in 3 Years?</td>\n",
       "      <td>https://www.fool.com/investing/2022/12/31/wher...</td>\n",
       "      <td>20221231T154500</td>\n",
       "      <td>['Leo Sun']</td>\n",
       "      <td>The game-engine developer still has a lot to p...</td>\n",
       "      <td>https://g.foolcdn.com/image/?url=https%3A%2F%2...</td>\n",
       "      <td>Motley Fool</td>\n",
       "      <td>NaN</td>\n",
       "      <td>www.fool.com</td>\n",
       "      <td>[{'topic': 'IPO', 'relevance_score': '0.769861...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.769861</td>\n",
       "      <td>0.360215</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3 Unstoppable Growth Stocks to Buy After a Sto...</td>\n",
       "      <td>https://www.fool.com/investing/2022/12/31/3-un...</td>\n",
       "      <td>20221231T140000</td>\n",
       "      <td>['Dani Cook']</td>\n",
       "      <td>These companies have grown by triple-digit per...</td>\n",
       "      <td>https://g.foolcdn.com/image/?url=https%3A%2F%2...</td>\n",
       "      <td>Motley Fool</td>\n",
       "      <td>NaN</td>\n",
       "      <td>www.fool.com</td>\n",
       "      <td>[{'topic': 'Financial Markets', 'relevance_sco...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Even in an Advertising Slowdown, These 3 Stock...</td>\n",
       "      <td>https://www.fool.com/investing/2022/12/31/even...</td>\n",
       "      <td>20221231T120000</td>\n",
       "      <td>['Adam Levy']</td>\n",
       "      <td>Ad sales growth will slow in 2023, but these s...</td>\n",
       "      <td>https://g.foolcdn.com/image/?url=https%3A%2F%2...</td>\n",
       "      <td>Motley Fool</td>\n",
       "      <td>NaN</td>\n",
       "      <td>www.fool.com</td>\n",
       "      <td>[{'topic': 'Earnings', 'relevance_score': '0.9...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.158519</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.158519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24437</th>\n",
       "      <td>Nvidia's High-End Chips Ended Up In Chinese Ha...</td>\n",
       "      <td>https://www.benzinga.com/markets/asia/24/04/38...</td>\n",
       "      <td>20240423T050111</td>\n",
       "      <td>['Benzinga Neuro']</td>\n",
       "      <td>Chinese entities have managed to acquire high-...</td>\n",
       "      <td>https://cdn.benzinga.com/files/images/story/20...</td>\n",
       "      <td>Benzinga</td>\n",
       "      <td>Markets</td>\n",
       "      <td>www.benzinga.com</td>\n",
       "      <td>[{'topic': 'Mergers &amp; Acquisitions', 'relevanc...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.108179</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24438</th>\n",
       "      <td>Broadcom Inc.  ( AVGO )  Gains But Lags Market...</td>\n",
       "      <td>https://www.zacks.com/stock/news/2259620/broad...</td>\n",
       "      <td>20240422T214516</td>\n",
       "      <td>['Zacks Equity Research']</td>\n",
       "      <td>In the most recent trading session, Broadcom I...</td>\n",
       "      <td>https://staticx-tuner.zacks.com/images/default...</td>\n",
       "      <td>Zacks Commentary</td>\n",
       "      <td>NaN</td>\n",
       "      <td>www.zacks.com</td>\n",
       "      <td>[{'topic': 'Earnings', 'relevance_score': '0.9...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24439</th>\n",
       "      <td>Data Center Interconnect Market size worth $ 3...</td>\n",
       "      <td>https://www.benzinga.com/pressreleases/24/04/g...</td>\n",
       "      <td>20240422T140000</td>\n",
       "      <td>['Globe Newswire']</td>\n",
       "      <td>Jersey City, New Jersey, April 22, 2024 ( GLOB...</td>\n",
       "      <td>https://www.benzinga.com/next-assets/images/sc...</td>\n",
       "      <td>Benzinga</td>\n",
       "      <td>General</td>\n",
       "      <td>www.benzinga.com</td>\n",
       "      <td>[{'topic': 'Financial Markets', 'relevance_sco...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24440</th>\n",
       "      <td>Is Nvidia Getting Ready to Disrupt Another Com...</td>\n",
       "      <td>https://www.fool.com/investing/2024/04/22/is-n...</td>\n",
       "      <td>20240422T123900</td>\n",
       "      <td>['Nicholas Rossolillo']</td>\n",
       "      <td>Some investors were second-guessing the streng...</td>\n",
       "      <td>https://g.foolcdn.com/editorial/images/773500/...</td>\n",
       "      <td>Motley Fool</td>\n",
       "      <td>NaN</td>\n",
       "      <td>www.fool.com</td>\n",
       "      <td>[{'topic': 'Financial Markets', 'relevance_sco...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24441</th>\n",
       "      <td>Biden Cabinet Member Isn't Impressed With Huaw...</td>\n",
       "      <td>https://www.benzinga.com/news/24/04/38350312/b...</td>\n",
       "      <td>20240422T040220</td>\n",
       "      <td>['Ananya Gairola']</td>\n",
       "      <td>U.S. Commerce Secretary Gina Raimondo dismisse...</td>\n",
       "      <td>https://cdn.benzinga.com/files/images/story/20...</td>\n",
       "      <td>Benzinga</td>\n",
       "      <td>News</td>\n",
       "      <td>www.benzinga.com</td>\n",
       "      <td>[{'topic': 'Manufacturing', 'relevance_score':...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24442 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title  \\\n",
       "0      Apple, AMC, Meta, And How Elon Musk Is Reactin...   \n",
       "1      Bulls In A Bear Market: These 10 Stocks Clocke...   \n",
       "2         Where Will Unity Software Stock Be in 3 Years?   \n",
       "3      3 Unstoppable Growth Stocks to Buy After a Sto...   \n",
       "4      Even in an Advertising Slowdown, These 3 Stock...   \n",
       "...                                                  ...   \n",
       "24437  Nvidia's High-End Chips Ended Up In Chinese Ha...   \n",
       "24438  Broadcom Inc.  ( AVGO )  Gains But Lags Market...   \n",
       "24439  Data Center Interconnect Market size worth $ 3...   \n",
       "24440  Is Nvidia Getting Ready to Disrupt Another Com...   \n",
       "24441  Biden Cabinet Member Isn't Impressed With Huaw...   \n",
       "\n",
       "                                                     url   time_published  \\\n",
       "0      https://www.benzinga.com/news/large-cap/22/12/...  20221231T190310   \n",
       "1      https://www.benzinga.com/analyst-ratings/analy...  20221231T161114   \n",
       "2      https://www.fool.com/investing/2022/12/31/wher...  20221231T154500   \n",
       "3      https://www.fool.com/investing/2022/12/31/3-un...  20221231T140000   \n",
       "4      https://www.fool.com/investing/2022/12/31/even...  20221231T120000   \n",
       "...                                                  ...              ...   \n",
       "24437  https://www.benzinga.com/markets/asia/24/04/38...  20240423T050111   \n",
       "24438  https://www.zacks.com/stock/news/2259620/broad...  20240422T214516   \n",
       "24439  https://www.benzinga.com/pressreleases/24/04/g...  20240422T140000   \n",
       "24440  https://www.fool.com/investing/2024/04/22/is-n...  20240422T123900   \n",
       "24441  https://www.benzinga.com/news/24/04/38350312/b...  20240422T040220   \n",
       "\n",
       "                         authors  \\\n",
       "0              ['Michael Cohen']   \n",
       "1           ['Shanthi Rexaline']   \n",
       "2                    ['Leo Sun']   \n",
       "3                  ['Dani Cook']   \n",
       "4                  ['Adam Levy']   \n",
       "...                          ...   \n",
       "24437         ['Benzinga Neuro']   \n",
       "24438  ['Zacks Equity Research']   \n",
       "24439         ['Globe Newswire']   \n",
       "24440    ['Nicholas Rossolillo']   \n",
       "24441         ['Ananya Gairola']   \n",
       "\n",
       "                                                 summary  \\\n",
       "0      Benzinga examined the prospects for many inves...   \n",
       "1      2022 would go down as one of the worst years f...   \n",
       "2      The game-engine developer still has a lot to p...   \n",
       "3      These companies have grown by triple-digit per...   \n",
       "4      Ad sales growth will slow in 2023, but these s...   \n",
       "...                                                  ...   \n",
       "24437  Chinese entities have managed to acquire high-...   \n",
       "24438  In the most recent trading session, Broadcom I...   \n",
       "24439  Jersey City, New Jersey, April 22, 2024 ( GLOB...   \n",
       "24440  Some investors were second-guessing the streng...   \n",
       "24441  U.S. Commerce Secretary Gina Raimondo dismisse...   \n",
       "\n",
       "                                            banner_image            source  \\\n",
       "0      https://cdn.benzinga.com/files/images/story/20...          Benzinga   \n",
       "1      https://cdn.benzinga.com/files/images/story/20...          Benzinga   \n",
       "2      https://g.foolcdn.com/image/?url=https%3A%2F%2...       Motley Fool   \n",
       "3      https://g.foolcdn.com/image/?url=https%3A%2F%2...       Motley Fool   \n",
       "4      https://g.foolcdn.com/image/?url=https%3A%2F%2...       Motley Fool   \n",
       "...                                                  ...               ...   \n",
       "24437  https://cdn.benzinga.com/files/images/story/20...          Benzinga   \n",
       "24438  https://staticx-tuner.zacks.com/images/default...  Zacks Commentary   \n",
       "24439  https://www.benzinga.com/next-assets/images/sc...          Benzinga   \n",
       "24440  https://g.foolcdn.com/editorial/images/773500/...       Motley Fool   \n",
       "24441  https://cdn.benzinga.com/files/images/story/20...          Benzinga   \n",
       "\n",
       "      category_within_source     source_domain  \\\n",
       "0                    Trading  www.benzinga.com   \n",
       "1                    General  www.benzinga.com   \n",
       "2                        NaN      www.fool.com   \n",
       "3                        NaN      www.fool.com   \n",
       "4                        NaN      www.fool.com   \n",
       "...                      ...               ...   \n",
       "24437                Markets  www.benzinga.com   \n",
       "24438                    NaN     www.zacks.com   \n",
       "24439                General  www.benzinga.com   \n",
       "24440                    NaN      www.fool.com   \n",
       "24441                   News  www.benzinga.com   \n",
       "\n",
       "                                                  topics  ...  \\\n",
       "0      [{'topic': 'Financial Markets', 'relevance_sco...  ...   \n",
       "1      [{'topic': 'Life Sciences', 'relevance_score':...  ...   \n",
       "2      [{'topic': 'IPO', 'relevance_score': '0.769861...  ...   \n",
       "3      [{'topic': 'Financial Markets', 'relevance_sco...  ...   \n",
       "4      [{'topic': 'Earnings', 'relevance_score': '0.9...  ...   \n",
       "...                                                  ...  ...   \n",
       "24437  [{'topic': 'Mergers & Acquisitions', 'relevanc...  ...   \n",
       "24438  [{'topic': 'Earnings', 'relevance_score': '0.9...  ...   \n",
       "24439  [{'topic': 'Financial Markets', 'relevance_sco...  ...   \n",
       "24440  [{'topic': 'Financial Markets', 'relevance_sco...  ...   \n",
       "24441  [{'topic': 'Manufacturing', 'relevance_score':...  ...   \n",
       "\n",
       "      Retail & Wholesale Economy - Monetary Life Sciences  \\\n",
       "0                    NaN                NaN           NaN   \n",
       "1                    NaN                NaN      0.166667   \n",
       "2                    NaN                NaN           NaN   \n",
       "3                    NaN                NaN           NaN   \n",
       "4                    NaN                NaN           NaN   \n",
       "...                  ...                ...           ...   \n",
       "24437                NaN                NaN           NaN   \n",
       "24438                NaN                NaN           NaN   \n",
       "24439                NaN                NaN           NaN   \n",
       "24440                NaN                NaN           NaN   \n",
       "24441                NaN                NaN           NaN   \n",
       "\n",
       "      Energy & Transportation Real Estate & Construction Blockchain       IPO  \\\n",
       "0                    0.333333                        NaN        NaN       NaN   \n",
       "1                    0.166667                   0.166667        NaN  0.158519   \n",
       "2                         NaN                        NaN        NaN  0.769861   \n",
       "3                         NaN                        NaN        NaN       NaN   \n",
       "4                         NaN                        NaN   0.158519       NaN   \n",
       "...                       ...                        ...        ...       ...   \n",
       "24437                     NaN                        NaN        NaN       NaN   \n",
       "24438                     NaN                        NaN        NaN       NaN   \n",
       "24439                     NaN                        NaN        NaN       NaN   \n",
       "24440                     NaN                        NaN        NaN       NaN   \n",
       "24441                     NaN                        NaN        NaN       NaN   \n",
       "\n",
       "       Mergers & Acquisitions  Economy - Fiscal  Economy - Macro  \n",
       "0                         NaN               NaN              NaN  \n",
       "1                         NaN               NaN              NaN  \n",
       "2                    0.360215               NaN              NaN  \n",
       "3                         NaN               NaN              NaN  \n",
       "4                         NaN               NaN         0.158519  \n",
       "...                       ...               ...              ...  \n",
       "24437                0.108179               NaN              NaN  \n",
       "24438                     NaN               NaN              NaN  \n",
       "24439                     NaN               NaN              NaN  \n",
       "24440                     NaN               NaN              NaN  \n",
       "24441                     NaN               NaN              NaN  \n",
       "\n",
       "[24442 rows x 32 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "topics = ['Technology', 'Financial Markets', 'Manufacturing', 'Earnings', 'Finance', 'Retail & Wholesale', 'Economy - Monetary', 'Life Sciences', 'Energy & Transportation', 'Real Estate & Construction', 'Blockchain', 'IPO', 'Mergers & Acquisitions', 'Economy - Fiscal', 'Economy - Macro']\n",
    "\n",
    "# Initialize columns for each topic with NaNs\n",
    "for topic in topics:\n",
    "    df[topic] = np.nan\n",
    "\n",
    "# Populate the topic columns with relevance scores\n",
    "for index, row in df.iterrows():\n",
    "    if row['parsed_topics'] is not None:\n",
    "        for item in row['parsed_topics']:\n",
    "            topic = item['topic']\n",
    "            relevance_score = float(item['relevance_score'])  # Convert string to float if necessary\n",
    "            if topic in df.columns:\n",
    "                df.at[index, topic] = relevance_score\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "34c7c5dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[topics] = df[topics].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "6ea33ea6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        20221231\n",
       "1        20221231\n",
       "2        20221231\n",
       "3        20221231\n",
       "4        20221231\n",
       "           ...   \n",
       "24437    20240423\n",
       "24438    20240422\n",
       "24439    20240422\n",
       "24440    20240422\n",
       "24441    20240422\n",
       "Name: date, Length: 24442, dtype: object"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['date']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "8540d663",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['date'] = pd.to_datetime(df['date'].astype(str), format='%Y%m%d', errors ='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "2fca4a77",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.dropna(subset='date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "ac7a2c89",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"news_data_processed.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "1881751d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['title', 'url', 'time_published', 'authors', 'summary', 'banner_image',\n",
       "       'source', 'category_within_source', 'source_domain', 'topics',\n",
       "       'overall_sentiment_score', 'overall_sentiment_label',\n",
       "       'ticker_sentiment', 'date', 'embeddings', 'sentiment_scores',\n",
       "       'parsed_topics', 'Technology', 'Financial Markets', 'Manufacturing',\n",
       "       'Earnings', 'Finance', 'Retail & Wholesale', 'Economy - Monetary',\n",
       "       'Life Sciences', 'Energy & Transportation',\n",
       "       'Real Estate & Construction', 'Blockchain', 'IPO',\n",
       "       'Mergers & Acquisitions', 'Economy - Fiscal', 'Economy - Macro',\n",
       "       'date_formatted'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92188594",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}