parsing_rss_feeds.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests as req\n",
    "import feedparser\n",
    "from bs4 import BeautifulSoup\n",
    "# import time\n",
    "# from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "resp = req.get('https://www.business-standard.com/rss-feeds/listing/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(resp.text, 'lxml').body"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "feeds = [f\"http://www.business-standard.com{a['href']}\" for a in soup.select('td a[href]') \n",
    "         if not a['href'].startswith('h')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "303"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(feeds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# takes around 2 - 5 minutes, depending on your internet speed\n",
    "# could be made faster with async\n",
    "\n",
    "parsed_feeds = list(map(feedparser.parse, feeds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "links = []\n",
    "last_updated = []\n",
    "all_entries = []\n",
    "counter = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx, feed in enumerate(parsed_feeds):\n",
    "    last_updated.append(\n",
    "        (feeds[idx], feed.get('updated', ''))\n",
    "    )\n",
    "    \n",
    "    for _, i in enumerate(feed.entries):\n",
    "        counter += 1\n",
    "        \n",
    "        links.append(i['link'])\n",
    "        \n",
    "        all_entries.append((\n",
    "            counter, i['link'], \n",
    "            i['title'], \n",
    "            i['summary'], \n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "last_updated.sort(key=lambda x: x[1], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6125"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "303"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(last_updated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1,\n",
       "  'https://www.business-standard.com/article/markets/jhunjhunwala-fpis-mfs-hike-stake-in-rallis-in-q4-stock-up-32-in-2020-120050400605_1.html',\n",
       "  'Jhunjhunwala, FPIs, MFs hike stake in Rallis India; stock up 32% in 2020',\n",
       "  'Jhunjhunwala acquired an additional 275,000 shares of Rallis India during the quarter, taking his stake in the company to almost 10 per cent mark'),\n",
       " (2,\n",
       "  'https://www.business-standard.com/article/defence/col-ashutosh-sharma-fulfilled-his-dream-of-joining-army-in-13th-attempt-120050400561_1.html',\n",
       "  'Col Ashutosh Sharma fulfilled his dream of joining Army in 13th attempt',\n",
       "  'Col Sharma was among five security forces personnel killed during an encounter with terrorists in north Kashmir late on Saturday night'),\n",
       " (3,\n",
       "  'https://www.business-standard.com/article/companies/no-sign-of-bailout-from-govt-yet-go-air-promoters-tell-employees-120050400562_1.html',\n",
       "  'No sign of bailout from govt yet, Go Air promoters tell employees',\n",
       "  'The low-cost airline has sent 90% of its employees on leave without pay till 4 May'),\n",
       " (4,\n",
       "  'https://www.business-standard.com/article/markets/sino-us-trade-war-pmi-numbers-5-factors-that-triggered-market-fall-today-120050400550_1.html',\n",
       "  \"Sino-US trade war, PMI numbers: Five factors behind today's market fall\",\n",
       "  'Volatility index, India VIX, saw a sharp surge of 27 per cent to 43.18 levels, suggesting heightened volatility in the markets.'),\n",
       " (5,\n",
       "  'https://www.business-standard.com/article/markets/sino-us-trade-war-pmi-numbers-5-factors-that-triggered-market-fall-today-120050400550_1.html',\n",
       "  \"Sino-US trade war, PMI numbers: Five factors behind today's market fall\",\n",
       "  'Volatility index, India VIX, saw a sharp surge of 27 per cent to 43.18 levels, suggesting heightened volatility in the markets.')]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_entries[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('http://www.business-standard.com/rss/companies-people-in-news-10109.rss',\n",
       "  'Wed, 24 Apr 2019 13:15:30 GMT'),\n",
       " ('http://www.business-standard.com/rss/income-tax-182.rss',\n",
       "  'Wed, 24 Apr 2019 13:15:27 GMT'),\n",
       " ('http://www.business-standard.com/rss/budget2018-188.rss',\n",
       "  'Wed, 24 Apr 2019 13:15:27 GMT'),\n",
       " ('http://www.business-standard.com/rss/punditry-174.rss',\n",
       "  'Wed, 24 Apr 2019 13:15:26 GMT'),\n",
       " ('http://www.business-standard.com/rss/events-175.rss',\n",
       "  'Wed, 24 Apr 2019 13:15:26 GMT')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_updated[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://www.business-standard.com/article/markets/bharti-airtel-surges-11-in-3-days-after-signing-4g-network-deal-with-nokia-120050400523_1.html',\n",
       " 'https://www.business-standard.com/article/international/will-cancel-trade-deal-if-china-fails-to-buy-goods-worth-200-bn-trump-120050400499_1.html',\n",
       " 'https://www.business-standard.com/article/companies/petrol-sales-fall-61-diesel-down-by-57-in-april-due-to-lockdown-report-120050400452_1.html',\n",
       " 'https://www.business-standard.com/article/current-affairs/raliway-board-in-a-huddle-after-cong-offers-to-pay-for-travel-of-migrants-120050400354_1.html',\n",
       " 'https://www.business-standard.com/article/markets/pharma-shares-trade-firm-in-weak-market-cipla-lupin-gain-over-2-120050400325_1.html']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "links[5:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}