WebScraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# not important for scrapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pymongo\n",
    "import json\n",
    "from threading import Thread\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime.time(20, 43, 38, 533681)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "datetime.now().time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "from fake_useragent import UserAgent\n",
    "#date parser\n",
    "from dateutil import parser as date_parser\n",
    "\n",
    "import random\n",
    "\n",
    "#creatinf fake user agent and header \n",
    "user_agent  = UserAgent()\n",
    "\n",
    "header ={\"user-agent\":user_agent.random}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://english.onlinekhabar.com/feed\n",
      "https://thehimalayantimes.com/feed/\n",
      "https://setopati.net/feed\n"
     ]
    }
   ],
   "source": [
    "urls = {\"onlinekhabar\":\"http://english.onlinekhabar.com/feed\",\n",
    "        \"the himalayan times\":\"https://thehimalayantimes.com/feed/\",\n",
    "        \"setopati\":\"https://setopati.net/feed\"}\n",
    "for url in urls:\n",
    "    print(urls[url])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "url  = \"https://thehimalayantimes.com/feed/\"\n",
    "\n",
    "# url = \"https://setopati.net/feed\"\n",
    "response =  requests.get(url,headers=header)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 200 is success\n",
    "# 404 is connection error\n",
    "# 403 is forbidden\n",
    "response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\"?><html><body><rss version=\"2.0\" xmlns:atom=\"http://www.w3.org/2005/Atom\" xmlns:content=\"http://purl.org/rss/1.0/modules/content/\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:slash=\"http://purl.org/rss/1.0/modules/slash/\" xmlns:sy=\"http://purl.org/rss/1.0/modules/syndication/\" xmlns:wfw=\"http://wellformedweb.org/CommentAPI/\">\n",
       "<channel>\n",
       "<title>Setopati – Nepal's Digital Newspaper</title>\n",
       "<atom:link href=\"https://setopati.net/feed\" rel=\"self\" type=\"application/rss+xml\"></atom:link>\n",
       "<link/>https://setopati.net\n",
       "\t<description></description>\n",
       "<lastbuilddate>Thu, 06 Jun 2019 06:53:01 +0000</lastbuilddate>\n",
       "<language>en-US</language>\n",
       "<sy:updateperiod>hourly</sy:updateperiod>\n",
       "<sy:updatefrequency>1</sy:updatefrequency>\n",
       "<generator>https://wordpress.org/?v=4.9.10</generator>\n",
       "<item>\n",
       "<title>Doctor accused of murdering 25 patient with painkiller overdose</title>\n",
       "<link/>https://setopati.net/social/147639\n",
       "\t\t<comments>https://setopati.net/social/147639#respond</comments>\n",
       "<pubdate>Thu, 06 Jun 2019 06:53:01 +0000</pubdate>\n",
       "<dc:creator></dc:creator>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<guid ispermalink=\"false\">https://setopati.net/?p=147639</guid>\n",
       "<description></description>\n",
       "<content:encoded>An Ohio doctor was charged with murder Wednesday in the deaths of 25 hospital patients who, authorities say, were killed with deliberate overdoses of painkillers, many of them administered by other medical workers on his orders.\n",
       "<p>In one of the biggest cases of its kind ever brought against an American health care professional, William Husel was accused of ordering outsize doses of the powerful painkiller fentanyl. Many of the patients who died were on ventilators and receiving palliative care. The deaths occurred between 2015 and 2018.</p>\n",
       "<p>Franklin County Prosecutor Ron O’Brien compared Husel’s actions to extinguishing a dwindling candle.</p>\n",
       "<p>“That candle, while there may be just a half an inch of wax left, if I blow that candle out, I’m causing that flame to go out sooner than it would naturally,” O’Brien said.</p>\n",
       "<p>Husel, 43, pleaded not guilty after turning himself in earlier in the day. A judge set bail at $1 million.</p>\n",
       "<p>The doctor is the lone defendant. Authorities are not prosecuting nurses, pharmacists and others involved in the deaths, though dozens of hospital employees have been reported to professional boards for investigation and potential disciplinary action.</p>\n",
       "<p>Husel’s lawyer said he was trying to provide “comfort care” for dying patients.</p>\n",
       "<p>“At no time did Dr. Husel ever intend to euthanize anyone — euthanize meaning speed up death,” defense attorney Richard Blake said.</p>\n",
       "<p>The patients were going to die whether they were being treated by Husel or another physician, Blake said.</p>\n",
       "<p>The Columbus-area Mount Carmel Health System has publicly apologized. It issued a statement Wednesday pledging to continue cooperating with authorities and making “meaningful changes” to ensure such events never happen again.</p>\n",
       "<p>The system found that Husel ordered potentially fatal drug doses for 29 patients, including five who might have received those drugs when there still was a chance to improve their conditions with treatment. The hospital system said six more patients got doses that were excessive but probably did not cause their deaths.</p>\n",
       "<p>The murder charges were brought only in cases that involved fentanyl doses of at least 500 micrograms. The prosecutor said the investigation remains open and other cases are still under review.</p>\n",
       "<p>Husel was fired in December and stripped of his medical license after concerns about his orders were brought to the attention of officials at Mount Carmel, where he had worked for five years.</p>\n",
       "<p>Mount Carmel has said it should have investigated and taken action sooner. It has acknowledged that the doctor was not removed from patient care for four weeks after the concerns were raised, and three patients died during that time.</p>\n",
       "<p>Police Sgt. Terry McConnell said none of the families who talked with investigators believed that what happened was “mercy treatment.”</p>\n",
       "<p>Amy Pfaff, whose mother was among the patients whose deaths prompted the charges, said she still wonders about his motives.</p>\n",
       "<p>“Trust me, I sit many hours sitting trying to figure out why would he do this to so many people, and I just don’t know,” Pfaff said.</p>\n",
       "<p>More than two dozen wrongful-death lawsuits have been filed against the doctor and the hospital system, including one by Pfaff over the October 2017 death of her mother, Beverlee Schirtzinger.</p>\n",
       "<p>The hospital system settled some of the cases for hundreds of thousands of dollars.</p>\n",
       "<p>All employees who had a role in administering medication to the victims have been removed from patient care as a precaution, hospital officials have said.</p>\n",
       "<p>All told, 48 nurses and pharmacists <a href=\"https://apnews.com/ab3a02dffb17476eaebb17d2c2ad3dd4\">were reported</a> to their respective professional boards. Thirty of those employees were put on leave, and 18 no longer work there, including some who left years ago, officials said.</p>\n",
       "<p>Records show no prior disciplinary action against Husel by the Ohio State Medical Board. The board will not disclose whether it received any complaints that did not result in action.</p>\n",
       "<p>The allegations against Husel recalled another Ohio case involving a former nurse’s aide dubbed the Angel of Death. That man, Donald Harvey, confessed in 1987 to killing 37 people, most of them hospital patients, over the span of two decades in Ohio and Kentucky. He was given multiple life sentences and died in 2017 after being attacked by a fellow inmate.</p>\n",
       "]]&gt;</content:encoded>\n",
       "<wfw:commentrss>https://setopati.net/social/147639/feed</wfw:commentrss>\n",
       "<slash:comments>0</slash:comments>\n",
       "</item>\n",
       "<item>\n",
       "<title>China sends back Nepali workers from Taklakot after murder of Chinese woman</title>\n",
       "<link/>https://setopati.net/social/147634\n",
       "\t\t<comments>https://setopati.net/social/147634#respond</comments>\n",
       "<pubdate>Thu, 06 Jun 2019 06:41:57 +0000</pubdate>\n",
       "<dc:creator></dc:creator>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<guid ispermalink=\"false\">https://setopati.net/?p=147634</guid>\n",
       "<description></description>\n",
       "<content:encoded>China has restricted entry of locals from Humla after murder of a Chinese woman.<br/>\n",
       "Tirtha Khadera of Sarkideu in Sarkegad rural municipality was arrested in Taklakot of Tibet where he had gone to work on charge of murdering a Chinese woman in the third week of May, according to Assistant Chief District Officer (CDO) of Humla Mahesh Kumar Pokharel.<br/>\n",
       "Chinese police started to send back Nepali workers going to Taklakot for work after that.<br/>\n",
       "“We have arrested him after he murdered a woman going to bank by hitting her with a rod. We are taking him to Lhasa for additional investigation,” Pokharel quoted the Chinese authorities as saying after Khadera’s arrest. “They had not talked about sending other workers back. But they started to send back Nepali workers from the next day,” he told Setopati.<br/>\n",
       "He revealed that almost 500 Nepali workers have been sent back in three weeks.<br/>\n",
       "Nepal-China treaty allows citizens of both the countries to travel up to 30 kilometers inside the other country by producing a pass. Nepalis in Humla go to Taklakot to work and for business using the facility.<br/>\n",
       "There are jobs in Taklakot during the season of Mansarovar pilgrimage from May till September. Around 5,000-7,000 Nepalis go to Taklakot during the season every year to work in construction projects, hotels and for other works.<br/>\n",
       "“It is a good source of income as one could earn up to Rs 5,000 a day for such seasonal works. But there is problem this year,” Pokharel stated. “We had informed the Foreign Ministry through the Home Ministry after the Chinese side started to send Nepali workers back. They have not sent back Nepali workers n the past two days apparently after diplomatic initiative.<br/>\n",
       "Locals in Taklakot have been warning they will not allow Nepalis to work from now onward even though the authorities have stopped sending the workers back.<br/>\n",
       "“W e do not have additional information about Khadera. We have proposed to hold a border meeting. The meeting will be convened by June end. We will put our concerns during the meeting,” Pokharel added.<br/>\n",
       "Secretary at the Foreign Ministry Shankar Das Bairagi, meanwhile, has said the government is taking diplomatic initiative to resolve the problems seen in Taklakot.\n",
       "]]&gt;</content:encoded>\n",
       "<wfw:commentrss>https://setopati.net/social/147634/feed</wfw:commentrss>\n",
       "<slash:comments>0</slash:comments>\n",
       "</item>\n",
       "<item>\n",
       "<title>NC protests saying CA medal has sun</title>\n",
       "<link/>https://setopati.net/political/147629\n",
       "\t\t<comments>https://setopati.net/political/147629#respond</comments>\n",
       "<pubdate>Wed, 05 Jun 2019 12:12:45 +0000</pubdate>\n",
       "<dc:creator></dc:creator>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<guid ispermalink=\"false\">https://setopati.net/?p=147629</guid>\n",
       "<description></description>\n",
       "<content:encoded>Nepali Congress (NC) has opposed distribution of medals to members of the first and second Constituent Assembly (CA) by President Bidya Devi Bhandari.<br/>\n",
       "“The style of distribution of identifying medals to those who made the Constitution is like thanking them. We don’t like that,” NC lawmaker Minendra Rijal said during the zero hour in the House of Representatives (HoR) on Wednesday. “There are talks that the artistic display of lines carved to support the Constitution in the medal have given other kind of message about Constitution promulgation and ownership on top of that.”<br/>\n",
       "“There are public grievances that the style of handing it and the design of medal have not been able to imbibe the spirit of the struggle filled with sacrifices and succession. I want to draw the attention of everyone concerned,” he stated.<br/>\n",
       "Rijal has yet to receive his medal. “Medals should not have been distributed in a way to thank for formulating Constitution. I don’t even like the medal’s design. The symbol of sun is there,” he listed the reasons for not receiving the medal.<br/>\n",
       "“I have heard that Rs 175,000 has been spent on one medal. Why should we distribute medals by spending so much from the state coffers?” he added. “The symbol of sun is kept in a way it is clearly visible. Can a single party monopolize the Constitution? Act as if it solely formulated the Constitution.”<br/>\n",
       "Sun is the election symbol of ruling CPN.<br/>\n",
       "“I have already briefed NC President Sher Bahadur Deuba about this. The party should discuss about the medal and take decision. If somebody has received, it should be returned,” he stressed.<br/>\n",
       "“BP (Koirala) did not receive any medal during his lifetime. Girija Prasad Koirala also did not receive when he was alive. Should we followers of BP receive medals?” he asked.<br/>\n",
       "He also opined that the handing out of medals is not in accordance to the spirit of republicanism. “Handing out medals is residue of feudal attitude. I did not like the style in which president handed over the medals.”<br/>\n",
       "NC lawmaker Gagan Thapa also has not received the medal. “We are people’s representatives. Should we attend parties by donning that medal? Our mindset is not republican. The people will respect us for promulgating the Constitution in itself. The CA gave us CA members logo. That suffices,” Thapa stated.<br/>\n",
       "Joint Secretary in the internal management division of the Home Ministry, that designed the medals, Mahadev Panta, however, said there is no sun in the medal. “There are 77 stars in the outermost circle. That depict 77 districts. There are seven rhododendrons inside which depict seven provinces,” he elaborated. “The straight lines are local bodies. They depict 753 local bodies.”<br/>\n",
       "He claimed that the lines that look like sun do not make a sun. “The lines that depict local bodies are golden. That means the future of local bodies is bright. That is not sun.”\n",
       "]]&gt;</content:encoded>\n",
       "<wfw:commentrss>https://setopati.net/political/147629/feed</wfw:commentrss>\n",
       "<slash:comments>0</slash:comments>\n",
       "</item>\n",
       "<item>\n",
       "<title>CPN discusses division of responsibilities</title>\n",
       "<link/>https://setopati.net/political/147625\n",
       "\t\t<comments>https://setopati.net/political/147625#respond</comments>\n",
       "<pubdate>Wed, 05 Jun 2019 06:39:01 +0000</pubdate>\n",
       "<dc:creator></dc:creator>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<guid ispermalink=\"false\">https://setopati.net/?p=147625</guid>\n",
       "<description></description>\n",
       "<content:encoded>Secretariat meeting of the ruling CPN has started at the prime minister’s residence in Baluwatar on Wednesday to discuss division of responsibilities among the central members and appointment of incharge and deputy incharge of districts.<br/>\n",
       "CPN Spokesperson Narayan Kaji Shrestha said the meeting will resume from the point at which it was stopped on Tuesday.<br/>\n",
       "The meeting on Tuesday was unable to conclude division of responsibilities after Chairman KP Sharma Oli and senior leader Madhav Kumar Nepal differed on the issue of providing dual responsibilities to leaders.<br/>\n",
       "Senior leaders Nepal, Jhala Nath Khanal and Bam Dev Gautam are firm that those who are ministers, lawmakers and elected local representatives should not be given another responsibility.<br/>\n",
       "Prime Minister (PM) Oli, General Secretary Bishnu Paudel and Ishwore Pokharel say providing party responsibility to such leaders will not make any difference.<br/>\n",
       "Leaders from the erstwhile CPN (Maoist Center) were almost silent during the secretariat meeting on Tuesday after the leaders from the former CPN-UML differed on the issue.<br/>\n",
       "Leaders say a middle way agreeable to both the sides would be sought on Wednesday.<br/>\n",
       "Chairmen Oli and Pushpa Kamal Dahal, Paudel and senior leader Nepal had also met at the prime minister’s residence in Baluwatar earlier on Tuesday to decide about department heads. But the four leaders could not forge consensus about the department heads.<br/>\n",
       "CPN has not been able to decide about leadership of different departments including organization, finance, publicity and party school with multiple leaders wishing for single position. The party also has yet to form the politburo.\n",
       "]]&gt;</content:encoded>\n",
       "<wfw:commentrss>https://setopati.net/political/147625/feed</wfw:commentrss>\n",
       "<slash:comments>0</slash:comments>\n",
       "</item>\n",
       "<item>\n",
       "<title>Scientist warns of Everest dangers from pollution, melting</title>\n",
       "<link/>https://setopati.net/social/147620\n",
       "\t\t<comments>https://setopati.net/social/147620#respond</comments>\n",
       "<pubdate>Wed, 05 Jun 2019 01:39:42 +0000</pubdate>\n",
       "<dc:creator></dc:creator>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<guid ispermalink=\"false\">https://setopati.net/?p=147620</guid>\n",
       "<description></description>\n",
       "<content:encoded>Mount Everest and its surrounding peaks are increasingly polluted and warmer, and nearby glaciers are melting at an alarming rate that is likely to make it more dangerous for future climbers, a U.S. scientist who spent weeks in the Everest region said Tuesday.\n",
       "<p>Professor John All of Western Washington University said after returning from the mountains that he and his team of fellow scientists found there was lot of pollution buried deep in the snow, and that the snow was surprisingly dark when they processed and filtered it.</p>\n",
       "<p>“What that means is there are little pieces of pollution that the snow is forming around, so the snow is actually trapping the pollution and pulling it down,” All said in Kathmandu, Nepal’s capital.</p>\n",
       "<p>All and his team spent weeks testing snow on Everest and its surrounding peaks, as well as plants on the foothills.</p>\n",
       "<p>“The warming temperature is melting the glaciers and the snow around Mount Everest very quickly, so what happens is even when there is a storm it melts in a couple of hours,” he said. “The glaciers are retreating dramatically because of global warming.”</p>\n",
       "<p>He said because the glaciers are getting thinner and smaller, it is making it more dangerous for climbers.</p>\n",
       "<p>The team had been planning to climb both Everest and sister peak Lhotse, but crowding on Everest forced them to change their plans. They climbed up to the last camp at 8,000 meters (26,240 feet), the last point the two mountains share, and only reached the top of Lhotse.</p>\n",
       "<p>Hundreds of climbers had lined up on May 22 and 23 to attempt to reach Everest’s summit, creating a traffic jam that is being blamed for the deaths of several climbers.</p>\n",
       "<p>All said it was too risky for his team to collect samples with that many climbers moving very slowly.</p>\n",
       "<p>The scientists said the samples and data would be processed once they return to United States, and they would then issue a report on their findings. They had done similar research in the area in 2009.</p>\n",
       "<p>“Overall, the past 10 years have seen a lot changes in the mountains, and they all have been for the negative environmentally in terms of long term survivability of the glaciers,” All said.</p>\n",
       "]]&gt;</content:encoded>\n",
       "<wfw:commentrss>https://setopati.net/social/147620/feed</wfw:commentrss>\n",
       "<slash:comments>0</slash:comments>\n",
       "</item>\n",
       "<item>\n",
       "<title>Hearing on Ncell tax case to continue on June 25</title>\n",
       "<link/>https://setopati.net/market/147613\n",
       "\t\t<comments>https://setopati.net/market/147613#respond</comments>\n",
       "<pubdate>Tue, 04 Jun 2019 12:19:45 +0000</pubdate>\n",
       "<dc:creator></dc:creator>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<guid ispermalink=\"false\">https://setopati.net/?p=147613</guid>\n",
       "<description></description>\n",
       "<content:encoded>The hearing on case about capital gain tax to be recovered from telecom service provider Ncell A full bench of the Supreme Court (SC) will continue on June 25.<br/>\n",
       "A full bench including Justices Tej Bahadur KC, Purushottam Bhandari, Dambar Bahadur Shahi, Sushma Lata Mathema and Manoj Kumar Sharma started hearing in the case on Tuesday. Different advocates pleaded on behalf of Ncell on Tuesday. The full bench has decided to conduct the next hearing three weeks to the day on June 25.<br/>\n",
       "A joint bench of Justices Sapana Pradhan Malla and Prakash Kumar Dhungana on May 7 had ordered the Large Taxpayers Office to not immediately recover the capital gain tax from Ncell and called for convening of a full bench to decide on the petition.<br/>\n",
       "A single bench of Justice Bam Kumar Shrestha hearing the petition on April 25 had issued a show cause notice in the name of the office to explain the reason for determining the tax amount and stopped immediate recovery of tax.<br/>\n",
       "Ncell had filed the petition against the tax determination a day before meeting the seven-day deadline issued by the Large Taxpayers Office to pay the capital gain tax.<br/>\n",
       "The Taxpayers Office on April 16 had determined that Ncell should pay Rs 39.06 billion more in capital gain tax. It had sent a notice to Ncell to pay the tax within seven days. Ncell, that has already deposited Rs 23.57 billion in two installments, has claimed that it should pay only Rs 14.365 billion more.<br/>\n",
       "It had pointed that the SC has just ordered to recover capital gain tax, and not interest and late fine. It added that it should pay 25 percent of the capital gain of Rs 143.65 billion during the sale which comes out at Rs 35.91 billion. It has contended that it, therefore, should pay only Rs 14.365 billion more as it has already deposited Rs 23.57 billion.<br/>\n",
       "Swedish-Finnish company TeliaSonera has not paid capital gain tax for sale of its 80% stake in Ncell to Malaysian company Axiata in April 2015. The Large Taxpayers Office earlier had determined that TeliaSonera had made capital gain of Rs 143.65 billion during the sale.<br/>\n",
       "The extended full bench of Supreme Court (SC) headed by Cholendra Shumsher Rana on February 6 had ordered that the capital gain tax be recovered from Ncell and Axiata.<br/>\n",
       "The Large Taxpayers Office issuing a notice on Tuesday had determined that Ncell must pay Rs 62.63 billion in capital gain tax including interest until now. It had told Ncell to pay the remaining Rs 39.06 billion within seven days.<br/>\n",
       "The full bench including Justices Meera Khadka, Bishwambhar Shrestha, Ananda Mohan Bhattarai and Tanka Moktan hearing a petition filed by Dwarika Nath Dhungel demanding the capital gain tax be recovered from Ncell on February 6 had issued a mandamus ruling that Ncell and Axiata should pay the tax.<br/>\n",
       "The mandamus reasoned that Ncell has to pay capital gain tax to the Nepal government despite TeliaSonera selling the shares to Axiata as its business is based in Nepal.<br/>\n",
       "The full judgement of the mandamus published on April 9 had ordered the government to recover the tax from the telecommunication company and its majority shareholder Axiata within three months.<br/>\n",
       "Malaysian company Axiata had bought Reynolds Holding, which held a majority stake in Ncell, from the Swedish-Finnish company TeliaSonera in April 2016. Reynolds Holding was TeliaSonera’s wholly-owned subsidiary, registered at Saint Kitts and Nevis, a tax haven.<br/>\n",
       "Though capital gains tax needs to be paid by the company that gains from the deal, TeliaSonera has already exited Nepal.<br/>\n",
       "Ncell has already deposited Rs 23.57 billion in two installments as tax applicable on the profit generated through sale of the telecom company. It had first paid Rs 9.97 billion in May, 2016 in tax return based on its own calculations. It had then paid Rs 13.60 billion on June 4, 2017 bowing to intense pressure and criticisms from a cross section of the society.\n",
       "]]&gt;</content:encoded>\n",
       "<wfw:commentrss>https://setopati.net/market/147613/feed</wfw:commentrss>\n",
       "<slash:comments>0</slash:comments>\n",
       "</item>\n",
       "<item>\n",
       "<title>CPN to conclude division of responsibilities of central members Wednesday</title>\n",
       "<link/>https://setopati.net/political/147610\n",
       "\t\t<comments>https://setopati.net/political/147610#respond</comments>\n",
       "<pubdate>Tue, 04 Jun 2019 12:00:30 +0000</pubdate>\n",
       "<dc:creator></dc:creator>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<guid ispermalink=\"false\">https://setopati.net/?p=147610</guid>\n",
       "<description></description>\n",
       "<content:encoded>The ruling CPN meeting will conclude division of responsibilities among the central members on Wednesday.<br/>\n",
       "The secretariat meeting Tuesday afternoon discussed division of responsibilities among the central members but could not reach conclusion, according to CPN Spokesperson Narayan Kaji Shrestha. “We will again convene secretariat meeting Wednesday noon and conclude that,” Shrestha said after the meeting.<br/>\n",
       "He added that the Wednesday’s meeting will also discuss remaining unification works if there is time, and another meeting will be convened if not.<br/>\n",
       "Chairmen KP Sharma Oli and Pushpa Kamal Dahal, General Secretary Bishnu Paudel and senior leader Madhav Kumar Nepal had met at the prime minister’s residence in Baluwatar earlier on Tuesday to decide about department heads. But the four leaders could not forge consensus about the department heads.<br/>\n",
       "CPN has not been able to decide about leadership of different departments including organization, finance, publicity and party school with multiple leaders wishing for single position. The party also has yet to form the politburo.\n",
       "]]&gt;</content:encoded>\n",
       "<wfw:commentrss>https://setopati.net/political/147610/feed</wfw:commentrss>\n",
       "<slash:comments>0</slash:comments>\n",
       "</item>\n",
       "<item>\n",
       "<title>Parliament cannot be defamed by giving us Rs 60m: Gagan Thapa</title>\n",
       "<link/>https://setopati.net/political/147604\n",
       "\t\t<comments>https://setopati.net/political/147604#respond</comments>\n",
       "<pubdate>Tue, 04 Jun 2019 11:02:46 +0000</pubdate>\n",
       "<dc:creator></dc:creator>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<guid ispermalink=\"false\">https://setopati.net/?p=147604</guid>\n",
       "<description></description>\n",
       "<content:encoded>Nepali Congress (NC) lawmaker Gagan Thapa has said the government has defamed the parliament by providing Rs 60 million to the lawmakers.\n",
       "<p>Speaking during the discussion on the recent budget in the House of Representatives (HoR) on Tuesday, he complained that the government has given Rs 60 million to the lawmakers ignoring policy issues and told them to go to their respective constituencies taking the money.</p>\n",
       "<p>“I am a directly elected lawmaker. I also know that we should be connected to the demand for development. I also want to be connected,” he added. “The budget talks about Melamchi project, constructing tunnel way to connect with the capital, encroached land, green urbanization, cycle city and other issues. These issues are related to my constituency. I want to be connected on that. I raise questions about how we can be connected but don’t get answers. I present proposals in the parliament but they are not discussed.”</p>\n",
       "<p>He slammed the government for ignoring elected lawmakers on the issues of development related to their respective constituencies and providing Rs 60 million to be spent by directly elected lawmakers to silence them.</p>\n",
       "<p>“The government keeps me apart from all these things and tells me ‘honorable lawmaker you have received Rs 60 million. Go to your constituency and build the walls of temples, community buildings, buildings for grieving persons. Get connected to the people’,” he pointed. “We don’t want Rs 60 million to be connected with the people but want to connect with policies worth Rs 6 billion and even Rs 600 billion related to our places. We want to be connected with plans.</p>\n",
       "<p>“And the parliament cannot be defamed by telling us to go away with Rs 60 million,” he stated.</p>\n",
       "<p>He claimed that the prime minister and even speaker have selected projects of their constituencies in the budget.</p>\n",
       "<p>He criticized the budget saying it is such abysmal as the ruling CPN does not have any principle. He claimed that the budget has ignored the past studies and it does not give the feel of a stable government. “I want to ask finance minister, should I be happy about the budget as a lawmaker or sad as a citizen? I am confused.”</p>\n",
       "<p>He fired a riposte at CPN General Secretary Bishnu Paudel who on Monday had challenged the main opposition NC to oppose the hike in elderly allowance.<br/>\n",
       "The budget presented on May 29 has raised elderly allowance to Rs 3,000 a month from Rs 2,000.<br/>\n",
       "“A powerful lawmaker who can shake the foundation of Baluwatar stood here and challenged if anyone has the courage to speak about elderly allowance. I have the courage,” Thapa riposted during the HoR meeting on Tuesday.<br/>\n",
       "Thapa reminded that he opposed the decision of the government of his party president Sher Bahadur Deuba to bring down the eligibility for elderly allowance to 65 years and raise it to Rs 5,000 a month in the past.<br/>\n",
       "“I did not just opposed the decision of the then outgoing Deuba government to bring down the eligibility for elderly allowance to 65 years and raise it to Rs 5,000 a month saying it cannot be raised on whims. I am a lawmaker who filed an amendment proposal saying the base year should be made 70 years when the law was drafted,” Thapa stated.<br/>\n",
       "“There are so many social security programs. They need to be integrated and rationalized, and there should be differentiation about which should be made universal, which targeted, and which connected to production. The biggest issue is social security is not just cash transfer. Is it not appropriate to discuss to provide facilities worth Rs 30,000 and not just Rs 3,000 through non-cash provisions?” he questioned.</p>\n",
       "<div class=\"video-container\"><iframe allow=\"accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture\" allowfullscreen=\"\" frameborder=\"0\" height=\"465\" src=\"https://www.youtube.com/embed/uV54ae2VR8c?feature=oembed\" width=\"826\"></iframe></div>\n",
       "]]&gt;</content:encoded>\n",
       "<wfw:commentrss>https://setopati.net/political/147604/feed</wfw:commentrss>\n",
       "<slash:comments>0</slash:comments>\n",
       "</item>\n",
       "<item>\n",
       "<title>President Bhandari uses commercial flight to travel to Nepalgunj</title>\n",
       "<link/>https://setopati.net/political/147602\n",
       "\t\t<comments>https://setopati.net/political/147602#respond</comments>\n",
       "<pubdate>Tue, 04 Jun 2019 09:30:23 +0000</pubdate>\n",
       "<dc:creator></dc:creator>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<guid ispermalink=\"false\">https://setopati.net/?p=147602</guid>\n",
       "<description></description>\n",
       "<content:encoded>President Bidya Devi Bhandari has traveled on a commercial flight for probably the first time inside Nepal.<br/>\n",
       "President Bhandari, who usually flies on a chartered Nepal Army chopper inside Nepal, reached Nepalgunj at 12:30 Tuesday afternoon on a Shree Airlines airplane to participate in a climate conference, according to Chief District Officer (CDO) of Banke Madan Bhujel.<br/>\n",
       "“She will inaugurate the seminar organized by the Ministry of Forest and Environment on Wednesday,” CDO Bhujel said.<br/>\n",
       "Elected representatives of different local bodies will also participate in the seminar.<br/>\n",
       "President Bhandari will also participate in the iftar dinner–the meal Muslims take after sunset during the holy month of Ramadan–in Nepalgunj Tuesday night.<br/>\n",
       "She will also address the golden jubilee celebrations of Prabhat Secondary School in Banke.\n",
       "]]&gt;</content:encoded>\n",
       "<wfw:commentrss>https://setopati.net/political/147602/feed</wfw:commentrss>\n",
       "<slash:comments>0</slash:comments>\n",
       "</item>\n",
       "<item>\n",
       "<title>CPN leaders fail to decide department heads</title>\n",
       "<link/>https://setopati.net/political/147594\n",
       "\t\t<comments>https://setopati.net/political/147594#respond</comments>\n",
       "<pubdate>Tue, 04 Jun 2019 08:16:46 +0000</pubdate>\n",
       "<dc:creator></dc:creator>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<category></category>\n",
       "<guid ispermalink=\"false\">https://setopati.net/?p=147594</guid>\n",
       "<description></description>\n",
       "<content:encoded>The meeting of top leaders of the ruling CPN Tuesday morning has failed to conclude division of responsibilities in the party.<br/>\n",
       "Chairmen KP Sharma Oli and Pushpa Kamal Dahal, General Secretary Bishnu Paudel and senior leader Madhav Kumar Nepal met at the prime minister’s residence in Baluwatar to decide about department heads. But the four leaders could not forge consensus about the department heads, according to a party source.<br/>\n",
       "CPN has not been able to decide about leadership of different departments including organization, finance, publicity and party school with multiple leaders wishing for single position.<br/>\n",
       "The top leaders have not been able to decide with some secretariat members claiming for the organization department. Paudel and Nepal also want to lead the organization department.<br/>\n",
       "The secretariat meeting scheduled for later Tuesday will now only discuss division of responsibilities among the central members after the four leaders failed to forge consensus about department heads.<br/>\n",
       "The issue of forming politburo was also raised during the meeting between four leaders but it was not discussed after Dahal insisted that division of responsibilities among the central members should be completed before formation of politburo.\n",
       "]]&gt;</content:encoded>\n",
       "<wfw:commentrss>https://setopati.net/political/147594/feed</wfw:commentrss>\n",
       "<slash:comments>0</slash:comments>\n",
       "</item>\n",
       "</channel>\n",
       "</rss>\n",
       "<!--\n",
       "Performance optimized by W3 Total Cache. Learn more: https://www.w3-edge.com/products/\n",
       "\n",
       "Object Caching 0/188 objects using apc\n",
       "Page Caching using disk: enhanced (SSL caching disabled) \n",
       "Database Caching 16/22 queries in 0.179 seconds using disk\n",
       "\n",
       "Served from: setopati.net @ 2019-06-06 20:45:57 by W3 Total Cache\n",
       "--></body></html>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# look at the soup using xml parser\n",
    "pagesource = response.content\n",
    "soup  = BeautifulSoup(pagesource,'xml')\n",
    "soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<item>\n",
       "<title>HoR approves proposal seeking consideration on Appropriation Bill</title>\n",
       "<link>https://thehimalayantimes.com/kathmandu/hor-approves-proposal-seeking-consideration-on-appropriation-bill/</link>\n",
       "<pubDate>Thu, 06 Jun 2019 13:11:01 +0000</pubDate>\n",
       "<dc:creator>Priyanka Adhikari</dc:creator>\n",
       "<category>Kathmandu</category>\n",
       "<category>Appropriation Bill</category>\n",
       "<category>House of Representatives</category>\n",
       "<guid isPermaLink=\"false\">http://thehimalayantimes.com/?p=593236</guid>\n",
       "<description>&lt;p&gt;Today's second meeting of the House of Representatives (HoR) has approved, by a majority, the proposal seeking consideration on the Appropriation Bill, 2076.&lt;/p&gt;\n",
       "&lt;p&gt;The post &lt;a rel=\"nofollow\" href=\"https://thehimalayantimes.com/kathmandu/hor-approves-proposal-seeking-consideration-on-appropriation-bill/\"&gt;HoR approves proposal seeking consideration on Appropriation Bill&lt;/a&gt; appeared first on &lt;a rel=\"nofollow\" href=\"https://thehimalayantimes.com\"&gt;The Himalayan Times&lt;/a&gt;.&lt;/p&gt;\n",
       "</description>\n",
       "<content:encoded>&lt;p&gt;&lt;strong&gt;KATHMANDU:&lt;/strong&gt; Today&amp;#8217;s second meeting of the House of Representatives (HoR) has approved, by a majority, the proposal seeking consideration on the Appropriation Bill, 2076.&lt;/p&gt;\n",
       "&lt;p&gt;Finance Minister Yuvaraj Khatiwada had presented the bill. The HoR will hold deliberations on the document from June 10 to 21. Speaker Krishna Bahadur Mahara informed the meeting that discussions on the bill would be held on the basis of party representation.&lt;/p&gt;\n",
       "&lt;p&gt;Responding to the questions regarding the bill put forth by lawmakers in the meeting, Minister Khatiwada said the government was working hard to make the import of chemical fertilisers easy for the development of agriculture sector. He said the government was making preparations to work on the government-to-government modality for systematising fertiliser import.&lt;/p&gt;\n",
       "&lt;p&gt;The Finance Minister stressed that the government has set the budget in such a way to facilitate the implementation of fundamental rights. Lawmakers including Prem Suwal and Parbati DC Chaudhary commented on the bill.&lt;/p&gt;\n",
       "&lt;p&gt;Before this, Minister Khatiwada had presented the Appropriation Bill, 2076, the bill on Mobilising Internal Loans, 2076, and the Loans and Guarantee (23rd Amendment) Bill, 2076.&lt;/p&gt;\n",
       "&lt;p&gt;The HoR will next meet at 11:00 am on June 10.&lt;/p&gt;\n",
       "&lt;p&gt;The post &lt;a rel=\"nofollow\" href=\"https://thehimalayantimes.com/kathmandu/hor-approves-proposal-seeking-consideration-on-appropriation-bill/\"&gt;HoR approves proposal seeking consideration on Appropriation Bill&lt;/a&gt; appeared first on &lt;a rel=\"nofollow\" href=\"https://thehimalayantimes.com\"&gt;The Himalayan Times&lt;/a&gt;.&lt;/p&gt;\n",
       "</content:encoded>\n",
       "<post-id xmlns=\"com-wordpress:feed-additions:1\">593236</post-id> </item>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "item_list = soup.find_all('item')\n",
    "# categorys=[]\n",
    "# print(soup)\n",
    "\n",
    "# for post in item_list:\n",
    "#     title = post.find('title')\n",
    "#     pubdate = post.find('pubDate').text\n",
    "#     category_list = post.find_all('category')\n",
    "#     for category in category_list:\n",
    "#         categorys.append(category)\n",
    "#     description =  post.find('description').text\n",
    "\n",
    "post  =  item_list[0]\n",
    "\n",
    "# to clean the description because there is html tags in between the text\n",
    "\n",
    "post"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorys=[]\n",
    "title = post.find('title').text # getting title\n",
    "pubdate = post.find('pubDate').text # getting publication data\n",
    "link  = post.find('link').text    # getting link of the article\n",
    "            \n",
    "category_list = post.find_all('category') # category list\n",
    "for category in category_list:\n",
    "    categorys.append(category.text)\n",
    "            \n",
    "description =  post.find('description').text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'HoR approves proposal seeking consideration on Appropriation Bill'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Thu, 06 Jun 2019 13:11:01 +0000'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pubdate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://thehimalayantimes.com/kathmandu/hor-approves-proposal-seeking-consideration-on-appropriation-bill/'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "link"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<category>Kathmandu</category>,\n",
       " <category>Appropriation Bill</category>,\n",
       " <category>House of Representatives</category>]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "category_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<p>Today\\'s second meeting of the House of Representatives (HoR) has approved, by a majority, the proposal seeking consideration on the Appropriation Bill, 2076.</p>\\n<p>The post <a rel=\"nofollow\" href=\"https://thehimalayantimes.com/kathmandu/hor-approves-proposal-seeking-consideration-on-appropriation-bill/\">HoR approves proposal seeking consideration on Appropriation Bill</a> appeared first on <a rel=\"nofollow\" href=\"https://thehimalayantimes.com\">The Himalayan Times</a>.</p>\\n'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# since the description stills contains html tags we need to clean i.e parse it and extract the text properly.\n",
    "\n",
    "def clean_description(description):\n",
    "    soup_description =BeautifulSoup(description,'html.parser')\n",
    "# print(soup_description.find('p'))\n",
    "    if (soup_description.find('p')==None):\n",
    "        c_description = soup_description\n",
    "#     print('seto')\n",
    "    else:\n",
    "#     print('kalo')\n",
    "        c_description = soup_description.find(\"p\").text\n",
    "    return c_description\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<p>Today's second meeting of the House of Representatives (HoR) has approved, by a majority, the proposal seeking consideration on the Appropriation Bill, 2076.</p>\n",
      "<p>The post <a rel=\"nofollow\" href=\"https://thehimalayantimes.com/kathmandu/hor-approves-proposal-seeking-consideration-on-appropriation-bill/\">HoR approves proposal seeking consideration on Appropriation Bill</a> appeared first on <a rel=\"nofollow\" href=\"https://thehimalayantimes.com\">The Himalayan Times</a>.</p>\n",
      "\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "-------------Look At Clean Description------------------------\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "\n",
      "Today's second meeting of the House of Representatives (HoR) has approved, by a majority, the proposal seeking consideration on the Appropriation Bill, 2076.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "d = post.find('description').text\n",
    "\n",
    "x = (clean_description(d)) # X is the cleaned description \n",
    "\n",
    "print(description)\n",
    "print(\"-----------------------------------------\")\n",
    "print(\"-----------------------------------------\")\n",
    "\n",
    "print(\"-------------Look At Clean Description------------------------\")\n",
    "print(\"-----------------------------------------\")\n",
    "print(\"-----------------------------------------\")\n",
    "print()\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " WE KNOw data but we need to store it in database...\n",
    " 1. creating a Dictionary named 'aDic' which stores the title,pubDate,link,category and description of each newsitem\n",
    " 2. storing the 'aDic' in a mongoDB collection  in our case ['allNews']  of Database ['breakfast']\n",
    " \n",
    " 3. breakfast is database\n",
    " 4. allNews is collection i.e table\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "myclient = pymongo.MongoClient(\"mongodb://localhost:27017/\")\n",
    "\n",
    "breakfastDB = myclient[\"breakfast\"]\n",
    "newsCollection = breakfastDB[\"allNews\"]\n",
    "\n",
    "def clean_description(description):\n",
    "    soup_description =BeautifulSoup(description,'html.parser')\n",
    "# print(soup_description.find('p'))\n",
    "    if (soup_description.find('p')==None):\n",
    "        c_description = soup_description.text\n",
    "#     print('seto')\n",
    "    else:\n",
    "#     print('kalo')\n",
    "        c_description = soup_description.find(\"p\").text\n",
    "    return c_description\n",
    "    \n",
    "# urls = [\"http://english.onlinekhabar.com/feed\",\"https://thehimalayantimes.com/feed/\",\"https://setopati.net/feed\"]\n",
    "urls = {\n",
    "    \"onlinekhabar\":\"http://english.onlinekhabar.com/feed\",\n",
    "    \"the himalayan times\":\"https://thehimalayantimes.com/feed/\",\n",
    "    \"setopati\":\"https://setopati.net/feed\"\n",
    "    }\n",
    "\n",
    "# final_df= pd.DataFrame()\n",
    "\n",
    "for url in urls:\n",
    "#     s_df= pd.DataFrame()\n",
    "    try:\n",
    "        response =  requests.get(urls[url],headers=header)\n",
    "        pagesource = response.content\n",
    "        soup  = BeautifulSoup(pagesource,'xml')\n",
    "        item_list = soup.find_all('item')\n",
    "\n",
    "        for post in item_list:\n",
    "\n",
    "            categorys=[]\n",
    "            aDic={}\n",
    "\n",
    "            title = post.find('title').text # getting title\n",
    "            pubdate = post.find('pubDate').text # getting publication data\n",
    "            link  = post.find('link').text    # getting link of the article\n",
    "            \n",
    "            category_list = post.find_all('category') # category list\n",
    "            for category in category_list:\n",
    "                categorys.append(category.text)\n",
    "            \n",
    "            description =  post.find('description').text\n",
    "            # cleaning the description \n",
    "            soup_description =BeautifulSoup(post.find('description').text,'html.parser')\n",
    "            # print(soup_description)\n",
    "#             if (soup_description.find('p')==None):\n",
    "#                 description = soup_description\n",
    "#             else:\n",
    "#                 description = soup_description.find('p').text\n",
    "            # cleaned\n",
    "            aDic['scrapdate']=datetime.now().time()\n",
    "            aDic['Publisher']=url\n",
    "            aDic['Title']=title\n",
    "            aDic['Publish_Date'] = pubdate\n",
    "            aDic['Link'] = link\n",
    "            aDic['Category']=categorys\n",
    "            aDic['Description']=clean_description(description)\n",
    "            newsCollection.insert_one(aDic)\n",
    "            # converting to dataframe \n",
    "           # instant_df = pd.DataFrame([aDic],columns=aDic.keys())\n",
    "            #s_df = pd.concat([s_df,instant_df],ignore_index=True,axis=0,sort=False)\n",
    "        #final_df = pd.concat([final_df,s_df],ignore_index=True,axis=0,sort=False)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        pass\n",
    "\n",
    "# final_df.to_json(\"all.json\",orient='records')    \n",
    "# final_df.to_json(\"index_all.json\",orient='index') \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MONGO_DB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "myclient = pymongo.MongoClient(\"mongodb://localhost:27017/\")\n",
    "\n",
    "breakfastDB = myclient[\"breakfast\"]\n",
    "newsCollection = breakfastDB[\"allNews\"]\n",
    "try :\n",
    "    x = newsCollection.insert_one(json_data)\n",
    "except :\n",
    "    pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = newsCollection.find_one()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'_id': ObjectId('5ceb51f0155c452670eaaa29'), '0': {'Publisher': 'onlinekhabar', 'Title': 'Two Nepalis die on Everest and Makalu', 'Publish_Date': 'Fri, 24 May 2019 10:56:11 +0000', 'Link': 'http://english.onlinekhabar.com/two-nepalis-die-on-everest-and-makalu.html', 'Category': ['Just In', 'Dhruba Bista', 'Nima Tenji Sherpa'], 'Description': 'Kathmandu, May 24 Two more have died on two different 8,000 metre plus peaks of the country on Friday. Reputed mountain guide Dhruba Bista died at the base camp of Mount Everest after suffering from mountain sickness. Jiwan Bhattarai, an official at Himalaya Ecstasy Treks, says that Bista fell ill at Camp III and was […]'}, '1': {'Publisher': 'onlinekhabar', 'Title': 'China completes Durbar High School reconstruction in 10 months', 'Publish_Date': 'Fri, 24 May 2019 10:34:06 +0000', 'Link': 'http://english.onlinekhabar.com/china-completes-durbar-high-school-reconstruction-in-10-months.html', 'Category': ['Heritage', 'Just In', 'Chinese support', 'Durbar High School', 'Nepal reconstruction', 'Post-earthquake reconstruction'], 'Description': 'Kathmandu, May 24 The Chinese government has completed the reconstruction of Durbar High School, the first public school of Nepal, in 10 months. The historical building was destroyed by the 2015 April earthquake. China had begun reconstruction in August last year. According to officials of the National Reconstruction Authority, the reconstruction could not begin on […]'}, '2': {'Publisher': 'onlinekhabar', 'Title': 'Fearful of displeasure, Oli and Dahal reluctant to form NCP politburo', 'Publish_Date': 'Fri, 24 May 2019 10:12:31 +0000', 'Link': 'http://english.onlinekhabar.com/fearful-of-displeasure-oli-dahal-reluctant-to-form-ncp-politburo.html', 'Category': ['Just In', 'Political', 'KP Sharma Oli', 'NCP', 'Nepal Communist Party', 'Pushpa Kamal Dahal', 'UML-Maoist merger'], 'Description': 'Kathmandu, May 24 Though it has already been two weeks since top leaders of the ruling Nepal Communist Party reportedly finalised a list of 147 leaders to include in the party’s politburo, the party has not declared the committee yet. It seems two chairpersons of the party–KP Sharma Oli and Pushpa Kamal Dahal–are reluctant to […]'}, '3': {'Publisher': 'onlinekhabar', 'Title': 'Nepal to announce Mount Everest’s fresh height within six months', 'Publish_Date': 'Fri, 24 May 2019 08:34:56 +0000', 'Link': 'http://english.onlinekhabar.com/nepal-to-announce-mount-everests-fresh-height-within-six-months.html', 'Category': ['Just In', 'Everest measurement', 'Mount Everest', 'Padma Aryal'], 'Description': 'Kathmandu, May 24 The government of Nepal says it will announce results of the measurement of the height of Mount Everest within next six months. Minister for Land Management, Cooperatives and Poverty Alleviation Padma Kumari Aryal made the announcement one day after the government team sent to measure the height returned to Kathmandu. Addressing a […]'}, '4': {'Publisher': 'onlinekhabar', 'Title': 'Two more die on Everest; toll reaches 7 this season', 'Publish_Date': 'Fri, 24 May 2019 08:01:04 +0000', 'Link': 'http://english.onlinekhabar.com/two-more-die-on-everest-toll-reaches-7-this-season.html', 'Category': ['Just In', 'Ing Landgraf', 'Nihal Bagwan'], 'Description': 'Kathmandu, May 24 Two more climbers have died on Mount Everest on Thursday. With this, the number of climbers killed on the highest peak of the world this season has reached seven. Ing Landgraf, 65, from Austria and Nihal Bagwan, 27, from India died on Everest after returning from the summit. According to Subas Shrestha, […]'}, '5': {'Publisher': 'onlinekhabar', 'Title': 'Yunus Ansari held with fake Indian banknotes at Kathmandu airport', 'Publish_Date': 'Fri, 24 May 2019 07:48:05 +0000', 'Link': 'http://english.onlinekhabar.com/yunus-ansari-held-with-fake-indian-banknotes-at-kathmandu-airport.html', 'Category': ['Just In', 'fake banknotes', 'Tribhuvan International Airport'], 'Description': 'Kathmandu, May 24 Police in Kathmandu on Friday arrested notorious smuggling suspect Yunus Ansari in possession of fake Indian banknotes. The son of former minister Salim Miya Anasari and the operator of National Television, Ansari had served a jail sentence for his involvement in illegal banknote transactions and drug dealing a few years ago. He […]'}, '6': {'Publisher': 'onlinekhabar', 'Title': 'Three killed in Dailekh minitruck accident', 'Publish_Date': 'Fri, 24 May 2019 06:30:56 +0000', 'Link': 'http://english.onlinekhabar.com/three-killed-in-dailekh-minitruck-accident.html', 'Category': ['Just In', 'Dailekh'], 'Description': 'Kathmandu, May 24 Three persons died when a minitruck fell off the road in Rigalpani, Gurans Rural Municipality-1 of Dailekh district in Karnali province on Friday morning. The vehicle fell off around 300 metres down the road, according to an official of the District Police Office. He informs that the three bodies have been spotted […]'}, '7': {'Publisher': 'onlinekhabar', 'Title': 'The barking deer enigma', 'Publish_Date': 'Fri, 24 May 2019 06:09:31 +0000', 'Link': 'http://english.onlinekhabar.com/the-barking-deer-enigma.html', 'Category': ['Cover Story'], 'Description': 'While found in forests across most parts of India, little is known about the Indian muntjac or barking deer, one of the most primitive deer groups in the world. The Indian muntjac is fascinating to scientists because of its abnormally low chromosome number. Females have six and males have seven, the least among mammals. Until […]'}, '8': {'Publisher': 'onlinekhabar', 'Title': 'Prakash Sharan Mahat to address G20 think tank summit', 'Publish_Date': 'Fri, 24 May 2019 06:02:51 +0000', 'Link': 'http://english.onlinekhabar.com/prakash-sharan-mahat-to-address-g20-think-tank-summit.html', 'Category': ['Just In', 'Nepali Congress', 'Prakash Sharan Mahat'], 'Description': 'Kathmandu, May 24 Nepali Congress joint general secretary Prakash Sharan Mahat left for Japan on Thursday night to attend the T20 Summit. T20 Japan is an official engagement group of Japan’s 2019 G20 presidency chaired by the Asian Development Bank Institute (ADBI). Mahat is scheduled to provide recommendations to G20, an international forum of developed […]'}, '9': {'Publisher': 'onlinekhabar', 'Title': 'With China’s announcement, Nepal expedites preparations to reopen Tatopani', 'Publish_Date': 'Fri, 24 May 2019 05:31:04 +0000', 'Link': 'http://english.onlinekhabar.com/with-chinas-announcement-nepal-expedites-preparations-to-reopen-tatopani.html', 'Category': ['Business', 'Just In', 'Tatopani border', 'Tatopani reopening'], 'Description': 'Sindhupalchok, May 24 With an announcement of Chinese Ambassador to Nepal Hou Yanqi that the Tatopani border point between the two countries, closed since the disastrous 2015 April earthquake, will reopen next week, the Nepali side has also expedited preparations. Till a few days ago, stakeholders were worried about the status of a bridge over […]'}, '10': {'Publisher': 'the himalayan times', 'Title': 'Death toll reaches 20 as British climber Robin Fisher dies on Mt Everest', 'Publish_Date': 'Sat, 25 May 2019 05:13:56 +0000', 'Link': 'https://thehimalayantimes.com/nepal/death-toll-reaches-20-as-british-climber-robin-fisher-dies-on-mt-everest/', 'Category': ['Nepal', 'British climber dead', 'Death on Mt Everest', 'Death tolls from Mt Everest', 'Department of Tourism', 'Everest summit point', 'Mt Everest', 'Robin Fisher', 'Robin Haynes Fisher'], 'Description': 'A British climber died while returning from the summit of Mt Everest early this morning taking the death toll on different mountains above 8,000 metres to 20 in the spring climbing season.'}, '11': {'Publisher': 'the himalayan times', 'Title': 'North Korean missile test violated UN resolution, says Bolton', 'Publish_Date': 'Sat, 25 May 2019 04:43:30 +0000', 'Link': 'https://thehimalayantimes.com/world/north-korean-missile-test-violated-un-resolution-says-bolton/', 'Category': ['World', 'denuclearisation talks', 'Kim Jong Un', \"North Korea's Kim\", 'North Korean missile test', 'UN Security Council resolution', 'US national security adviser', 'US North Korea'], 'Description': \"US National Security Adviser John Bolton said on Saturday North Korea's recent missile launches violated a UN Security Council resolution and urged leader Kim Jong Un to return to denuclearisation talks.\"}, '12': {'Publisher': 'the himalayan times', 'Title': 'Durbar High School building nearing completion', 'Publish_Date': 'Sat, 25 May 2019 03:46:57 +0000', 'Link': 'https://thehimalayantimes.com/kathmandu/durbar-high-school-building-nearing-completion/', 'Category': ['Kathmandu', 'Durbar High School', 'Durbar High School building'], 'Description': 'Major work on reconstruction of the historic Durbar High School, which was damaged in the earthquakes of 2015, is over. This was disclosed at a programme organised on the school premises on Friday.'}, '13': {'Publisher': 'the himalayan times', 'Title': 'Punarbas Municipality tense after police foil child marriage', 'Publish_Date': 'Sat, 25 May 2019 03:41:04 +0000', 'Link': 'https://thehimalayantimes.com/nepal/punarbas-municipality-tense-after-police-foil-child-marriage/', 'Category': ['Nepal', 'attempted child marriage', 'Kanchanpur', 'Punarbas Municipality'], 'Description': 'Tension that built up in Punarbas Municipality, Kanchanpur, for almost the entire day on Friday over a dispute related to an attempted child marriage yesterday subsided following an agreement with the police in the evening.'}, '14': {'Publisher': 'the himalayan times', 'Title': '‘Panel will ensure NHRC’s autonomy’', 'Publish_Date': 'Sat, 25 May 2019 03:35:59 +0000', 'Link': 'https://thehimalayantimes.com/kathmandu/panel-will-ensure-national-human-rights-commission-autonomy/', 'Category': ['Kathmandu', 'House of Representatives', 'Krishna Bhakta Pokharel', 'Law Justice and Human Rights Committee', 'National Human Rights Commission', 'NHRC Act', 'NHRC Act Bill'], 'Description': 'Chairperson of Law, Justice and Human Rights Committee of the House of Representatives Krishna Bhakta Pokharel said his committee would address all the concerns expressed by office bearers of the National Human Rights Commission on NHRC Act (Amendment) Bill.'}, '15': {'Publisher': 'the himalayan times', 'Title': 'Vice-president to visit Austria', 'Publish_Date': 'Sat, 25 May 2019 03:31:06 +0000', 'Link': 'https://thehimalayantimes.com/nepal/vice-president-nanda-bahadur-pun-to-visit-austria/', 'Category': ['Nepal', 'Alexander Van der Bellen', 'President of Austria', 'R20 Austrian World Summit', \"Vice-president's Austria visit\"], 'Description': 'Vice-president Nanda Bahadur Pun will visit Vienna, Austria, at the invitation of the President of Austria\\xa0 Alexander Van der Bellen, to attend the R20 Austrian World Summit being held in Vienna on May 28.'}, '16': {'Publisher': 'the himalayan times', 'Title': '‘Tax hike on tobacco products will bring down patients of cancer, heart diseases’', 'Publish_Date': 'Sat, 25 May 2019 03:26:00 +0000', 'Link': 'https://thehimalayantimes.com/kathmandu/tax-hike-on-tobacco-products-will-bring-down-patients-of-cancer-heart-diseases/', 'Category': ['Kathmandu', 'Bhagwan Koirala', 'cancer patients', 'heart diseases patients', 'Tax hike on tobacco products'], 'Description': 'Risk of heart attack is four times higher in a smoker than in a non-smoker; the cause of 80 per cent deaths by asthma is smoking; and smoking is behind 90 per cent deaths caused by lung cancer in the country, revealed Bhagwan Koirala, senior cardiothoracic and vascular surgeon.'}, '17': {'Publisher': 'the himalayan times', 'Title': 'HoR to restart process on Passport Bill', 'Publish_Date': 'Sat, 25 May 2019 03:11:12 +0000', 'Link': 'https://thehimalayantimes.com/nepal/house-of-representatives-to-restart-process-on-passport-bill/', 'Category': ['Nepal', 'HoR', 'House of Representatives', 'laws relating to passport', 'Passport Bill'], 'Description': 'The House of Representatives will restart the process of passing the bill to amend and integrate laws relating to passport, also known as Passport Bill, after President Bidhya Devi Bhandari sent the bill back to the HoR for reconsideration.'}, '18': {'Publisher': 'the himalayan times', 'Title': 'Government issues Shivapuri Nagarjun National Park Rules', 'Publish_Date': 'Sat, 25 May 2019 03:06:02 +0000', 'Link': 'https://thehimalayantimes.com/kathmandu/government-issues-shivapuri-nagarjun-national-park-rules/', 'Category': ['Kathmandu', 'Shivapuri Nagarjun National Park', 'wildlife and natural resources'], 'Description': 'The government has issued Shivapuri Nagarjun National Park Rules for protection and conservation of its vicinity by imposing restriction on activities that pose threat to wildlife and natural resources.'}, '19': {'Publisher': 'the himalayan times', 'Title': 'Lawmakers demand quota for women within reservation clusters', 'Publish_Date': 'Sat, 25 May 2019 03:01:30 +0000', 'Link': 'https://thehimalayantimes.com/nepal/lawmakers-demand-quota-for-women-within-reservation-clusters/', 'Category': ['Nepal', 'Civil Servant Act', 'Federal Civil Servant Bill', 'HoR lawmakers', 'House of Representatives', 'House of Representatives lawmakers', 'Nepal Communist Party (NCP)', 'parliamentary State Affairs and Good Governance Committee', 'quotas for women', 'SAGGC'], 'Description': 'Lawmakers of the House of Representatives demanded for a provision of reservation quotas for women within each reservation cluster.'}, '20': {'Publisher': 'the himalayan times', 'Title': 'Four more die on Everest, season’s death toll hits 19', 'Publish_Date': 'Sat, 25 May 2019 02:59:43 +0000', 'Link': 'https://thehimalayantimes.com/nepal/four-more-die-on-mt-everest-seasons-death-toll-hits-19/', 'Category': ['Nepal', 'Death toll from Mt Everest', 'Indian climber', \"Mountaineers' death toll\", 'Mt Everest deaths', 'Mt Makalu', 'National mountain guide', 'spring climbing season'], 'Description': 'At least four more climbers died on Mt Everest while a Sherpa climber breathed his last on Mt Makalu in the last 24 hours, bringing the death toll to 19 on different mountains above 8,000 metres in the spring climbing season, according to expedition officials.'}, '21': {'Publisher': 'the himalayan times', 'Title': 'Caesarean section births increasing at alarming rate', 'Publish_Date': 'Sat, 25 May 2019 02:56:04 +0000', 'Link': 'https://thehimalayantimes.com/kathmandu/caesarean-section-births-increasing-at-alarming-rate/', 'Category': ['Kathmandu', 'C-section', 'caesarean section', 'caesarean section births', 'Nepal Demographic Health Survey-2006'], 'Description': 'Rate of caesarean section births has increased alarmingly in the country in the past few years.'}, '22': {'Publisher': 'the himalayan times', 'Title': 'Survivor of Nanga Parbat summits Lhotse, Everest', 'Publish_Date': 'Sat, 25 May 2019 02:51:38 +0000', 'Link': 'https://thehimalayantimes.com/nepal/survivor-of-nanga-parbat-summits-lhotse-everest/', 'Category': ['Nepal', 'Elisabeth Revol', 'French climber', 'French climber Elisabeth Revol', 'Mt Everest', 'Mt Lhotse', 'Nanga Parbat survivor'], 'Description': 'French climber Elisabeth Revol, who was dramatically rescued last year from Pakistan’s Mount Nanga Parbat, scaled Mt Lhotse this morning, a day after reaching the top of Mt Everest, her expedition organiser said.'}, '23': {'Publisher': 'the himalayan times', 'Title': 'Arubote carnage: Nepal Police team collecting evidences', 'Publish_Date': 'Sat, 25 May 2019 02:51:07 +0000', 'Link': 'https://thehimalayantimes.com/nepal/arubote-carnage-nepal-police-team-collecting-evidences/', 'Category': ['Nepal', 'Arubote carnage', 'Central Bureau of Investigation', 'Miklajung', 'Nepal Police', 'Panchthar carnage'], 'Description': 'A team deployed from Nepal Police’s Central Bureau of Investigation to investigate the carnage in which nine persons were killed continued its work to collect evidences from the incident site and its surrounding areas at Arubote of Miklajung-3, in Panchthar on Friday as well.'}, '24': {'Publisher': 'the himalayan times', 'Title': '384,206 traffic rules violators booked', 'Publish_Date': 'Sat, 25 May 2019 02:46:05 +0000', 'Link': 'https://thehimalayantimes.com/kathmandu/384206-traffic-rules-violators-booked/', 'Category': ['Kathmandu', 'Metropolitan Traffic Police Division', 'traffic rules violation', 'traffic rules violators'], 'Description': 'Traffic police said they took action against 384,206 motorists and two-wheeler riders for violating traffic rules in Kathmandu valley in the last 10 months of the current fiscal.'}, '25': {'Publisher': 'the himalayan times', 'Title': 'Talk programme on Nepal, Germany ties', 'Publish_Date': 'Sat, 25 May 2019 02:41:09 +0000', 'Link': 'https://thehimalayantimes.com/nepal/talk-programme-on-nepal-germany-ties/', 'Category': ['Nepal', 'Nepal-German Academic Association', 'Nepal-Germany Relations'], 'Description': 'Nepal-German Academic Association organised a Talk Programme on ‘Retrospect and Prospect of Nepal-Germany Relations: The Role of Alumni in Nepal’s Development’.'}, '26': {'Publisher': 'the himalayan times', 'Title': 'Low cost treatment for sickle cell disease at govt hospital', 'Publish_Date': 'Sat, 25 May 2019 02:36:06 +0000', 'Link': 'https://thehimalayantimes.com/kathmandu/low-cost-treatment-for-sickle-cell-disease-at-government-hospital/', 'Category': ['Kathmandu', 'Civil Service Hospital', 'Government hospital', 'red blood cell exchange transfusion', 'sickle cell treatment', 'sickle-cell disease'], 'Description': 'Sickle cell patients can now get quality treatment at a low cost as a government hospital has unveiled the procedure of red blood cell exchange transfusion for sickle cell disease with machine.'}, '27': {'Publisher': 'the himalayan times', 'Title': 'TIC building gathers dust', 'Publish_Date': 'Sat, 25 May 2019 02:31:10 +0000', 'Link': 'https://thehimalayantimes.com/nepal/tourism-information-centre-building-gathers-dust/', 'Category': ['Nepal', 'Bhojpur tourism information centre', 'Tourism information Centre'], 'Description': 'Bhojpur, May 24 A tourism information centre building in Bhojpur, built\\xa0 around six years ago with an aim to promote tourism in eastern Nepal, is'}, '28': {'Publisher': 'the himalayan times', 'Title': 'Fake Indian currency worth Rs 76.7 million confiscated', 'Publish_Date': 'Sat, 25 May 2019 02:29:34 +0000', 'Link': 'https://thehimalayantimes.com/nepal/fake-indian-currency-worth-rs-76-7-million-confiscated/', 'Category': ['Nepal', 'counterfeit Indian currency notes smuggling', 'Fake Indian currency', 'international racket', 'Tribhuvan International Airport'], 'Description': 'Nepal Police has busted an international racket involved in smuggling counterfeit Indian currency notes to Nepal via Tribhuvan International Airport and circulating them across India.'}, '29': {'Publisher': 'the himalayan times', 'Title': 'Call to prioritise infrastructure development', 'Publish_Date': 'Sat, 25 May 2019 02:20:36 +0000', 'Link': 'https://thehimalayantimes.com/temp/call-to-prioritise-infrastructure-development/', 'Category': ['Temp', 'budget', 'infrastructure development', 'road and air connectivity', 'Tourism development'], 'Description': 'Tourism stakeholders have urged the government to prioritise tourism infrastructure development in the budget for the next fiscal year.'}, '30': {'Publisher': 'the himalayan times', 'Title': 'VIRGO – MAY 25', 'Publish_Date': 'Sat, 25 May 2019 01:15:44 +0000', 'Link': 'https://thehimalayantimes.com/horoscopes/virgo/virgo-may-25-4/', 'Category': ['Virgo (Aug 23 - Sep 22)'], 'Description': 'You have a lot of ground to cover, and quickly at that. Know what is happening around a project. You might decide to dedicate today'}, '31': {'Publisher': 'the himalayan times', 'Title': 'TAURUS – MAY 25', 'Publish_Date': 'Sat, 25 May 2019 01:15:42 +0000', 'Link': 'https://thehimalayantimes.com/horoscopes/taurus/taurus-may-25-3/', 'Category': ['Taurus (Apr 20 - May 20)'], 'Description': 'You might be worried about your finances. You have an innate desire to spend and to enjoy various baubles and pretty items. Sometimes you find'}, '32': {'Publisher': 'the himalayan times', 'Title': 'SCORPIO – MAY 25', 'Publish_Date': 'Sat, 25 May 2019 01:15:41 +0000', 'Link': 'https://thehimalayantimes.com/horoscopes/scorpio/scorpio-may-25-4/', 'Category': ['Scorpio (Oct 23 - Nov 21)'], 'Description': 'Allow a family member to participate more than usual in making plans. Invite others over for a spontaneous afternoon and early dinner. You could be'}, '33': {'Publisher': 'the himalayan times', 'Title': 'SAGITTARIUS – MAY 25', 'Publish_Date': 'Sat, 25 May 2019 01:15:41 +0000', 'Link': 'https://thehimalayantimes.com/horoscopes/sagittarius/sagittarius-may-25-4/', 'Category': ['Sagittarius (Nov 22 - Dec 21)'], 'Description': 'Be more forthright in how you deal with a problem. Your optimism helps others swing from their positions more easily. You could experience sufficient discomfort'}, '34': {'Publisher': 'the himalayan times', 'Title': 'AQUARIUS – MAY 25', 'Publish_Date': 'Sat, 25 May 2019 01:15:41 +0000', 'Link': 'https://thehimalayantimes.com/horoscopes/aquarius/aquarius-may-25-4/', 'Category': ['Aquarius (Jan 20 - Feb 18)'], 'Description': 'You seem more positive and upbeat than you have in a while. A friend could be cheering you on as you decide to take an'}, '35': {'Publisher': 'the himalayan times', 'Title': 'PISCES – MAY 25', 'Publish_Date': 'Sat, 25 May 2019 01:15:40 +0000', 'Link': 'https://thehimalayantimes.com/horoscopes/pisces/pisces-may-25-4/', 'Category': ['Pisces (Feb 19 - Mar 20)'], 'Description': 'Play it low-key for now. If you have something personal you want to deal with, now is the time. Check in on an older relative'}, '36': {'Publisher': 'the himalayan times', 'Title': 'LIBRA – MAY 25', 'Publish_Date': 'Sat, 25 May 2019 01:15:37 +0000', 'Link': 'https://thehimalayantimes.com/horoscopes/libra/libra-may-25-4/', 'Category': ['Libra ( Sep 23 - Oct 22)'], 'Description': 'You could be viewed as more dynamic if you just share more of what is on your mind. You come up with unusual ideas even'}, '37': {'Publisher': 'the himalayan times', 'Title': 'GEMINI – MAY 25', 'Publish_Date': 'Sat, 25 May 2019 01:15:32 +0000', 'Link': 'https://thehimalayantimes.com/horoscopes/gemini/gemini-may-25-4/', 'Category': ['Gemini ( May 21 - June 20)'], 'Description': 'Defer your present quandary for the moment. You will get many different impressions at present. Interactions will also tell you which way would be best'}, '38': {'Publisher': 'the himalayan times', 'Title': 'LEO – MAY 25', 'Publish_Date': 'Sat, 25 May 2019 01:15:29 +0000', 'Link': 'https://thehimalayantimes.com/horoscopes/leo/leo-may-25-4/', 'Category': ['Leo (Jul 23 - Aug 22)'], 'Description': 'You will defer to others. Your creativity soars as you hang out with a friend. Many people will share what is going on and give'}, '39': {'Publisher': 'the himalayan times', 'Title': 'CAPRICORN – MAY 25', 'Publish_Date': 'Sat, 25 May 2019 01:15:25 +0000', 'Link': 'https://thehimalayantimes.com/horoscopes/capricorn/capricorn-may-25-4/', 'Category': ['Capricorn ( Dec 22 - Jan 19)'], 'Description': 'You might want to head in a new direction but be stopped by various issues or people. They need to get your advice or share'}, '40': {'Publisher': 'setopati', 'Title': 'Media bill will be amended: Chairman of Human rights Committee of HoR', 'Publish_Date': 'Fri, 24 May 2019 12:29:05 +0000', 'Link': 'https://setopati.net/political/147236', 'Category': ['Politics', 'Special', 'Bill', 'human rights', 'Krishna Bhakta Pokharel', 'media', 'media bill', 'NHRC', 'press', 'press council'], 'Description': 'Chairman of the Human Rights Committee of the House of Representatives (HoR) Krishna Bhakta Pokharel has assured that the media bill will be amended. Talking with journalists at the Singh Durbar on Friday, he assured that parliamentary committees will hold discussions with the stakeholders concerned at the time of passing bills. “It is the parliament’s … '}, '41': {'Publisher': 'setopati', 'Title': 'British PM May to resign on June 7', 'Publish_Date': 'Fri, 24 May 2019 09:24:41 +0000', 'Link': 'https://setopati.net/political/147233', 'Category': ['Politics', 'Special', 'Brexit', 'bRITAIN', 'Conservative', 'PM', 'theresa may'], 'Description': 'Theresa May says she will step down as U.K. Conservative Party leader on June 7, sparking a contest to become Britain’s next prime minister. She will stay as caretaker prime minister until the new leader is chosen, a process likely to take several weeks. May has bowed to relentless pressure from her party to quit … '}, '42': {'Publisher': 'setopati', 'Title': 'Formation of CPN politburo delayed due to dispute between Dahal and Thapa', 'Publish_Date': 'Fri, 24 May 2019 08:40:45 +0000', 'Link': 'https://setopati.net/political/147228', 'Category': ['Politics', 'Special', 'cpn', 'dahal', 'maoist', 'politburo', 'rAM bAHADUR Thapa', 'uml'], 'Description': 'Unification process of the ruling CPN has again been stuck with those coming from the former Maoist Center not able to decide the name for 66 politburo members allotted to them. CPN Chairman Pushpa Kamal Dahal is unhappy after Home Minister Ram Bahadur Thapa included many leaders close to him in the list of 66. … '}, '43': {'Publisher': 'setopati', 'Title': 'Three killed in Dailekh mini-truck plunge', 'Publish_Date': 'Fri, 24 May 2019 08:01:09 +0000', 'Link': 'https://setopati.net/social/147224', 'Category': ['Society', 'mini-truck', 'plunge', 'police', 'victims'], 'Description': 'Three people were killed when the mini-truck they were traveling in fell some 300 meters off the road at Nigalpani of Dungeshwar rural municipality-1 in the district Friday. The mini truck was heading toward Jumla from Surkhet. The bodies of the three victims were found in charred condition, Inspector Prem Chhetri of the District Police … '}, '44': {'Publisher': 'setopati', 'Title': 'Younus Ansari arrested with two briefcases of counterfeit Indian currency notes', 'Publish_Date': 'Fri, 24 May 2019 07:28:26 +0000', 'Link': 'https://setopati.net/social/147216', 'Category': ['Society', 'Special', 'AIRPORT', 'counterfeit', 'currency', 'IC', 'indian', 'Younus Ansari'], 'Description': 'Younus Ansari has been arrested with counterfeit Indian currency notes worth IRs 76.794 million from the Tribhuvan International Airport (TIA) on Friday. Ansari has been arrested by a team of special bureau of Nepal Police and airport police from the airport where he had gone to receive two Pakistani nationals. The two Pakistanis coming from … '}, '45': {'Publisher': 'setopati', 'Title': 'Ncell brings ‘Unlimited Roaming Data Pack’ for India', 'Publish_Date': 'Fri, 24 May 2019 07:27:09 +0000', 'Link': 'https://setopati.net/market/147215', 'Category': ['Business', 'india', 'Ncell', 'travel', 'Unlimited Roaming Data Pack'], 'Description': 'Ncell customers can now enjoy worry free data roaming at more affordable rate while traveling to India. They can simply take their Ncell SIM card with them in India to use unlimited data at just Rs 100 per day and remain connected as never away from home. Ncell Private Limited has launched an attractive ‘Unlimited … '}, '46': {'Publisher': 'setopati', 'Title': 'Three killed, 29 injured in separate road accidents', 'Publish_Date': 'Fri, 24 May 2019 07:17:46 +0000', 'Link': 'https://setopati.net/social/147213', 'Category': ['Society', 'collision', 'police', 'road accident', 'truck'], 'Description': 'Three persons were killed, and at least 29 others injured in separate road accidents in Siraha, Makawanpur and Chitwan districts. Two persons were killed when a Kathmandu-bound (Na 8 Kha 982) passenger bus collided head on with a truck (Na 3 Kha 4264) near Mirchaiya of Siraha district along the highway on Thursday night. At … '}, '47': {'Publisher': 'setopati', 'Title': 'Media bill won’t be passed as it is: Narayan Kaji Shrestha', 'Publish_Date': 'Fri, 24 May 2019 06:28:36 +0000', 'Link': 'https://setopati.net/political/147208', 'Category': ['Politics', 'Special', 'cpn', 'media', 'media bill', 'Narayan Kaji Shrestha', 'press', 'press council'], 'Description': 'Secretariat member and Spokesperson of ruling CPN Narayan Kaji Shrestha has stated that the media bill registered in the parliament will not be passed as it is. “The bill that has come now, I claim it will not be passed as it is,” Shrestha said addressing a program organized by human rights academy in Kathmandu … '}, '48': {'Publisher': 'setopati', 'Title': 'Cleanliness campaign organized at Rani Mahal', 'Publish_Date': 'Fri, 24 May 2019 06:22:54 +0000', 'Link': 'https://setopati.net/social/147203', 'Category': ['Society', 'cleanliness', 'drive', 'Rani Mahal', 'tourists'], 'Description': 'An intense cleanliness drive was carried out at Rani Mahal, a historic palace based in Palpa. The palace dating back to the Rana regime remains as the touristic site for both domestic and foreign tourists. Over 40 Nepal Army (NA) personnel from the NA Gorakhdal Battalion, staff of the Tansen municipality sanitation section, from the … '}, '49': {'Publisher': 'setopati', 'Title': 'Same-sex couples start registering marriages in Taiwan', 'Publish_Date': 'Fri, 24 May 2019 06:21:14 +0000', 'Link': 'https://setopati.net/social/147204', 'Category': ['Society', 'Special', 'legalize', 'LGBT', 'marriage', 'register', 'rights', 'same-sex couples', 'Taiwan'], 'Description': 'Hundreds of same-sex couples in Taiwan are rushing to get married on the first day a landmark decision to legalize same-sex marriage takes effect. One household registration office in Taipei was packed Friday as couples seized the earliest opportunity to tie the knot. Taiwan became the first place in Asia to allow same-sex marriage last … '}}\n"
     ]
    }
   ],
   "source": [
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "newsCollection.drop()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'name': 'admin', 'sizeOnDisk': 139264.0, 'empty': False}\n",
      "{'name': 'breakfast', 'sizeOnDisk': 69632.0, 'empty': False}\n",
      "{'name': 'config', 'sizeOnDisk': 73728.0, 'empty': False}\n",
      "{'name': 'local', 'sizeOnDisk': 106496.0, 'empty': False}\n",
      "{'name': 'mydatabase', 'sizeOnDisk': 73728.0, 'empty': False}\n"
     ]
    }
   ],
   "source": [
    " for i in myclient.list_databases():\n",
    "        print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'_id': ObjectId('5ceb501d155c452670eaaa23'), 'name': 'John', 'address': 'Highway 37'}\n"
     ]
    }
   ],
   "source": [
    "x = mycol.find_one()\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pymongo.results.InsertOneResult at 0x7f02fc2e4548>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "import pymongo\n",
    "\n",
    "myc = pymongo.MongoClient(\"mongodb://localhost:27017/\")\n",
    "mydb = myc[\"mydatabase\"]\n",
    "mycol = mydb[\"customers\"]\n",
    "\n",
    "mydict = { \"name\": \"John\", \"address\": \"Highway 37\" }\n",
    "\n",
    "mycol.insert_one(mydict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "breakfastDB.list_collection_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'dict' object has no attribute 'inserted_id'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-57-1e2239575e2c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minserted_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'dict' object has no attribute 'inserted_id'"
     ]
    }
   ],
   "source": [
    "print(x.inserted_id)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final Clean scrap Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#myclient = pymongo.MongoClient(\"mongodb://localhost:27017/\")\n",
    "\n",
    "#breakfastDB = myclient[\"breakfast\"]\n",
    "#newsCollection = breakfastDB[\"newNews\"]\n",
    "\n",
    "def clean_description(description):\n",
    "    soup_description =BeautifulSoup(description,'html.parser')\n",
    "    if (soup_description.find('p')==None):\n",
    "        c_description = soup_description.text\n",
    "    else:\n",
    "        c_description = soup_description.find(\"p\").text\n",
    "    return c_description\n",
    "    \n",
    "urls = {\n",
    "    \"onlinekhabar\":\"http://english.onlinekhabar.com/feed\",\n",
    "    \"the himalayan times\":\"https://thehimalayantimes.com/feed/\",\n",
    "    \"setopati\":\"https://setopati.net/feed\"\n",
    "    }\n",
    "\n",
    "\n",
    "for url in urls:\n",
    "    try:\n",
    "        response =  requests.get(urls[url],headers=header)\n",
    "        pagesource = response.content\n",
    "        soup  = BeautifulSoup(pagesource,'xml')\n",
    "        item_list = soup.find_all('item')\n",
    "\n",
    "        for post in item_list:\n",
    "\n",
    "            categorys=[]\n",
    "            aDic={}\n",
    "\n",
    "            title = post.find('title').text # getting title\n",
    "            pubdate = post.find('pubDate').text # getting publication data\n",
    "            link  = post.find('link').text    # getting link of the article\n",
    "            \n",
    "            category_list = post.find_all('category') # category list\n",
    "            for category in category_list:\n",
    "                categorys.append(category.text)\n",
    "            \n",
    "            description =  post.find('description').text\n",
    "            soup_description =BeautifulSoup(post.find('description').text,'html.parser')\n",
    "            # cleaned\n",
    "            aDic['Publisher']=url\n",
    "            aDic['Title']=title\n",
    "            aDic['Publish_Date'] = date_parser.parse(pubdate)\n",
    "            aDic['Link'] = link\n",
    "            aDic['Category']=categorys\n",
    "            aDic['Description']=clean_description(description)\n",
    "            newsCollection.insert_one(aDic)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        pass\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Sun, 26 May 2019 14:18:31 GMT'"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pubdate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime.datetime(2019, 5, 26, 14, 18, 31, tzinfo=tzutc())"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "date_parser.parse(pubdate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I'm working...\n",
      "I'm working...\n",
      "I'm working...\n",
      "I'm working...\n",
      "I'm working...\n",
      "I'm working...\n",
      "I'm working...\n",
      "I'm working...\n",
      "I'm working...\n",
      "I'm working...\n",
      "I'm working...\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-91-d07482e08ece>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0mschedule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_pending\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m     \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import schedule\n",
    "import time\n",
    "\n",
    "def job():\n",
    "    print(\"I'm working...\")\n",
    "\n",
    "schedule.every(3).seconds.do(job)\n",
    "\n",
    "\"\"\"\n",
    "schedule.every().day.at(\"00:15\").do(main)\n",
    "schedule.every().day.at(\"06:15\").do(main)\n",
    "schedule.every().day.at(\"12:15\").do(main)\n",
    "\"\"\"\n",
    "while True:\n",
    "    schedule.run_pending()\n",
    "    time.sleep(1)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "breakfast",
   "language": "python",
   "name": "breakfast"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}