All_page_Daraz_JKstore_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "60f8f6db",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "from selenium import webdriver\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f005ab0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dict = {'Title': [],'Category': [], 'Price': [], 'Image': []}\n",
    "driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "\n",
    "total_pages = 2\n",
    "page = 1\n",
    "\n",
    "while page <= int(total_pages):\n",
    "    time.sleep(2)\n",
    "    \n",
    "    print(f'Scrapping page: {page}')\n",
    "\n",
    "    scrapping_site_url = f'https://www.daraz.pk/jk-store/?from=wangpu&lang=en&langFlag=en&page={page}&pageTypeId=2&q=All-Products'\n",
    "\n",
    "    driver.get(scrapping_site_url)\n",
    "    \n",
    "    html = driver.page_source\n",
    "    product_url = []\n",
    "    parsed_html = BeautifulSoup(html, \"html.parser\")\n",
    "    get_specific_url = parsed_html.find_all('a',age=\"0\")\n",
    "    for i in get_specific_url:\n",
    "        product_url.append('http:'+i['href'])\n",
    "    #Removing duplicates\n",
    "    product_url= list(set(product_url))\n",
    "    \n",
    "    try:\n",
    "        total_pages = driver.find_element_by_xpath('//ul[@class=\"ant-pagination \"]/li[position() = (last() - 1)]').text\n",
    "        print(total_pages)\n",
    "    except:\n",
    "        continue\n",
    "    for j in range(len(product_url)):\n",
    "        driver.get(product_url[j])\n",
    "        \n",
    "        name = driver.find_element_by_xpath('//*[@id=\"module_product_title_1\"]/div/div/span').text\n",
    "        data_dict['Title'].append(name)\n",
    "    \n",
    "        prices = driver.find_element_by_xpath('//*[@id=\"module_product_price_1\"]/div/div/span').text\n",
    "        data_dict['Price'].append(prices)\n",
    "    \n",
    "        image = driver.find_element_by_xpath('//*[@id=\"module_item_gallery_1\"]/div/div[1]/div/img')\n",
    "        data_dict['Image'].append(image.get_attribute(\"src\"))\n",
    "    \n",
    "        category = driver.find_element_by_xpath('//*[@id=\"J_breadcrumb\"]').text\n",
    "        data_dict['Category'].append(category)\n",
    "                \n",
    "    page += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "ac7064c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame.from_dict(data_dict, orient=\"columns\")\n",
    "#df['Price'] = df['Price'].apply(lambda x: x.replace('Rs.', '').strip())\n",
    "df.to_excel(\"scrapped_data_all_pages_9.xlsx\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}