Modules in Python.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Python Modules"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dates using datetime library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dt.date.today(): 2020-07-26\n",
      "dt.datetime.now(): 2020-07-26 05:26:00.634716\n",
      "\n",
      "Formatted date using strftime(): 26/07/20\n",
      "Formatted datetime using strftime(): 26-07-2020 05:26:00 AM\n",
      "Month of the current date: 7\n",
      "\n",
      "Date from String using strptime(): 2014-05-21 12:30:00\n"
     ]
    }
   ],
   "source": [
    "# dates in python\n",
    "# Press tab button in Jupyter notebooks to explore various functions in library\n",
    "\n",
    "import datetime as dt\n",
    "current_date=dt.date.today() # today's date\n",
    "print(\"dt.date.today():\",current_date)\n",
    "\n",
    "current_datetime=dt.datetime.now() # today's date and time\n",
    "print(\"dt.datetime.now():\",current_datetime)\n",
    "\n",
    "# Converting date into String\n",
    "print(\"\\nFormatted date using strftime():\",current_date.strftime('%d/%m/%y')) # string format time\n",
    "print(\"Formatted datetime using strftime():\",current_datetime.strftime('%d-%m-%Y %H:%M:%S %p'))\n",
    "\n",
    "# Accessing information from date\n",
    "print(\"Month of the current date:\",current_date.month)\n",
    "\n",
    "# Converting String into date\n",
    "print('\\nDate from String using strptime():',dt.datetime.strptime('21/05/2014 12:30','%d/%m/%Y %H:%M'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating Random Numbers using random library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating three random numbers using rd.random():\n",
      "0.3083432821792196\n",
      "0.9611291577745781\n",
      "0.7086895527119456\n",
      "\n",
      "Generating a random number between a defined range using rd.randint(1,10): 1\n",
      "\n",
      "Sequence c: [2, 5, 73, 7, 34, 234, 67, 2354, 62, 22, 67, 82, 5]\n",
      "Choosing a number from a sequence using rd.choice(c): 5\n"
     ]
    }
   ],
   "source": [
    "import random as rd\n",
    "print(\"Generating three random numbers using rd.random():\")\n",
    "for i in range(3):\n",
    "    a=rd.random() # random() generates random numbers between 0.0 and 1.0 (including 0.0 but excluding 1.0)\n",
    "    print(a)\n",
    "    \n",
    "b=rd.randint(1,10) # randint(low,high) returns an integer number between low and high value (including both)\n",
    "print(\"\\nGenerating a random number between a defined range using rd.randint(1,10):\",b)\n",
    "\n",
    "c=[2,5,73,7,34,234,67,2354,62,22,67,82,5]\n",
    "print(\"\\nSequence c:\",c)\n",
    "d=rd.choice(c) # choice(sequence) returns a random number from the sequence\n",
    "print(\"Choosing a number from a sequence using rd.choice(c):\",d)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating punctuations using string library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Special characters: !\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\n"
     ]
    }
   ],
   "source": [
    "import string as st\n",
    "print(\"\\nSpecial characters:\",st.punctuation) # explore other options using tab button"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Copy library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "a: [1, 2, 3, 4, 5] \n",
      "b: [1, 2, 3, 4, 5] \n",
      "c: [1, 2, 3, 4, 5]\n",
      "\n",
      "b is a: False\n",
      "c is a: True\n",
      "c is b: False\n",
      "\n",
      "a==b: True\n",
      "c==b: True\n"
     ]
    }
   ],
   "source": [
    "import copy as cp\n",
    "a=[1,2,3,4,5]\n",
    "b=cp.copy(a) # copy() duplicates the object\n",
    "c=a\n",
    "print(\"\\na:\",a,\"\\nb:\",b,\"\\nc:\",c)\n",
    "\n",
    "print(\"\\nb is a:\",b is a) # both refers to the different object\n",
    "print(\"c is a:\",c is a) # both refers to the same object\n",
    "print(\"c is b:\",c is b) # both refers to the different object\n",
    "\n",
    "print(\"\\na==b:\",a==b)\n",
    "print(\"c==b:\",c==b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## JSON library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "myjson: {\"A\":1,\"B\":2,\"C\":3}  has type: <class 'str'>\n",
      "myjson1: {'A': 1, 'B': 2, 'C': 3}  has type: <class 'dict'>\n",
      "\n",
      "dictionary: {'X': 1, 'Y': 2, 'Z': 3}  has type: <class 'dict'>\n",
      "json_file: {\"X\": 1, \"Y\": 2, \"Z\": 3}  has type: <class 'str'>\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "myjson='{\"A\":1,\"B\":2,\"C\":3}'\n",
    "print(\"myjson:\",myjson,\" has type:\",type(myjson))\n",
    "\n",
    "myjson1=json.loads(myjson) # Parse to dict\n",
    "print(\"myjson1:\",myjson1,\" has type:\",type(myjson1))\n",
    "\n",
    "dictionary={\"X\":1,\"Y\":2,\"Z\":3}\n",
    "print(\"\\ndictionary:\",dictionary,\" has type:\",type(dictionary))\n",
    "json_file=json.dumps(dictionary)\n",
    "print(\"json_file:\",json_file,\" has type:\",type(json_file))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Os Module\n",
    "OS module provides functions for interacting with the operating system. This module provides a portable way of using operating system dependent functionality. The *os* and *os.path* modules include many functions to interact with the file system."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'nt'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.name # gives the name of the operating system dependent module imported"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note**: It may give different output on different interpreters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'L:\\\\GIT\\\\Python-Beginners-Guide'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BASE_PATH=os.getcwd() # returns the Current Working Directory(CWD) of the file \n",
    "BASE_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.path.exists(BASE_PATH) # To check if the path exist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.path.exists('Demo')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.mkdir('Demo') # Create an empty directory named Demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.path.isdir('Demo') # check the presence of directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.git',\n",
       " '.gitignore',\n",
       " '.ipynb_checkpoints',\n",
       " 'Demo',\n",
       " 'Learn Python in Jupyter Notebook.ipynb',\n",
       " 'LICENSE',\n",
       " 'Modules in Python.ipynb',\n",
       " 'README.md',\n",
       " 'sample.html',\n",
       " 'Test1.txt',\n",
       " 'Test2.txt']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir() # print files and directories in the current directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".git                                     : size 4096\n",
      ".gitignore                               : size 1307\n",
      ".ipynb_checkpoints                       : size 0\n",
      "Demo                                     : size 0\n",
      "Learn Python in Jupyter Notebook.ipynb   : size 117591\n",
      "LICENSE                                  : size 1093\n",
      "Modules in Python.ipynb                  : size 139218\n",
      "README.md                                : size 649\n",
      "sample.html                              : size 1020\n",
      "Test1.txt                                : size 97\n",
      "Test2.txt                                : size 97\n"
     ]
    }
   ],
   "source": [
    "# Directory listing with size\n",
    "for file in os.listdir():\n",
    "    info = os.stat(file)\n",
    "    print('%-40s : size %d' % (file, info.st_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L:\\GIT\\Python-Beginners-Guide\\.git is a directory\n",
      "L:\\GIT\\Python-Beginners-Guide\\.gitignore is a file\n",
      "L:\\GIT\\Python-Beginners-Guide\\.ipynb_checkpoints is a directory\n",
      "L:\\GIT\\Python-Beginners-Guide\\Demo is a directory\n",
      "L:\\GIT\\Python-Beginners-Guide\\Learn Python in Jupyter Notebook.ipynb is a file\n",
      "L:\\GIT\\Python-Beginners-Guide\\LICENSE is a file\n",
      "L:\\GIT\\Python-Beginners-Guide\\Modules in Python.ipynb is a file\n",
      "L:\\GIT\\Python-Beginners-Guide\\README.md is a file\n",
      "L:\\GIT\\Python-Beginners-Guide\\sample.html is a file\n",
      "L:\\GIT\\Python-Beginners-Guide\\Test1.txt is a file\n",
      "L:\\GIT\\Python-Beginners-Guide\\Test2.txt is a file\n"
     ]
    }
   ],
   "source": [
    "for file in os.listdir():\n",
    "    fullname=os.path.join(BASE_PATH,file) # to join two patha together\n",
    "    if os.path.isdir(fullname):\n",
    "        print('{} is a directory'.format(fullname))\n",
    "    else:\n",
    "        print('{} is a file'.format(fullname))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir('Demo') # Changing the working directory path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'L:\\\\GIT\\\\Python-Beginners-Guide\\\\Demo'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DEMO_PATH=os.path.abspath('.') # print absolute path on our system\n",
    "DEMO_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello in Demo_1 file\n",
      "<class 'OSError'>\n",
      "Problem reading: Demo2.txt\n"
     ]
    }
   ],
   "source": [
    "# Demo1.txt file is created\n",
    "\n",
    "filename = 'Demo1.txt'\n",
    "with open(filename, 'w') as file:\n",
    "    file.write(\"Hello in Demo_1 file\")\n",
    "\n",
    "with open(filename, 'r') as file:\n",
    "    text=file.read()\n",
    "    print(text)\n",
    "    \n",
    "try: \n",
    "    filename = 'Demo2.txt' # Demo2.txt file does not exist\n",
    "    with open(filename, 'r') as f:\n",
    "        text = f.read() \n",
    "    \n",
    "except IOError:  \n",
    "    print(os.error) # os.error is an alias for built-in OSError exception\n",
    "    print('Problem reading: ' + filename) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Demo1.txt']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(os.listdir())\n",
    "os.path.isfile('Demo1.txt') # Check if the given path exists and is file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.path.getsize('Demo1.txt') # returns the file size in bytes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "timestamp=os.path.getmtime('Demo1.txt') # check when file was last modified, returns a epoch timestamp\n",
    "\n",
    "import datetime\n",
    "datetime.datetime.fromtimestamp(timestamp) # returns timestamp in datetime format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Demo2.txt']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.rename('Demo1.txt','Demo2.txt') # renaming File\n",
    "os.listdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.remove('Demo2.txt') # Remove (delete) the file\n",
    "os.listdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.git',\n",
       " '.gitignore',\n",
       " '.ipynb_checkpoints',\n",
       " 'Demo',\n",
       " 'Learn Python in Jupyter Notebook.ipynb',\n",
       " 'LICENSE',\n",
       " 'Modules in Python.ipynb',\n",
       " 'README.md',\n",
       " 'sample.html',\n",
       " 'Test1.txt',\n",
       " 'Test2.txt']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.chdir(BASE_PATH)\n",
    "os.listdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.git',\n",
       " '.gitignore',\n",
       " '.ipynb_checkpoints',\n",
       " 'Learn Python in Jupyter Notebook.ipynb',\n",
       " 'LICENSE',\n",
       " 'Modules in Python.ipynb',\n",
       " 'README.md',\n",
       " 'sample.html',\n",
       " 'Test1.txt',\n",
       " 'Test2.txt']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.rmdir(DEMO_PATH) # Remove (delete) the directory\n",
    "os.listdir()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Environment variables\n",
    "* To print all the environment variables\n",
    "```python\n",
    "os.environ\n",
    "```\n",
    "* If we are looking for any specific value\n",
    "```python\n",
    "os.environ.get(‘LOGNAME’)\n",
    "```\n",
    "* To get the process id\n",
    "```python\n",
    "os.getpid()\n",
    "```\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Common useful OS operations \n",
    "* https://medium.com/@jbrendan86/python-has-extensive-support-for-operating-system-tasks-such-as-file-and-folder-management-f4f879384e0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collections Module\n",
    "### Counter\n",
    "* Counter is a subclass of dictionary object. \n",
    "* The Counter() function in collections module takes an iterable or a mapping as the argument and returns a Dictionary.\n",
    "* In this dictionary, a key is an element in the iterable or the mapping and value is the number of times that element exists in the iterable or the mapping."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collection of Fruits: ['apples', 'apples', 'mangoes', 'mangoes', 'mangoes', 'mangoes', 'bananas', 'bananas', 'bananas', 'bananas', 'bananas', 'bananas', 'kiwi', 'kiwi', 'kiwi', 'pomegranate', 'pomegranate', 'pomegranate', 'lichi', 'lichi', 'lichi', 'lichi', 'lichi', 'lichi', 'lichi', 'lichi', 'lichi', 'lichi', 'oranges', 'oranges', 'oranges', 'oranges', 'oranges', 'oranges']\n",
      "\n",
      "Total Fruits: [('lichi', 10), ('bananas', 6), ('oranges', 6), ('mangoes', 4), ('kiwi', 3), ('pomegranate', 3), ('apples', 2)]\n",
      "Mangoes: 4\n",
      "Cherries: 0\n",
      "\n",
      "Fruits Purchased: ['apples', 'oranges', 'mangoes', 'mangoes', 'mangoes', 'lichi', 'bananas', 'papaya']\n",
      "Total Fruits with their updated quantities: [('lichi', 11), ('mangoes', 7), ('bananas', 7), ('oranges', 7), ('apples', 3), ('kiwi', 3), ('pomegranate', 3), ('papaya', 1)]\n",
      "\n",
      "Fruits Consumed: ['lichi', 'lichi', 'mangoes', 'pomegranate', 'mangoes', 'apples', 'bananas', 'pomegranate', 'apples', 'oranges', 'mangoes']\n",
      "Total Fruits with their updated quantities: [('lichi', 9), ('bananas', 6), ('oranges', 6), ('mangoes', 4), ('kiwi', 3), ('apples', 1), ('pomegranate', 1), ('papaya', 1)]\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "c=Counter() # Counter() can often substitute for a dictionary\n",
    "\n",
    "# Adding fruits \n",
    "c['apples']+=2\n",
    "c['mangoes']+=4\n",
    "c['bananas']+=6\n",
    "c['kiwi']+=3\n",
    "c['pomegranate']+=2\n",
    "c['lichi']+=10\n",
    "c['oranges']+=6\n",
    "c['pomegranate']+=1\n",
    "\n",
    "print('Collection of Fruits:',list(c.elements())) # elements displays the elements of the counter in a list\n",
    "\n",
    "print('\\nTotal Fruits:',c.most_common()) # most_common() returns list of elements and counts, sorted by counts\n",
    "print('Mangoes:',c['mangoes'])\n",
    "print('Cherries:',c['cherries'])\n",
    "\n",
    "purchased=['apples','oranges','mangoes','mangoes','mangoes','lichi','bananas','papaya']\n",
    "c.update(purchased) # update() adds to the count instead of replacing them\n",
    "print('\\nFruits Purchased:',purchased)\n",
    "print('Total Fruits with their updated quantities:',c.most_common())\n",
    "\n",
    "consumed=['lichi','lichi','mangoes','pomegranate','mangoes','apples','bananas','pomegranate','apples','oranges','mangoes']\n",
    "c.subtract(consumed) # subtract() is the complement of update()\n",
    "print('\\nFruits Consumed:',consumed)\n",
    "print('Total Fruits with their updated quantities:',c.most_common())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### defaultdict\n",
    "* The defaultdict works exactly like a python dictionary, except for it does not throw KeyError when a non-existent key is accessed.\n",
    "* It initializes the key with the element of the data type that is passed as an argument at the creation of defaultdict. The data type is called default_factory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "str_dict['Test']: \n",
      "str_dict: defaultdict(<class 'str'>, {'Test': ''})\n",
      "\n",
      "float_dict['Test']: 0.0\n",
      "float_dict: defaultdict(<class 'float'>, {'Test': 0.0})\n",
      "\n",
      "int_dict['Test']: 0\n",
      "int_dict: defaultdict(<class 'int'>, {'Test': 0})\n",
      "int_dict: defaultdict(<class 'int'>, {'Test': 0, 'Exam': 1, 'Subject': 'Mathematics'})\n",
      "\n",
      "class_dict['Test']: 1/2\n",
      "class_dict: defaultdict(<class '__main__.Fraction'>, {'Test': 1/2})\n",
      "\n",
      "func_dict['Test']: Default\n",
      "func_dict: defaultdict(<function test_func at 0x0000019271CC8488>, {'Test': 'Default'})\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "# defaultdict generates default values for missing keys\n",
    "# Accessing a missing key raises no errors in a defaultdict \n",
    "\n",
    "str_dict=defaultdict(str)\n",
    "print(\"str_dict['Test']:\",str_dict['Test']) # Default Value ''\n",
    "print('str_dict:',str_dict)\n",
    "\n",
    "float_dict=defaultdict(float)\n",
    "print(\"\\nfloat_dict['Test']:\",float_dict['Test']) # Default Value 0.0\n",
    "print('float_dict:',float_dict)\n",
    "\n",
    "int_dict=defaultdict(int)\n",
    "print(\"\\nint_dict['Test']:\",int_dict['Test']) # Default Value 0\n",
    "print('int_dict:',int_dict)\n",
    "\n",
    "# Adding new elements in the int_dict\n",
    "int_dict['Exam']+=1\n",
    "int_dict['Subject']='Mathematics'\n",
    "print(\"int_dict:\",int_dict)\n",
    "\n",
    "class Fraction(object):\n",
    "    def __init__(self):\n",
    "        self.n=1\n",
    "        self.d=2\n",
    "    def __repr__(self):\n",
    "        return '{0}/{1}'.format(self.n,self.d)\n",
    "    \n",
    "class_dict=defaultdict(Fraction)\n",
    "print(\"\\nclass_dict['Test']:\",class_dict['Test']) # Default Value 1/2 (as defined in the above class)\n",
    "print('class_dict:',class_dict)\n",
    "\n",
    "def test_func():\n",
    "    return 'Default'\n",
    "\n",
    "func_dict=defaultdict(test_func)\n",
    "print(\"\\nfunc_dict['Test']:\",func_dict['Test']) # Default Value 'Default' (as defined in the above function)\n",
    "print('func_dict:',func_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OrderedDict\n",
    "* OrderedDict is a dictionary where keys maintain the order in which they are inserted, which means if the value of a key is changed later, it will not change the position of the key."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fruit_dict: OrderedDict([('apples', 8), ('oranges', 8), ('kiwi', 3), ('pineapples', 4), ('mangoes', 7), ('lichi', 6), ('bananas', 5), ('papaya', 4)])\n",
      "\n",
      "fruit_dict: OrderedDict([('apples', 8), ('oranges', 8), ('kiwi', 3), ('pineapples', 4), ('mangoes', 7), ('lichi', 6), ('bananas', 5), ('papaya', 4), ('cheeries', 5)])\n",
      "\n",
      "fruit_dict: OrderedDict([('apples', 11), ('oranges', 8), ('kiwi', 3), ('pineapples', 4), ('mangoes', 7), ('lichi', 6), ('bananas', 5), ('papaya', 4), ('cheeries', 5)])\n",
      "\n",
      "fruit_dict: OrderedDict([('apples', 11), ('oranges', 8), ('pineapples', 4), ('mangoes', 7), ('lichi', 6), ('bananas', 5), ('papaya', 4), ('cheeries', 5), ('kiwi', 3)])\n",
      "\n",
      "fruit_dict: OrderedDict([('kiwi', 3), ('apples', 11), ('oranges', 8), ('pineapples', 4), ('mangoes', 7), ('lichi', 6), ('bananas', 5), ('papaya', 4), ('cheeries', 5)])\n",
      "\n",
      "fruit_dict: OrderedDict([('kiwi', 3), ('apples', 11), ('oranges', 8), ('pineapples', 4), ('mangoes', 7), ('lichi', 6), ('bananas', 5), ('papaya', 4)])\n",
      "\n",
      "fruit_dict: OrderedDict([('apples', 11), ('oranges', 8), ('pineapples', 4), ('mangoes', 7), ('lichi', 6), ('bananas', 5), ('papaya', 4)])\n"
     ]
    }
   ],
   "source": [
    "from collections import OrderedDict\n",
    "import random\n",
    "\n",
    "# Insertion Order of keys is maintained\n",
    "# Accessing a missing key raises an error in a OrderedDict \n",
    "\n",
    "fruit_dict=OrderedDict()\n",
    "fruits=['apples','oranges','kiwi','pineapples','mangoes','lichi','bananas','papaya']\n",
    "for each in fruits:\n",
    "    fruit_dict[each]=random.randint(1,10)\n",
    "print(\"fruit_dict:\",fruit_dict)\n",
    "\n",
    "# Inserting a new key in the OrderedDict\n",
    "fruit_dict['cheeries']=5\n",
    "print(\"\\nfruit_dict:\",fruit_dict)\n",
    "\n",
    "# Changing a value of key in the OrderedDict\n",
    "fruit_dict['apples']=fruit_dict['apples']+3\n",
    "print(\"\\nfruit_dict:\",fruit_dict)\n",
    "\n",
    "# Moving the key 'kiwi' at the last position\n",
    "fruit_dict.move_to_end('kiwi') # by default, it moves to the right\n",
    "print(\"\\nfruit_dict:\",fruit_dict)\n",
    "\n",
    "# Moving the key 'kiwi' at the first position\n",
    "fruit_dict.move_to_end('kiwi',last=False) # by default, it moves to the right\n",
    "print(\"\\nfruit_dict:\",fruit_dict)\n",
    "\n",
    "# Removing the last key\n",
    "fruit_dict.popitem()\n",
    "print(\"\\nfruit_dict:\",fruit_dict)\n",
    "\n",
    "# Removing the first key\n",
    "fruit_dict.popitem(last=False)\n",
    "print(\"\\nfruit_dict:\",fruit_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### namedtuple\n",
    "The namedtuple() returns a tuple with names for each position in the tuple. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Student_1: student(Name='Jack', City='Berlin', Age=26)\n",
      "Student_1 Name: Jack\n",
      "\n",
      "Student_2: student(Name='Jim', City='London', Age=22)\n",
      "Student_2 City: London\n",
      "\n",
      "Student_3: student(Name='John', City='New York', Age=24)\n",
      "Student_3 Age: 24\n",
      "\n",
      "Student_4: student(Name='Joe', City='Tokyo', Age='23')\n",
      "\n",
      "Student_1 OrderedDict: OrderedDict([('Name', 'Jack'), ('City', 'Berlin'), ('Age', 26)])\n",
      "\n",
      "Student_1 Earlier: student(Name='Jack', City='Berlin', Age=26)\n",
      "Student_1 Now: student(Name='Jack', City='Berlin', Age='27')\n"
     ]
    }
   ],
   "source": [
    "from collections import namedtuple\n",
    "\n",
    "student=namedtuple(typename='student',field_names=['Name','City','Age'])\n",
    "s1=student('Jack','Berlin',26)\n",
    "s2=student('Jim','London',22)\n",
    "s3=student('John','New York',24)\n",
    "\n",
    "print('Student_1:',s1)\n",
    "print('Student_1 Name:',s1.Name)\n",
    "\n",
    "print('\\nStudent_2:',s2)\n",
    "print('Student_2 City:',s2.City)\n",
    "\n",
    "print('\\nStudent_3:',s3)\n",
    "print('Student_3 Age:',s3.Age)\n",
    "\n",
    "# _make() to create a namedtuple instance with a list\n",
    "s4 = student._make(['Joe','Tokyo','23'])\n",
    "print('\\nStudent_4:',s4)\n",
    "\n",
    "# _asdict() function can be used to create an OrderedDict instance from an existing instance\n",
    "s1_dict = s1._asdict()\n",
    "print('\\nStudent_1 OrderedDict:',s1_dict)\n",
    "\n",
    "# _replace() function changes the value of a field of an instance; it creates a new instance\n",
    "s5 = s1._replace(Age='27')\n",
    "print('\\nStudent_1 Earlier:',s1)\n",
    "print('Student_1 Now:',s5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### deque (Double Ended Queue)\n",
    "* The deque is a list optimized for inserting and removing items."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deck: deque([2, 3, 5, 7, 11, 13, 17, 19])\n",
      "Deck is rotated by 1: deque([19, 2, 3, 5, 7, 11, 13, 17])\n",
      "Deck is rotated by 2: deque([13, 17, 19, 2, 3, 5, 7, 11])\n",
      "Deck is rotated by 3: deque([5, 7, 11, 13, 17, 19, 2, 3])\n",
      "Deck is rotated by 4: deque([17, 19, 2, 3, 5, 7, 11, 13])\n",
      "Deck is rotated by 5: deque([3, 5, 7, 11, 13, 17, 19, 2])\n",
      "Deck is rotated by 6: deque([7, 11, 13, 17, 19, 2, 3, 5])\n",
      "Deck is rotated by 7: deque([11, 13, 17, 19, 2, 3, 5, 7])\n",
      "Deck is rotated by 8: deque([11, 13, 17, 19, 2, 3, 5, 7])\n",
      "Deck is rotated by 9(1): deque([7, 11, 13, 17, 19, 2, 3, 5])\n",
      "Deck is rotated by 10(2): deque([3, 5, 7, 11, 13, 17, 19, 2])\n",
      "Deck is rotated by 11(3): deque([17, 19, 2, 3, 5, 7, 11, 13])\n",
      "Deck is rotated by 12(4): deque([5, 7, 11, 13, 17, 19, 2, 3])\n",
      "Deck is rotated by 13(5): deque([13, 17, 19, 2, 3, 5, 7, 11])\n",
      "Deck is rotated by 14(6): deque([19, 2, 3, 5, 7, 11, 13, 17])\n",
      "Deck is rotated by 15(7): deque([2, 3, 5, 7, 11, 13, 17, 19])\n",
      "\n",
      "Deck: deque([2, 3, 5, 7, 11, 13, 17, 19])\n",
      "Deck is rotated by -1: deque([3, 5, 7, 11, 13, 17, 19, 2])\n",
      "Deck is rotated by -2: deque([7, 11, 13, 17, 19, 2, 3, 5])\n",
      "Deck is rotated by -3: deque([17, 19, 2, 3, 5, 7, 11, 13])\n",
      "Deck is rotated by -4: deque([5, 7, 11, 13, 17, 19, 2, 3])\n",
      "Deck is rotated by -5: deque([19, 2, 3, 5, 7, 11, 13, 17])\n",
      "Deck is rotated by -6: deque([13, 17, 19, 2, 3, 5, 7, 11])\n",
      "Deck is rotated by -7: deque([11, 13, 17, 19, 2, 3, 5, 7])\n",
      "Deck is rotated by -8: deque([11, 13, 17, 19, 2, 3, 5, 7])\n",
      "Deck is rotated by -9(-1): deque([13, 17, 19, 2, 3, 5, 7, 11])\n",
      "Deck is rotated by -10(-2): deque([19, 2, 3, 5, 7, 11, 13, 17])\n",
      "Deck is rotated by -11(-3): deque([5, 7, 11, 13, 17, 19, 2, 3])\n",
      "Deck is rotated by -12(-4): deque([17, 19, 2, 3, 5, 7, 11, 13])\n",
      "Deck is rotated by -13(-5): deque([7, 11, 13, 17, 19, 2, 3, 5])\n",
      "Deck is rotated by -14(-6): deque([3, 5, 7, 11, 13, 17, 19, 2])\n",
      "Deck is rotated by -15(-7): deque([2, 3, 5, 7, 11, 13, 17, 19])\n",
      "\n",
      "New_Deck: deque([1, 2, 3, 4, 5], maxlen=5)\n",
      "New_Deck after extending elements at the right side: deque([3, 4, 5, 6, 7], maxlen=5)\n",
      "New_Deck after extending elements at the left side: deque([7, 6, 3, 4, 5], maxlen=5)\n",
      "New_Deck after appending an element at the right side: deque([6, 3, 4, 5, 8], maxlen=5)\n",
      "New_Deck after appending an element at the left side: deque([8, 6, 3, 4, 5], maxlen=5)\n",
      "New_Deck after removing the rightmost element: deque([8, 6, 3, 4], maxlen=5)\n",
      "New_Deck after removing the leftmost element: deque([6, 3, 4], maxlen=5)\n",
      "\n",
      "Deck: deque([10, 20, 30, 20, 20, 30, 10, 30, 20])\n",
      "Count of an element in deck: 4\n",
      "Removing all elements in deque: deque([])\n"
     ]
    }
   ],
   "source": [
    "from collections import deque\n",
    "\n",
    "# Python has no specific stack or queue type\n",
    "# Deque is similar to the list type (minus sorting)\n",
    "# Both the ends of deque can be modified\n",
    "\n",
    "prime_no=[2,3,5,7,11,13,17,19]\n",
    "deck=deque(prime_no)\n",
    "print('Deck:',deck)\n",
    "\n",
    "for i in range(1,20):\n",
    "    deck.rotate(i)\n",
    "    if i>len(deck):\n",
    "        print('Deck is rotated by {0}({1}):'.format(i,i-len(deck)),deck)\n",
    "    else:\n",
    "        print('Deck is rotated by {0}:'.format(i),deck)\n",
    "    if deck==deque(prime_no):\n",
    "        break\n",
    "        \n",
    "print('\\nDeck:',deck)\n",
    "for i in range(-1,-20,-1):\n",
    "    deck.rotate(i)\n",
    "    if i<-len(deck):\n",
    "        print('Deck is rotated by {0}({1}):'.format(i,i-(-len(deck))),deck)\n",
    "    else:\n",
    "        print('Deck is rotated by {0}:'.format(i),deck)\n",
    "    if deck==deque(prime_no):\n",
    "        break\n",
    "        \n",
    "new_deck=deque(maxlen=5)\n",
    "new_deck.extend(range(1,6))\n",
    "print('\\nNew_Deck:',new_deck)\n",
    "\n",
    "# Extending new elements\n",
    "new_deck.extend([6,7]) # [6,7] are extended in the right of the queue, [1,2] are removed from the left of queue\n",
    "print('New_Deck after extending elements at the right side:',new_deck)\n",
    "\n",
    "# Extending new elements at left\n",
    "new_deck.extendleft([6,7]) # [7,6] are extended in the left of the queue, [6,7] are removed from the right of queue\n",
    "print('New_Deck after extending elements at the left side:',new_deck)\n",
    "\n",
    "# Appending new element\n",
    "new_deck.append(8)\n",
    "print('New_Deck after appending an element at the right side:',new_deck)\n",
    "\n",
    "# Appending new element at left\n",
    "new_deck.appendleft(8)\n",
    "print('New_Deck after appending an element at the left side:',new_deck)\n",
    "\n",
    "# Removing the rightmost element\n",
    "new_deck.pop()\n",
    "print('New_Deck after removing the rightmost element:',new_deck)\n",
    "\n",
    "# Removing the leftmost element\n",
    "new_deck.popleft()\n",
    "print('New_Deck after removing the leftmost element:',new_deck)\n",
    "\n",
    "\n",
    "deck=deque([10,20,30,20,20,30,10,30,20])\n",
    "print('\\nDeck:',deck)\n",
    "print('Count of an element in deck:',deck.count(20))\n",
    "deck.clear()\n",
    "print('Removing all elements in deque:',deck)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ChainMap\n",
    "ChainMap is used to combine several dictionaries or mappings. It returns a list of dictionaries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ChainMap({'a': 1, 'b': 2}, {'c': 3, 'b': 4})\n",
      "List of Dictionaries: [{'a': 1, 'b': 2}, {'c': 3, 'b': 4}]\n",
      "\n",
      "chain_map['a']: 1\n",
      "ChainMap({'a': 1, 'b': 2}, {'c': 5, 'b': 4})\n",
      "Keys: ['c', 'b', 'a']\n",
      "Values: [5, 2, 1]\n",
      "\n",
      "Updated chain_map: ChainMap({'e': 5, 'f': 6}, {'a': 1, 'b': 2}, {'c': 5, 'b': 4})\n"
     ]
    }
   ],
   "source": [
    "from collections import ChainMap\n",
    "\n",
    "dict1 = { 'a' : 1, 'b' : 2 }\n",
    "dict2 = { 'c' : 3, 'b' : 4 }\n",
    "chain_map = ChainMap(dict1, dict2)\n",
    "print(chain_map)\n",
    "\n",
    "# Fetching list of dictionaries\n",
    "print('List of Dictionaries:',chain_map.maps)\n",
    "\n",
    "# Accessing chain map values by key name\n",
    "print(\"\\nchain_map['a']:\",chain_map['a'])\n",
    "\n",
    "# Updating a key in dictionary\n",
    "dict2['c'] = 5\n",
    "print(chain_map)\n",
    "\n",
    "# Fetching Keys\n",
    "print('Keys:',list(chain_map.keys())) # printing only unique keys\n",
    "\n",
    "# Fetching Values\n",
    "print('Values:',list(chain_map.values())) \n",
    "# if one key appears in more than one associated dictionaries, ChainMap takes the value for that key from the first dictionary. \n",
    "\n",
    "# Adding new dictionary to existing chain_map\n",
    "dict3 = {'e' : 5, 'f' : 6}\n",
    "new_chain_map = chain_map.new_child(dict3)\n",
    "print('\\nUpdated chain_map:',new_chain_map)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regular Expressions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Regular Expressions**, also known as **Regexes**, are used to find different patterns of text. In general, regexes work by first specifying the rules for the set of possible patterns that we want to find, example, phone numbers, names, and email addresses, etc. \n",
    "\n",
    "**Raw Strings** are used to create our regexes due to there nature of treating `backslashes (\\)` as literal characters and not interpreting them as special characters. In these cases, we can prefix the string literal with the letter `r` or **rawstrings**. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tRegular\n",
      "Expressions\n"
     ]
    }
   ],
   "source": [
    "print('\\tRegular\\nExpressions')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\tRegular\\nExpressions\n"
     ]
    }
   ],
   "source": [
    "print(r'\\tRegular\\nExpressions')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can clearly see that by adding an r before the first quote of the string literal, both \\n and \\t, are no longer treated as special characters. It is important to note, that the `r` doesn't change the type of the string literal, but rather, it just changes how the string literal is interpreted. So, without the `r`, backslashes are used to escape characters and with the r, backslashes are treated as literal characters."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### re Module\n",
    "The `re.compile(pattern)` function converts a regular expression `pattern` into a regular expression object. This allows us to save our regular expressions into objects that can be used later to perform pattern matching using various methods, such as `.match()`, `.search()`, `.findall()`, and `.finditer()`.\n",
    "\n",
    "Steps for regular expressions: \n",
    "* Define a pattern \n",
    "* Compile the pattern \n",
    "* Match the compiled expression with the string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(6, 7), match='a'>\n",
      "<re.Match object; span=(11, 12), match='a'>\n",
      "<re.Match object; span=(17, 18), match='a'>\n",
      "<re.Match object; span=(22, 23), match='a'>\n"
     ]
    }
   ],
   "source": [
    "# Example 1\n",
    "\n",
    "sample_text = 'Alice and Walter are walking to the store.'\n",
    "\n",
    "# Create a regular expression object with the regular expression 'a'\n",
    "regex = re.compile(r'a')\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "# `.finditer()` method scans the strings from left-to-right, and returns the matches (iterator) in the order found\n",
    "matches = regex.finditer(sample_text) \n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that each match corresponds to a Match Object with a given `span` and corresponding `match`. The `span=(start,end)` is a tuple that indicates the `start` and `end` indices of the given `match` in the string `sample_text`. For example, if we look at the `span` of the first match, we can see that the first `a` is located between indices `6` through `7`. Therefore, if we print the `sample_text` string from index `6` through `7` we will see that it corresponds to the letter `a`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a\n"
     ]
    }
   ],
   "source": [
    "# Print the sample_text string from index 6 through 7\n",
    "print(sample_text[6:7])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice, however that even though the first letter in our `sample_text` is an uppercase `A`, the `.finditer()` method didn't return it as a match. This is because, regular expressions are **case sensitive**. We should note that the `re` module allows us to perform **case-insensitive** searches by the means of **Flags**. \n",
    "\n",
    "In regular expressions, the `.` is a special character known as a **Metacharacter**. Metacharacters are used to give special instructions and can't be searched for directly. If we want to search for these metacharacters directly in strings we need to escape them first. Just like with Python string literals, we can use the backslash (`\\`) to escape all the metacharacters. Here’s a complete list of the metacharacters used in regular expressions:\n",
    "\n",
    "```python\n",
    ". ^ $ * + ? { } [ ] \\ | ( )\n",
    "```\n",
    "<div class=\"alert alert-info\"><li>Used for matching patterns in strings and text<li>The '<b>.</b>' symbol matches with any character except for newline (`\\n`) character.<li>The '<b>+</b>' symbol repeats the previous symbol one or more times. So, '<b>.+</b>' will match with any string with one or more characters.<li>The '<b>@</b>' symbol only matches with the '@' character.<li>The caret '<b>^</b>' is used to match a sequence of characters when they appear at the beginning of a string.<li>The dollar sign '<b>$</b>' is used to match a sequence of characters when they appear at the end of a string.\n",
    "</div>\n",
    "\n",
    "We can form any pattern we want by using the metacharacters. The backslash can also be followed by various characters to signal various special sequences. Here is a list of the special sequences:\n",
    "* `\\d` - Matches any decimal digit; this is equivalent to the set [0-9]\n",
    "* `\\D` - Matches any non-digit character; this is equivalent to the set [^0-9]\n",
    "* `\\s` - Matches any whitespace character, this is equivalent to the set [ \\t\\n\\r\\f\\v]\n",
    "* `\\S` - Matches any non-whitespace character; this is equivalent to the set [^ \\t\\n\\r\\f\\v]\n",
    "* `\\w` - Matches any alphanumeric character and the underscore; this is equivalent to the set [a-zA-Z0-9_]\n",
    "* `\\W` - Matches any non-alphanumeric character; this is equivalent to the set [^a-zA-Z0-9_]\n",
    "\n",
    "We can see that there is a difference between lowercase and uppercase sequences. For example, while `\\d` matches any digit, `\\D` matches everything that is **not** a digit. Similarly, while `\\s` matches any whitespace character like tabs (\\t), newlines (\\n), carriage returns (\\r), form feeds (\\f), vertical tabs (\\v), `\\S` matches everything that is **not** a whitespace character; and while `\\w` matches any alphanumeric character, `\\W` matches everything that is **not** an alphanumeric character.\n",
    "\n",
    "**Character Sets** are specified using the `[]` metacharacters and are used to indicate a set of characters that we wish to match. **Within** a character set, when a dash (`-`) is placed **between** digits or letters, it is used to specify a range. For example, the character set `[6-9]` is equivalent to the character set `[6789]` and the character set `[a-f]` is equivalent to the character set `[abcdef]`.\n",
    "\n",
    "We already learned that **outside** of a character set, the caret matches a sequence of characters when they are located at the beginning of a string. However, when the caret (`^`) appears at the **beginning** of a character set it **negates** the set. This means it matches everything that is **not** in that character set. For example, the regular expression `[^6-9]` will match any character that is **not** a `6`, `7`, `8`, or `9`. Similarly, the regular expression `[^a-zA-Z] `will match any character that is **not** a lowercase or uppercase letter.\n",
    "\n",
    "We can use the **qualifier** `{m,n}` in our regular expression.  This qualifier means there must be at least `m` repetitions, and at most `n` repetitions of the previous regular expression. For example, `a/{1,3}b` will match `a/b`, `a//b`, and `a///b`. It won’t match `ab`, which has no slashes, or `a////b`, which has four slashes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(30, 36), match='$25.99'>\n",
      "\n",
      "Match from the original text: $25.99\n"
     ]
    }
   ],
   "source": [
    "# Example 2 : Getting the price\n",
    "\n",
    "sample_text = 'John bought a winter coat for $25.99 dollars.'\n",
    "\n",
    "# '\\.' specifically matches with the '.' character\n",
    "pattern=r'\\$[\\d]+\\.[\\d]+' # \\d is used for matching a digit, [\\d]+ means one or more digits\n",
    "regex = re.compile(pattern)\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)\n",
    "    \n",
    "    # Using the span information from the match, print the match from the original string\n",
    "    print('\\nMatch from the original text:', sample_text[match.span()[0]:match.span()[1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(1, 23), match='jamwinemusic@gmail.com'>\n",
      "('jamwinemusic', 'gmail', 'com')\n",
      "\n",
      "<re.Match object; span=(39, 51), match='a.w@email.us'>\n",
      "('a.w', 'email', 'us')\n",
      "\n",
      "<re.Match object; span=(62, 88), match='maria.jenson04@fps.iit.edu'>\n",
      "('maria.jenson04', 'fps.iit', 'edu')\n",
      "\n",
      "<re.Match object; span=(97, 113), match='jam.wine@jam.ind'>\n",
      "('jam.wine', 'jam', 'ind')\n",
      "\n",
      "<re.Match object; span=(124, 141), match='jam_wine@kite.org'>\n",
      "('jam_wine', 'kite', 'org')\n",
      "\n",
      "<re.Match object; span=(152, 166), match='wine@music.gov'>\n",
      "('wine', 'music', 'gov')\n",
      "\n",
      "<re.Match object; span=(204, 220), match='jam7wine@cog.org'>\n",
      "('jam7wine', 'cog', 'org')\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Example 3 : Extracting Emails from the list\n",
    "\n",
    "emails='''\n",
    "jamwinemusic@gmail.com\n",
    "Alice Walters: a.w@email.us\n",
    "Official: maria.jenson04@fps.iit.edu\n",
    "Other - jam.wine@jam.ind\n",
    "Another > jam_wine@kite.org\n",
    "Email=jam-wine@music.gov \n",
    "jam_wine.@xyz.com\n",
    "jamwine_@abc.org\n",
    "3jam7wine@cog.org\n",
    "'''\n",
    "pattern = r'([a-zA-Z][\\w\\.]+[^_\\.])@([a-zA-Z\\.]+[^\\.])\\.([a-zA-Z]+)' # the parentheses here specifies a group.\n",
    "# the '\\.' specifically matches with the '.' character.\n",
    "\n",
    "regex = re.compile(pattern)\n",
    "result = regex.finditer(emails)\n",
    "\n",
    "for each in result:\n",
    "    print(each)\n",
    "    print(each.groups())\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample Text:\n",
      " \n",
      "123\t45\t7895\n",
      "1\t222\t33\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['123', '45', '7895', '1', '222', '33']"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example 4 : Fetching all numbers from the text containing numbers and spaces\n",
    "\n",
    "sample_text = '''\n",
    "123\\t45\\t7895\n",
    "1\\t222\\t33\n",
    "'''\n",
    "# Print sample_text\n",
    "print('Sample Text:\\n', sample_text)\n",
    "\n",
    "# Create a regular expression object with the regular expression\n",
    "regex = re.compile(r'\\s') \n",
    "\n",
    "# split() works on the compiled pattern\n",
    "matches = regex.split(sample_text) \n",
    "\n",
    "# fetching all numbers\n",
    "matches[1:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result Object: ['Amit ', ' Mishra']\n",
      "\n",
      "Names:\n",
      "Akshay Kumar\n",
      "Sohan Agarwaal\n",
      "Neha Ahuja\n"
     ]
    }
   ],
   "source": [
    "# Example 5 : Determining First and Last name from the list of names\n",
    "\n",
    "name='Amit A. Mishra'\n",
    "pattern=r'[A-Z]\\.'\n",
    "regex=re.compile(pattern)\n",
    "result=regex.split(name)\n",
    "print('Result Object:',result)\n",
    "\n",
    "from collections import namedtuple\n",
    "names=['Akshay S. Kumar','Sohan B. Agarwaal','Neha A. Ahuja']\n",
    "Person=namedtuple(typename='Person',field_names=['First','Last'])\n",
    "people=[Person(*regex.split(name)) for name in names]\n",
    "print('\\nNames:')\n",
    "for p in people:\n",
    "    print(p.First.strip(),p.Last.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result Object: <re.Match object; span=(0, 3), match='200'>\n",
      "First Index the searched Instance 0\n",
      "Last Index of the searched Instance 3\n",
      "Index Range of the Searched Instance (0, 3)\n",
      "\n",
      "Result Object using match(): None\n",
      "Result Object using search(): <re.Match object; span=(4, 6), match='OK'>\n"
     ]
    }
   ],
   "source": [
    "# Example 6 : Differentiating search() and match()\n",
    "\n",
    "response='200 OK'\n",
    "\n",
    "pattern=r'\\d{3}'\n",
    "regex=re.compile(pattern)\n",
    "result=regex.search(response) # search() will look for the compiled pattern\n",
    "print('Result Object:',result)\n",
    "\n",
    "print('First Index the searched Instance',result.start())\n",
    "print('Last Index of the searched Instance',result.end())\n",
    "print('Index Range of the Searched Instance',result.span())\n",
    "\n",
    "pattern=r'[A-Z]+'\n",
    "regex=re.compile(pattern)\n",
    "result=regex.match(response) # None will be returned as there is no match \n",
    "print('\\nResult Object using match():',result)\n",
    "result=regex.search(response)\n",
    "print('Result Object using search():',result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result Object: <re.Match object; span=(15, 27), match='865-345-3453'>\n",
      "Groups: ('865', '345', '3453')\n",
      "\n",
      "Result Object: <re.Match object; span=(15, 27), match='865-345-3453'>\n",
      "Groups: ('865', '345', '3453')\n",
      "Groups with names: {'area_code': '865', 'prefix': '345', 'extension': '3453'}\n"
     ]
    }
   ],
   "source": [
    "# Example 7 : extracting phone number\n",
    "\n",
    "message='Please call at 865-345-3453 for any query.'\n",
    "\n",
    "pattern=r'(\\d{3})-(\\d{3})-(\\d{4})'\n",
    "regex=re.compile(pattern)\n",
    "result=regex.search(message)\n",
    "print('Result Object:',result)\n",
    "print('Groups:',result.groups())\n",
    "\n",
    "# Naming Groups\n",
    "pattern=r'(?P<area_code>\\d{3})-(?P<prefix>\\d{3})-(?P<extension>\\d{4})' # ?P<desired_name> is mentioned for adding name to group\n",
    "regex=re.compile(pattern)\n",
    "result=regex.search(message)\n",
    "print('\\nResult Object:',result)\n",
    "print('Groups:',result.groups())\n",
    "print('Groups with names:',result.groupdict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(12, 27), match='+1-555-123-4567'>\n",
      "<re.Match object; span=(40, 56), match='+61 455 555 4549'>\n",
      "<re.Match object; span=(70, 87), match='+375-655-777-7346'>\n",
      "<re.Match object; span=(100, 117), match='+213(555)999-8464'>\n"
     ]
    }
   ],
   "source": [
    "# (Extending) Example 7 : extracting phone number\n",
    "\n",
    "# Sample text\n",
    "sample_text = '''\n",
    "Mr. Brown: +1-555-123-4567\n",
    "Mrs. Smith: +61 455 555 4549\n",
    "Mr. Jackson: +375-655-777-7346\n",
    "Ms. Wilson: +213(555)999-8464\n",
    "'''\n",
    "\n",
    "# Create a regular expression object with a regular expression\n",
    "regex = re.compile(r'\\+\\d{1,3}.\\d{3}.\\d{3}.\\d{4}')\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 4 <b> tags having content: ['.', '+', '.+', '@']\n",
      "\n",
      "There are 0 <B> tags having content: []\n",
      "\n",
      "There are 4 bold tags (<b>,<B>) having content: ['.', '+', '.+', '@']\n"
     ]
    }
   ],
   "source": [
    "# Example 8: fetching content inside bold tags\n",
    "\n",
    "html=\"\"\"\n",
    "\n",
    "<div class=\"alert alert-info\"><li>Used for matching patterns in strings and text\n",
    "<li>The '<b>.</b>' symbol matches with any character.\n",
    "<li>The '<b>+</b>' symbol repeats the previous symbol one or more times. \n",
    "So, '<b>.+</b>' will match with any string with one or more characters.\n",
    "<li>The '<b>@</b>' symbol only matches with the '@' character.</div>\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "regex=re.compile('<b>(.+)</b>') # Finding the content of all bold tags\n",
    "results=regex.findall(html)\n",
    "print('There are {} <b> tags having content:'.format(len(results)), results)\n",
    "\n",
    "regex=re.compile('<B>(.+)</B>') # the bold tags are capitalized\n",
    "results=regex.findall(html)\n",
    "print('\\nThere are {} <B> tags having content:'.format(len(results)), results) # No results due to case sensitivity\n",
    "\n",
    "regex=re.compile('<B>(.+)</B>',flags=re.IGNORECASE) # Using re.IGNORECASE, case sensitivity is removed\n",
    "results=regex.findall(html)\n",
    "print('\\nThere are {} bold tags (<b>,<B>) having content:'.format(len(results)), results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original text: \n",
      "Ben & Jerry\n",
      "Jack & Jill\n",
      "\n",
      "Modified text: \n",
      "Ben and Jerry\n",
      "Jack and Jill\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Example 9 : substituting text\n",
    "\n",
    "# Sample text\n",
    "sample_text = '''\n",
    "Ben & Jerry\n",
    "Jack & Jill\n",
    "'''\n",
    "\n",
    "# Create a regular expression object with the regular expression '&'\n",
    "regex = re.compile(r'&')\n",
    "\n",
    "# Substitute all & in the sample_text with 'and'\n",
    "new_text = regex.sub(r'and', sample_text)  # sub() is used to substitute the compiled expression\n",
    "\n",
    "# Print Original and Modified texts\n",
    "print('Original text:', sample_text)\n",
    "print('Modified text:', new_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original Text:\n",
      " 1 &lt; 2\n",
      "\n",
      "Text after substitution:\n",
      " 1 < 2\n",
      "\n",
      "----\n",
      "\n",
      "Original Text:\n",
      " 4 &gt; 3\n",
      "\n",
      "Text after substitution:\n",
      " 4 &gt; 3\n"
     ]
    }
   ],
   "source": [
    "# Example 9 : substituting text (Extending)\n",
    "\n",
    "text='1 &lt; 2'\n",
    "print('Original Text:\\n',text)\n",
    "regex=re.compile('&lt;') # &lt; represents less than (<) sign\n",
    "result=regex.sub('<',text)\n",
    "print('\\nText after substitution:\\n',result)\n",
    "\n",
    "text='4 &gt; 3'\n",
    "print('\\n----\\n\\nOriginal Text:\\n',text)\n",
    "regex=re.compile('&lt;')\n",
    "result=regex.sub('>',text) # Substitution won't work as the compiled expression is unavailable in the original text\n",
    "print('\\nText after substitution:\\n',result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Word Boundaries \n",
    "`\\b` is a special sequence that doesn't really match a particular set of characters, but rather determines word boundaries. A word in this context is defined as a sequence of alphanumeric characters, while a boundary is defined as a white space, a non-alphanumeric character, or the beginning or end of a string. We can have boundaries either before or after a word. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(12, 17), match='class'>\n",
      "<re.Match object; span=(47, 52), match='class'>\n",
      "<re.Match object; span=(85, 90), match='class'>\n"
     ]
    }
   ],
   "source": [
    "# Example 10  : Word Boundaries\n",
    "\n",
    "# Sample text\n",
    "sample_text = 'The biology class will meet in the first floor classroom to learn about Theria, a subclass of mammals.'\n",
    "\n",
    "# Create a regular expression object with the regular expression 'class'\n",
    "regex = re.compile(r'class')\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that we have three matches, corresponding to all the instances (`class`,`classroom`,`subclass`) of the word `class`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(12, 17), match='class'>\n",
      "<re.Match object; span=(47, 52), match='class'>\n"
     ]
    }
   ],
   "source": [
    "# Adding `\\b` before the word `class` in our regular expression \n",
    "# to only find the word `class` when it appears at the beginning of a word\n",
    "\n",
    "# Sample text\n",
    "sample_text = 'The biology class will meet in the first floor classroom to learn about Theria, a subclass of mammals.'\n",
    "\n",
    "# Create a regular expression object with the regular expression '\\bclass'\n",
    "regex = re.compile(r'\\bclass')\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that now we only have two matches because it's only matching the stand-alone word, `class`, and the `class` in `classroom` since both of them have a word boundary (in this case a white space) directly before them. We can also see that it is not matching the `class` in `subclass` because there is no word boundary directly before it. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(12, 17), match='class'>\n",
      "<re.Match object; span=(85, 90), match='class'>\n"
     ]
    }
   ],
   "source": [
    "# Adding `\\b` after the word `class` in our regular expression \n",
    "# to only find the word `class` when it appears at the end of a word\n",
    "\n",
    "# Sample text\n",
    "sample_text = 'The biology class will meet in the first floor classroom to learn about Theria, a subclass of mammals.'\n",
    "\n",
    "# Create a regular expression object with the regular expression 'class\\b'\n",
    "regex = re.compile(r'class\\b')\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that in this case we have two matches as well because it's matching the stand-alone word, `class` again, and the `class` in `subclass` since both of them have a word boundary (in this case a white space) directly after them. We can also see that it is not matching the `class` in `classroom` because there is no word boundary directly after it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(12, 17), match='class'>\n"
     ]
    }
   ],
   "source": [
    "# Adding `\\b` both before and after the word `class` in our regular expression \n",
    "# to only find the word `class` when it appears as a stand-alone word\n",
    "\n",
    "# Sample text\n",
    "sample_text = 'The biology class will meet in the first floor classroom to learn about Theria, a subclass of mammals.'\n",
    "\n",
    "# Create a regular expression object with the regular expression '\\bclass\\b'\n",
    "regex = re.compile(r'\\bclass\\b')\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that now we only have one match because the stand-alone word, `class`, is the only one that has a word boundary (in this case a white space) directly before and after it.\n",
    "\n",
    "`\\B` indicates the opposite of `\\b`, i.e. if `\\b` is used to indicate a word boundary, `\\B` is used to indicate **not** a word boundary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(85, 90), match='class'>\n"
     ]
    }
   ],
   "source": [
    "# Sample text\n",
    "sample_text = 'The biology class will meet in the first floor classroom to learn about Theria, a subclass of mammals.'\n",
    "\n",
    "# Create a regular expression object with the regular expression '\\Bclass'\n",
    "regex = re.compile(r'\\Bclass')\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that we only get one match because the `class` in `subclass` is the only one that **doesn't** have a word boundary directly before it. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(47, 52), match='class'>\n"
     ]
    }
   ],
   "source": [
    "# Sample text\n",
    "sample_text = 'The biology class will meet in the first floor classroom to learn about Theria, a subclass of mammals.'\n",
    "\n",
    "# Create a regular expression object with the regular expression 'class\\B'\n",
    "regex = re.compile(r'class\\B')\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that again we only have one match because the `class` in `classroom` is the only one that **doesn't** have a boundary directly after it. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sample text\n",
    "sample_text = 'The biology class will meet in the first floor classroom to learn about Theria, a subclass of mammals.'\n",
    "\n",
    "# Create a regular expression object with the regular expression '\\Bclass\\B'\n",
    "regex = re.compile(r'\\Bclass\\B')\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this case, we can see that we get no matches. This is because all instances of the word `class` in our `sample_text` string, have a boundary either before or after it. In order to have a match in this case, the word `class` will have to appear in the middle of a word, such as in the word `declassified`. Let's see an example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(2, 7), match='class'>\n"
     ]
    }
   ],
   "source": [
    "# Sample text\n",
    "sample_text = 'declassified'\n",
    "\n",
    "# Create a regular expression object with the regular expression '\\Bclass\\B'\n",
    "regex = re.compile(r'\\Bclass\\B')\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(13, 16), match='the'>\n",
      "<re.Match object; span=(30, 33), match='his'>\n",
      "<re.Match object; span=(34, 37), match='car'>\n",
      "<re.Match object; span=(43, 46), match='but'>\n",
      "<re.Match object; span=(57, 60), match='buy'>\n"
     ]
    }
   ],
   "source": [
    "# Example 11 : match all 3-letter words\n",
    "\n",
    "# Sample text\n",
    "sample_text = 'John went to the store 456 in his car A5T, but forgot to buy bread B-G.'\n",
    "\n",
    "# Create a regular expression object with the regular expression \n",
    "regex = re.compile(r'\\b[a-zA-Z]{3}\\b')\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(2, 3), match='3'>\n",
      "<re.Match object; span=(8, 9), match='3'>\n",
      "<re.Match object; span=(16, 17), match='3'>\n",
      "<re.Match object; span=(24, 25), match='3'>\n",
      "<re.Match object; span=(31, 32), match='3'>\n",
      "\n",
      "Total Number of Matches: 5\n"
     ]
    }
   ],
   "source": [
    "# Example 12 : count how many numbers (greater than 3), have 3 as their last digit, for example: 73\n",
    "\n",
    "# Sample text\n",
    "sample_text = '203 3 403 687 283 234 983 345 23 3 74 978'\n",
    "\n",
    "# Create a regular expression object with the regular expression\n",
    "regex = re.compile(r'\\B3\\b')\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# Set counter\n",
    "num_matches = 0\n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)\n",
    "    num_matches += 1\n",
    "    \n",
    "# Print the total number of matches    \n",
    "print('\\nTotal Number of Matches:', num_matches)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `?` metacharacter in our regular expression matches 0 or 1 repetitions of the preceding regular expression. For example, the regular expression `ab?` will match either `a` or `ab`. In other words, the `?` after the `b` indicates that the `b` after the `a` is optional.\n",
    "\n",
    "The `*` metacharacter, matches 0 or more repetitions of the preceding regular expression. In other words, it matches 0 or as many repetitions as possible of the preceding regular expression. For example, the regular expression `ab*` will match `a` or `a` followed by any number of `b`'s, such as `ab` or `abbbbb`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(1, 11), match='Mt Everest'>\n",
      "<re.Match object; span=(28, 34), match='Mt. K2'>\n",
      "<re.Match object; span=(51, 67), match='Mt Kangchenjunga'>\n",
      "<re.Match object; span=(84, 94), match='Mt. Lhotse'>\n",
      "<re.Match object; span=(111, 121), match='Mnt makalu'>\n"
     ]
    }
   ],
   "source": [
    "# Example 13\n",
    "\n",
    "# Sample text\n",
    "sample_text = '''\n",
    "Mt Everest: Height 8,848 m\n",
    "Mt. K2: Height 8,611 m\n",
    "Mt Kangchenjunga: Height 8,586 m\n",
    "Mt. Lhotse: Height 8,516 m\n",
    "Mnt makalu: Height 8,485 m\n",
    "'''\n",
    "\n",
    "# Create a regular expression object with a regular expression that can match all the mountain names\n",
    "regex = re.compile(r'M(t|nt)\\.?\\s[a-zA-Z]\\w*')\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "First Name: John\n",
      "Middle Name: David\n",
      "Last Name: Smith\n",
      "\n",
      "First Name: Alice\n",
      "Middle Name: None\n",
      "Last Name: Jackson\n",
      "\n",
      "First Name: Mary\n",
      "Middle Name: Elizabeth\n",
      "Last Name: Wilson\n",
      "\n",
      "First Name: Mike\n",
      "Middle Name: None\n",
      "Last Name: Brown\n",
      "\n",
      "John Smith\n",
      "Alice Jackson\n",
      "Mary Wilson\n",
      "Mike Brown\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Example 14 : replace all names in the string with just the first and last name\n",
    "\n",
    "# Sample text\n",
    "sample_text = '''\n",
    "John David Smith\n",
    "Alice Jackson\n",
    "Mary Elizabeth Wilson\n",
    "Mike Brown\n",
    "'''\n",
    "\n",
    "# Create a regular expression object with a regular expression that can find all\n",
    "# the names in the sample_text and group the first, middle, and last names separately\n",
    "regex = re.compile(r'([a-zA-z]+)[ ]?([a-zA-z]+)?[ ]([a-zA-z]+)')\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# For each match print the first, middle, and last name separately\n",
    "for match in matches:\n",
    "    print('\\nFirst Name: '+ match.group(1))\n",
    "    if match.group(2) is None:\n",
    "        print('Middle Name: None')\n",
    "    else:\n",
    "        print('Middle Name: '+ match.group(2))\n",
    "    print('Last Name: '+ match.group(3))\n",
    "    \n",
    "# Substitute all names in the sample_text with the first and last name\n",
    "new_text = regex.sub(r'\\1 \\3', sample_text) # replacing every match with the first and third groups\n",
    "\n",
    "# Print the modified text\n",
    "print(new_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(10, 16), match='WaLtEr'>\n",
      "<re.Match object; span=(40, 46), match='wAlTer'>\n"
     ]
    }
   ],
   "source": [
    "# Example 15: find the name Walter (by ignoring case sensitivity)\n",
    "\n",
    "# Sample text\n",
    "sample_text = 'Alice and WaLtEr Brown are talking with wAlTer Jackson.'\n",
    "\n",
    "# Create a regular expression object with the regular expression 'walter' that ignores the case of the letters\n",
    "regex = re.compile(r'walter', re.IGNORECASE)\n",
    "\n",
    "# Search the sample_text for the regular expression\n",
    "matches = regex.finditer(sample_text)\n",
    "\n",
    "# Print all the matches\n",
    "for match in matches:\n",
    "    print(match)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Regex Cross­word\n",
    "\n",
    "* https://regexcrossword.com/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Web Scraping\n",
    "\n",
    "**Web scraping** is the automated extraction of data from websites by making HTTP requests programmatically. Website content is first fetched usually using HTTP and then parsed to extract specific information. Web scraping involves two phases, fetching content from a target URL and then parsing the content. \n",
    "\n",
    "\n",
    "### HTTP Requests and Responses\n",
    "For accessing content that is hosted on the web, we usually open up our web browser and type in a URL. A **URL** is a resource locator which allows us to uniquely identify where we want to get our content from by identifying the target resource. \n",
    "\n",
    "Under the hood, our web page is making an **HTTP request** to a server either from a browser on our desktop or from our mobile phone. This HTTP request is then propagated through the internet, and it hits the web server where the resources hosted are located, where the website is hosted. This, in result, returns an actual content from the target resource in the form of an **HTTP response**. In addition, the HTTP response will also contain metadata information in the form of headers and includes a status code, like 404 or 200. 200 says the response was sent back successfully whereas 404 says the page was not found. \n",
    "\n",
    "**HTTP** stands for **hypertext transfer protocol** is a simple, ubiquitous text-based protocol used by browsers and apps to access and modify web content. HTTP is a client-server protocol. The client makes requests and receives responses from the server. Common web servers that are capable of hosting web content are Apache, nginx, IIS from Microsoft, and GWS from Google. \n",
    "\n",
    "There are actually different types of HTTP requests GET, POST, PUT, DELETE, etc. The **GET request** fetches resources from the target URL. **POST request** is used to create or update resources. If our update or creation is idempotent, the **PUT request** is used, i.e. if the same PUT request is made to a server more than once, the additional PUT request should have no effect (idempotent). **HEAD request** is used to make a request to get only the metadata header information from a target URL. To delete resources on a site, the **DELETE request** is used. \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "### urllib Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "type(urllib): <class 'module'>\n",
      "type(urllib.request): <class 'module'>\n",
      "\n",
      "urllib.__path__: ['c:\\\\users\\\\shaur\\\\appdata\\\\local\\\\programs\\\\python\\\\python37\\\\lib\\\\urllib']\n",
      "Error, module object has no attribute '__path__'\n",
      "\n",
      "Type: <class 'http.client.HTTPResponse'>\n",
      "Accessing Url: https://en.wikipedia.org/wiki/List_of_state_and_union_territory_capitals_in_India\n",
      "page.status: 200\n",
      "page.reason: OK\n",
      "\n",
      "Printing Data:\n",
      " \n",
      " <!DOCTYPE html>\n",
      " <html class=\"client-nojs\" lang=\"en\" dir=\"ltr\">\n",
      " <head>\n",
      " <meta charset=\"UTF-8\"/>\n",
      " <title>List of state and union territory capitals in India - Wikipedia</title>\n",
      " <script>document.documentElement.className=\"client-js\";RLCONF={\"wgBreakFrames\":!1,\"wgSeparatorTransformTable\":[\"\",\"\"],\"w\n"
     ]
    }
   ],
   "source": [
    "import urllib # urllib is a package\n",
    "import urllib.request # urllib.request is a module\n",
    "print('type(urllib):',type(urllib))\n",
    "print('type(urllib.request):',type(urllib.request))\n",
    "\n",
    "print('\\nurllib.__path__:',urllib.__path__) # Packages are generally directories\n",
    "try: \n",
    "    print('urllib.request.__path__:',urllib.request.__path__) # Modules are generally files\n",
    "except:     \n",
    "    print(\"Error, module object has no attribute '__path__'\") # path is not available\n",
    "\n",
    "# urllib.request.urlopen is used for opening URLs, usually employing the HTTP protocol\n",
    "from urllib.request import urlopen\n",
    "url = \"https://en.wikipedia.org/wiki/List_of_state_and_union_territory_capitals_in_India\"\n",
    "\n",
    "# request for opening url\n",
    "page=urlopen(url)\n",
    "print('\\nType:',type(page))# page contains the HTTPResponse object\n",
    "print('Accessing Url:',page.url)\n",
    "print('page.status:',page.status)\n",
    "print('page.reason:',page.reason)\n",
    "\n",
    "lines=page.readlines() # contains the content of the webpage\n",
    "data=' '.join([line.decode('utf-8') for line in lines]) # converting webpage from bytes into python string\n",
    "print('\\nPrinting Data:\\n',data[:300])\n",
    "\n",
    "page.close() # Closing the connection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "URL: https://www.google.com/search?ei=w98kXbKUKbniz7sP_cOhgAc&q=yes+bank+share+price&oq=yes+bank+&gs_l=psy-ab.3.0.35i39i285i70i250j35i39j0i131i20i263j0i131i67j0l3j0i131j0l2.8198.9575..10578...0.0..0.678.2226.2-2j2j1j1......0....1..gws-wiz.......0i71j35i39i285j0i67.6E-N83j_AI0\n",
      "\n",
      "Scheme: https\n",
      "Path: /search\n",
      "Query: ei=w98kXbKUKbniz7sP_cOhgAc&q=yes+bank+share+price&oq=yes+bank+&gs_l=psy-ab.3.0.35i39i285i70i250j35i39j0i131i20i263j0i131i67j0l3j0i131j0l2.8198.9575..10578...0.0..0.678.2226.2-2j2j1j1......0....1..gws-wiz.......0i71j35i39i285j0i67.6E-N83j_AI0\n"
     ]
    }
   ],
   "source": [
    "# urllib.parse is used to fetch components of url (Parsing URLs)\n",
    "\n",
    "from urllib.request import urlparse\n",
    "\n",
    "url = 'https://www.google.com/search?ei=w98kXbKUKbniz7sP_cOhgAc&q=yes+bank+share+price&oq=yes+bank+&gs_l=psy-ab.3.0.35i39i285i70i250j35i39j0i131i20i263j0i131i67j0l3j0i131j0l2.8198.9575..10578...0.0..0.678.2226.2-2j2j1j1......0....1..gws-wiz.......0i71j35i39i285j0i67.6E-N83j_AI0'\n",
    "\n",
    "result=urlparse(url)\n",
    "print('URL:',url)\n",
    "print('\\nScheme:',result.scheme)\n",
    "print('Path:',result.path)\n",
    "print('Query:',result.query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<!DOCTYPE',\n",
       " 'html>',\n",
       " '<html',\n",
       " 'class=\"client-nojs\"',\n",
       " 'lang=\"en\"',\n",
       " 'dir=\"ltr\">',\n",
       " '<head>',\n",
       " '<meta',\n",
       " 'charset=\"UTF-8\"/>',\n",
       " '<title>List',\n",
       " 'of',\n",
       " 'state',\n",
       " 'and',\n",
       " 'union',\n",
       " 'territory',\n",
       " 'capitals',\n",
       " 'in',\n",
       " 'India',\n",
       " '-',\n",
       " 'Wikipedia</title>',\n",
       " '<script>document.documentElement.className=\"client-js\";RLCONF={\"wgBreakFrames\":!1,\"wgSeparatorTransformTable\":[\"\",\"\"],\"wgDigitTransformTable\":[\"\",\"\"],\"wgDefaultDateFormat\":\"dmy\",\"wgMonthNames\":[\"\",\"January\",\"February\",\"March\",\"April\",\"May\",\"June\",\"July\",\"August\",\"September\",\"October\",\"November\",\"December\"],\"wgRequestId\":\"5ac4ac5f-7cd3-40b1-bfd3-8ccab7637956\",\"wgCSPNonce\":!1,\"wgCanonicalNamespace\":\"\",\"wgCanonicalSpecialPageName\":!1,\"wgNamespaceNumber\":0,\"wgPageName\":\"List_of_state_and_union_territory_capitals_in_India\",\"wgTitle\":\"List',\n",
       " 'of',\n",
       " 'state',\n",
       " 'and',\n",
       " 'union',\n",
       " 'territory',\n",
       " 'capitals',\n",
       " 'in',\n",
       " 'India\",\"wgCurRevisionId\":969382718,\"wgRevisionId\":969382718,\"wgArticleId\":2371868,\"wgIsArticle\":!0,\"wgIsRedirect\":!1,\"wgAction\":\"view\",\"wgUserName\":null,\"wgUserGroups\":[\"*\"],\"wgCategories\":[\"Harv',\n",
       " 'and']"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fetching all words from a url\n",
    "\n",
    "from urllib.request import urlopen\n",
    "url = \"https://en.wikipedia.org/wiki/List_of_state_and_union_territory_capitals_in_India\"\n",
    "\n",
    "with urlopen(url) as page:\n",
    "    page_words=[]\n",
    "    for line in page:\n",
    "        line_words=line.decode('utf-8').split()\n",
    "        for word in line_words:\n",
    "            page_words.append(word)\n",
    "        \n",
    "page_words[:30]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BeautifulSoup Library \n",
    "**BeautifulSoup** is used to pull data out of HTML and XML files. Using **parsers**, it transforms file into a tree of Python objects that can be easily searched. \n",
    "\n",
    "The parser is a piece of software whose primary job is to build a data structure in the form of a hierarchical tree that gives a structural representation of the HTML or XML file. In other words, the parser divides these complex files into simpler parts while keeping track of how these parts are related to each other. \n",
    "\n",
    "BeautifulSoup supports a number of parsers. The **lxml parser** can be used to parse both HTML and XML files and has the advantage of being very fast. In order to use the lxml parser, we must have lxml installed. \n",
    "```bash\n",
    "$ pip install lxml\n",
    "```\n",
    "If we're working with perfectly formatted HTML or XML files (i.e. files that don't contain any missing information or mistakes) then, our choice of parser shouldn't really matter. However, if the files we are working with have missing information or mistakes, then our choice of parser will matter because each parser has different rules for dealing with missing information or mistakes. Consequently, in these cases, different parsers will create different parse trees for the same document. \n",
    "\n",
    "##### Differences between parsers: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#differences-between-parsers\n",
    "\n",
    "\n",
    "#### Parsing an HTML File\n",
    "\n",
    "To parse an HTML or XML document, we need to pass the document into the BeautifulSoup constructor. The BeautifulSoup constructor, `BeautifulSoup(file, 'parser')`, parses the given `file` using the given `parser` and returns a BeautifulSoup object. We can pass our `file` to the constructor either as a string or as an open filehandle, while the `parser` is a string that indicates the parser we want to use.\n",
    "\n",
    "The BeautifulSoup constructor will transform the HTML or XML file into a complex tree of Python objects. One of this objects is the BeautifulSoup object returned by the constructor. The BeautifulSoup object itself represents the document as a whole and can be searched using various methods, as we will see in the following lessons. \n",
    "\n",
    "In the code below, we start by importing the BeautifulSoup library by using the statement:\n",
    "\n",
    "```python\n",
    "from bs4 import BeautifulSoup\n",
    "```\n",
    "\n",
    "We then open the `sample.html` file and pass the open filehandle `f` to the `BeautifulSoup` constructor. We also set the `parser` in the constructor to `lxml` to indicate that we want to use lxml’s HTML parser. The `BeautifulSoup` constructor will return a BeautifulSoup object that we will save in the `page_content` variable. We then print the BeautifulSoup object to see what it looks like."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html lang=\"en-US\">\n",
      "<head>\n",
      "<title>AI For Trading</title>\n",
      "<meta charset=\"utf-8\"/>\n",
      "<link href=\"./teststyle.css\" rel=\"stylesheet\"/>\n",
      "<style>.h2style {background-color: tomato;color: white;padding: 10px;}</style>\n",
      "</head>\n",
      "<body>\n",
      "<h1 id=\"intro\">Get Help From Peers and Mentors</h1>\n",
      "<div class=\"section\">\n",
      "<h2 class=\"h2style\" id=\"hub\">Student Hub</h2>\n",
      "<p>Student Hub is our real time collaboration platform where you can work with peers and mentors. You will find Community rooms with other students and alumni.</p>\n",
      "</div>\n",
      "<hr/>\n",
      "<div class=\"section\">\n",
      "<h2 class=\"h2style\" id=\"know\">Knowledge</h2>\n",
      "<p>Search or ask questions in <a href=\"https://knowledge.udacity.com/\">Knowledge</a></p>\n",
      "</div>\n",
      "<div class=\"outro\">\n",
      "<h3 id=\"know\">Good Luck</h3>\n",
      "<p>Good luck and we hope you enjoy the course</p>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n"
     ]
    }
   ],
   "source": [
    "# Import BeautifulSoup\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Open the HTML file and create a BeautifulSoup Object\n",
    "with open('./sample.html') as f:\n",
    "    page_content = BeautifulSoup(f, 'lxml')\n",
    "\n",
    "# Print the BeautifulSoup Object\n",
    "print(page_content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see, `page_content`, holds the entire contents of our `sample.html` file. Notice that when we print the BeautifulSoup object, it is not printed in a nice format and it is very hard to read. Luckily, the BeautifulSoup object has the `.prettify()` method that allows our BeautifulSoup object to be printed with all the tags nicely indented. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html lang=\"en-US\">\n",
      " <head>\n",
      "  <title>\n",
      "   AI For Trading\n",
      "  </title>\n",
      "  <meta charset=\"utf-8\"/>\n",
      "  <link href=\"./teststyle.css\" rel=\"stylesheet\"/>\n",
      "  <style>\n",
      "   .h2style {background-color: tomato;color: white;padding: 10px;}\n",
      "  </style>\n",
      " </head>\n",
      " <body>\n",
      "  <h1 id=\"intro\">\n",
      "   Get Help From Peers and Mentors\n",
      "  </h1>\n",
      "  <div class=\"section\">\n",
      "   <h2 class=\"h2style\" id=\"hub\">\n",
      "    Student Hub\n",
      "   </h2>\n",
      "   <p>\n",
      "    Student Hub is our real time collaboration platform where you can work with peers and mentors. You will find Community rooms with other students and alumni.\n",
      "   </p>\n",
      "  </div>\n",
      "  <hr/>\n",
      "  <div class=\"section\">\n",
      "   <h2 class=\"h2style\" id=\"know\">\n",
      "    Knowledge\n",
      "   </h2>\n",
      "   <p>\n",
      "    Search or ask questions in\n",
      "    <a href=\"https://knowledge.udacity.com/\">\n",
      "     Knowledge\n",
      "    </a>\n",
      "   </p>\n",
      "  </div>\n",
      "  <div class=\"outro\">\n",
      "   <h3 id=\"know\">\n",
      "    Good Luck\n",
      "   </h3>\n",
      "   <p>\n",
      "    Good luck and we hope you enjoy the course\n",
      "   </p>\n",
      "  </div>\n",
      " </body>\n",
      "</html>\n"
     ]
    }
   ],
   "source": [
    "# Print the BeautifulSoup Object with prettify\n",
    "print(page_content.prettify())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Navigating The Parse Tree\n",
    "\n",
    "The most straightforward way of navigating the parse tree created by BeautifulSoup is by accessing the HTML or XML tags. Whenever we access a tag in this manner, we get a **Tag** object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<head>\n",
      " <title>\n",
      "  AI For Trading\n",
      " </title>\n",
      " <meta charset=\"utf-8\"/>\n",
      " <link href=\"./teststyle.css\" rel=\"stylesheet\"/>\n",
      " <style>\n",
      "  .h2style {background-color: tomato;color: white;padding: 10px;}\n",
      " </style>\n",
      "</head>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Import BeautifulSoup\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Open the HTML file and create a BeautifulSoup Object\n",
    "with open('./sample.html') as f:\n",
    "    page_content = BeautifulSoup(f, 'lxml')\n",
    "\n",
    "# Access the head tag\n",
    "page_head = page_content.head\n",
    "\n",
    "# Print the Tag Object\n",
    "print(page_head.prettify())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see, the `page_head` object has the entire contents of the `<head>` tag only, including all the opening and closing tags within it. These sub tags are known as children of the `<head>` tag. For example, the `<title>` tag is a child of the `<head>` tag. \n",
    "\n",
    "We can access these child tags within the `<head>` tag as if they were attributes of the `page_head` object. For example, if we wanted to access the `<title>` tag within the `<head>` tag, we can use:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<title>AI For Trading</title>"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Access the title tag within the head tag from the page_head object\n",
    "page_head.title"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that the above statement, `page_head.title`, is equivalent to the statement:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<title>AI For Trading</title>"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Access the title tag within the head tag from the page_content object\n",
    "page_content.head.title"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tag objects contain the beginning and end tags as well as the text within them. In most cases however we do not want the tags, but rather, only the text contained within the tags. For example, let's suppose we only wanted to get the text `AI for Trading` contained within the `title` tags. In these cases we can use the `.get_text()` method. The `.get_text()` method only gets the text part of a document or tag."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AI For Trading\n"
     ]
    }
   ],
   "source": [
    "# Print only the text in the title tag within the head tag\n",
    "print(page_head.title.get_text())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "AI For Trading\n",
      "\n",
      "\n",
      ".h2style {background-color: tomato;color: white;padding: 10px;}\n",
      "\n",
      "\n",
      "Get Help From Peers and Mentors\n",
      "\n",
      "Student Hub\n",
      "Student Hub is our real time collaboration platform where you can work with peers and mentors. You will find Community rooms with other students and alumni.\n",
      "\n",
      "\n",
      "\n",
      "Knowledge\n",
      "Search or ask questions in Knowledge\n",
      "\n",
      "\n",
      "Good Luck\n",
      "Good luck and we hope you enjoy the course\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# remove all the HTML tags and print only the text in the whole document\n",
    "print(page_content.get_text())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<meta charset=\"utf-8\"/>\n"
     ]
    }
   ],
   "source": [
    "# Accessing The Meta Tag\n",
    "page_meta = page_content.head.meta\n",
    "\n",
    "# Print the Tag Object\n",
    "print(page_meta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<h1 id=\"intro\">Get Help From Peers and Mentors</h1>\n"
     ]
    }
   ],
   "source": [
    "# Access the h1 tag\n",
    "page_h1 = page_content.body.h1\n",
    "\n",
    "# Print the Tag Object\n",
    "print(page_h1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Getting Attributes\n",
    "\n",
    "An HTML or XML tag can have many attributes. For example, the tag:\n",
    "\n",
    "```python\n",
    "<h1 id='intro'>\n",
    "```\n",
    "has the attribute `id` whose value is `'intro'`. BeautifulSoup allows us to get the value of a tag’s attribute by treating the tag like a dictionary. For example, in the code below we get the value of the `id` attribute of the `<h1>` tag by using:\n",
    "\n",
    "```python\n",
    "page_h1['id']\n",
    "```\n",
    "\n",
    "where `page_h1` is the Tag object that holds the contents of the `<h1>` tag."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "intro\n"
     ]
    }
   ],
   "source": [
    "# Get the value of the id attribute from the h1 tag\n",
    "h1_id_attr = page_h1['id']\n",
    "\n",
    "# Print the value of the id attribute\n",
    "print(h1_id_attr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://knowledge.udacity.com/\n"
     ]
    }
   ],
   "source": [
    "# Access the <a> (hyperlink) tag\n",
    "page_hyperlink = page_content.a\n",
    "\n",
    "# Get the href attribute from the a tag\n",
    "href_attr = page_hyperlink['href']\n",
    "\n",
    "# Print the href attribute\n",
    "print(href_attr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Finding all tags\n",
    "Our `sample.html` file has two `<h2>` tags:\n",
    "\n",
    "```python\n",
    "<h2 class=\"h2style\" id=\"hub\">Student Hub</h2>\n",
    "```\n",
    "\n",
    "and\n",
    "\n",
    "```python\n",
    "<h2 class=\"h2style\" id=\"know\">Knowledge</h2>\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<h2 class=\"h2style\" id=\"hub\">Student Hub</h2>"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get h2 tag\n",
    "page_content.body.h2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see, we only get the first `<h2>` tag and not both. This is because, when we access a tag as an attribute, we only get the first occurrence of that tag in the document. \n",
    "\n",
    "In order to get all the `<h2>` tags we need to use the `.find_all()` search method.\n",
    "\n",
    "BeautifulSoup provides a number of methods for searching the parse tree, example `.find_all()` method. \n",
    "\n",
    "> [BeautifulSoup Documentation](https://www.crummy.com/software/BeautifulSoup/bs4/doc/).\n",
    "\n",
    "#### Searching The Parse Tree\n",
    "\n",
    "The `.find_all(filter)` method will search an entire document for the given `filter`. The `filter` can be a string containing the HTML or XML tag name, a tag attribute, or even a regular expression. I\n",
    "\n",
    "So let's begin by using the `.find_all()` method to find all `<h2>` tags in our `sample.html` file. To do this, we will pass the string `'h2'` to the `.find_all()` method as shown in the code below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<h2 class=\"h2style\" id=\"hub\">Student Hub</h2>, <h2 class=\"h2style\" id=\"know\">Knowledge</h2>]\n"
     ]
    }
   ],
   "source": [
    "# Find all the h2 tags\n",
    "h2_list = page_content.find_all('h2')\n",
    "\n",
    "# Print the h2_list\n",
    "print(h2_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<h2 class=\"h2style\" id=\"hub\">Student Hub</h2>\n",
      "<h2 class=\"h2style\" id=\"know\">Knowledge</h2>\n"
     ]
    }
   ],
   "source": [
    "# Print each tag in the h2_list\n",
    "for tag in h2_list:\n",
    "    print(tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<p>\n",
      " Student Hub is our real time collaboration platform where you can work with peers and mentors. You will find Community rooms with other students and alumni.\n",
      "</p>\n",
      "\n",
      "<p>\n",
      " Search or ask questions in\n",
      " <a href=\"https://knowledge.udacity.com/\">\n",
      "  Knowledge\n",
      " </a>\n",
      "</p>\n",
      "\n",
      "<p>\n",
      " Good luck and we hope you enjoy the course\n",
      "</p>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Find All The <p> Tags\n",
    "p_list = page_content.find_all('p')\n",
    "\n",
    "# Print each tag in the p_list\n",
    "for tag in p_list:\n",
    "    print(tag.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<h2 class=\"h2style\" id=\"hub\">\n",
      " Student Hub\n",
      "</h2>\n",
      "\n",
      "<p>\n",
      " Student Hub is our real time collaboration platform where you can work with peers and mentors. You will find Community rooms with other students and alumni.\n",
      "</p>\n",
      "\n",
      "<h2 class=\"h2style\" id=\"know\">\n",
      " Knowledge\n",
      "</h2>\n",
      "\n",
      "<p>\n",
      " Search or ask questions in\n",
      " <a href=\"https://knowledge.udacity.com/\">\n",
      "  Knowledge\n",
      " </a>\n",
      "</p>\n",
      "\n",
      "<p>\n",
      " Good luck and we hope you enjoy the course\n",
      "</p>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print all the h2 and p tags\n",
    "for tag in page_content.find_all(['h2', 'p']):\n",
    "    print(tag.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<link href=\"./teststyle.css\" rel=\"stylesheet\"/>\n",
      "\n",
      "<a href=\"https://knowledge.udacity.com/\">\n",
      " Knowledge\n",
      "</a>\n"
     ]
    }
   ],
   "source": [
    "# Print all the <a> and <link> tags\n",
    "for tag in page_content.find_all(['a', 'link']):\n",
    "    print(tag.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<h2 class=\"h2style\" id=\"know\">Knowledge</h2>\n"
     ]
    }
   ],
   "source": [
    "# Find the h2 tag with id = know\n",
    "h2_know = page_content.find_all('h2', id = 'know')\n",
    "\n",
    "# Print each item in the h2_know\n",
    "for tag in h2_know:\n",
    "    print(tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<h1 id=\"intro\">Get Help From Peers and Mentors</h1>\n"
     ]
    }
   ],
   "source": [
    "# Print all the h1 tags with id = intro\n",
    "for tag in page_content.find_all('h1', id = 'intro'):\n",
    "    print(tag)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `.find_all()` method also allows us to search for tag attributes directly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<h1 id=\"intro\">Get Help From Peers and Mentors</h1>\n"
     ]
    }
   ],
   "source": [
    "# Print all the tags with id = intro\n",
    "for tag in page_content.find_all(id = 'intro'):\n",
    "    print(tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<h2 class=\"h2style\" id=\"hub\">Student Hub</h2>\n"
     ]
    }
   ],
   "source": [
    "# Print all the tags with id = hub\n",
    "for tag in page_content.find_all(id = 'hub'):\n",
    "    print(tag)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Searching by `class`\n",
    "\n",
    "Let's suppose we wanted to find all the tags that had the attribute `class=\"h2style\"`. Unfortunately, in this case, we can't simply pass this attribute to the `.find_all()` method. The reason is that the **CSS** attribute, `class`, is a reserved word in Python. Therefore, using `class` as a keyword argument in the `.find_all()` method, will give us a syntax error. To get around this problem, BeautifulSoup has implemented the keyword `class_` (notice the underscore at the end) that can be used to search for the `class` attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<h2 class=\"h2style\" id=\"hub\">Student Hub</h2>\n",
      "<h2 class=\"h2style\" id=\"know\">Knowledge</h2>\n"
     ]
    }
   ],
   "source": [
    "# Print the tags that have the attribute class_ = 'h2style'\n",
    "for tag in page_content.find_all(class_ = 'h2style'):\n",
    "    print(tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<div class=\"section\">\n",
      " <h2 class=\"h2style\" id=\"hub\">\n",
      "  Student Hub\n",
      " </h2>\n",
      " <p>\n",
      "  Student Hub is our real time collaboration platform where you can work with peers and mentors. You will find Community rooms with other students and alumni.\n",
      " </p>\n",
      "</div>\n",
      "\n",
      "<div class=\"section\">\n",
      " <h2 class=\"h2style\" id=\"know\">\n",
      "  Knowledge\n",
      " </h2>\n",
      " <p>\n",
      "  Search or ask questions in\n",
      "  <a href=\"https://knowledge.udacity.com/\">\n",
      "   Knowledge\n",
      "  </a>\n",
      " </p>\n",
      "</div>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print the tags that have the attribute class_ = 'section'\n",
    "for tag in page_content.find_all(class_ = 'section'):\n",
    "    print(tag.prettify())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Searching With Regular Expressions\n",
    "\n",
    "We can also pass a regular expression object to the `.find_all()` method. Let's see an example. The `.name` attribute of the `Tag` object is used to print the name of tag itself."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "title\n",
      "link\n",
      "div\n",
      "div\n",
      "div\n"
     ]
    }
   ],
   "source": [
    "import re \n",
    "\n",
    "# Print only the tag names of all the tags whose name contain the letter i\n",
    "for tag in page_content.find_all(re.compile(r'i')):\n",
    "    print(tag.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "html\n",
      "head\n",
      "h1\n",
      "h2\n",
      "hr\n",
      "h2\n",
      "h3\n"
     ]
    }
   ],
   "source": [
    "# Print only the tag names of all the tags whose names start with the letter h\n",
    "for tag in page_content.find_all(re.compile(r'^h')):\n",
    "    print(tag.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tags may contain other tags and strings within them. These elements are known as the **tag’s children**. For example, the `<head>` tag is a direct child of the `<html>` tag. Similarly, the `<title>` tag is a direct child of the `<head>` tag. We also see that the `<title>` tag itself has a child, namely the string `'AI For Trading'`.\n",
    "\n",
    "We can view a tag's children by using the `.contents` attribute of the Tag object. The `.contents` attribute returns a list with all the tag's children. By counting the number of elements in this list we can see how many children a parent tag has."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<head>\n",
      "<title>AI For Trading</title>\n",
      "<meta charset=\"utf-8\"/>\n",
      "<link href=\"./teststyle.css\" rel=\"stylesheet\"/>\n",
      "<style>.h2style {background-color: tomato;color: white;padding: 10px;}</style>\n",
      "</head>\n",
      "\n",
      "['\\n', <title>AI For Trading</title>, '\\n', <meta charset=\"utf-8\"/>, '\\n', <link href=\"./teststyle.css\" rel=\"stylesheet\"/>, '\\n', <style>.h2style {background-color: tomato;color: white;padding: 10px;}</style>, '\\n']\n",
      "\n",
      "The <head> tag contains 9 children\n"
     ]
    }
   ],
   "source": [
    "# Printing the head tag\n",
    "print(page_head)\n",
    "\n",
    "# Print the children of the head tag\n",
    "print()\n",
    "print(page_head.contents)\n",
    "\n",
    "# Print the number of children of the head tag\n",
    "print('\\nThe <head> tag contains {} children'.format(len(page_head.contents)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We should note, that strings do not have children because they can’t contain anything. \n",
    "\n",
    "Instead of getting a tag's children as a list, we can also get an iterator that we loop over by using the `.children` attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "<title>AI For Trading</title>\n",
      "\n",
      "\n",
      "<meta charset=\"utf-8\"/>\n",
      "\n",
      "\n",
      "<link href=\"./teststyle.css\" rel=\"stylesheet\"/>\n",
      "\n",
      "\n",
      "<style>.h2style {background-color: tomato;color: white;padding: 10px;}</style>\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print the children of the head tag\n",
    "for child in page_head.children:\n",
    "    print(child)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<title>AI For Trading</title>\n",
      "\n",
      "['AI For Trading']\n",
      "\n",
      "The <title> contains 1 children\n"
     ]
    }
   ],
   "source": [
    "# Printing the title tag\n",
    "page_title = page_content.head.title\n",
    "print(page_title)\n",
    "\n",
    "# Print the children of the title tag\n",
    "print()\n",
    "print(page_title.contents)\n",
    "\n",
    "# Print the number of children of the title tag\n",
    "print('\\nThe <title> contains {} children'.format(len(page_title.contents)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AI For Trading\n"
     ]
    }
   ],
   "source": [
    "# Print the children of the title tag\n",
    "for child in page_title.children:\n",
    "    print(child)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The Recursive Argument\n",
    "\n",
    "If we use the `.find_all()` method on a Tag object, `tag.find_all()`,then the `find_all()` method will search all the tag's children, its children’s children, and so on. However, there will be times where we only want BeautifulSoup to search a tag's direct children. To do this, we can pass the `recursive=False` argument, to the `.find_all()` method. \n",
    "\n",
    "For example, the `<head>` tag is directly beneath the `<html>` tag and that the `<title>` tag is directly beneath the `<head>` tag. Even though the `<title>` tag is beneath the `<html>` tag, it’s **not** directly beneath it, because the `<head>` tag is in between them. \n",
    "\n",
    "Now, if we search the `<html>` tag for the `<title>` tag, using the `.find_all()` method, BeautifulSoup will find it because it is searching in all of the descendants of the `<html>` tag:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<title>AI For Trading</title>\n"
     ]
    }
   ],
   "source": [
    "# Search the html tag for the title tag\n",
    "for tag in page_content.html.find_all('title'):\n",
    "    print(tag)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that the we get a match.\n",
    "\n",
    "Now, let's restrict our search to only look at the `<html>` tag’s direct children, by using the `recursive=False` argument:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Search the html tag's direct children for the title tag\n",
    "for tag in page_content.html.find_all('title', recursive = False):\n",
    "    print(tag)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that now we don't get any matches because the `<title>` tag is **not** a direct descendent of the `<html>` tag."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<head>\n",
      " <title>\n",
      "  AI For Trading\n",
      " </title>\n",
      " <meta charset=\"utf-8\"/>\n",
      " <link href=\"./teststyle.css\" rel=\"stylesheet\"/>\n",
      " <style>\n",
      "  .h2style {background-color: tomato;color: white;padding: 10px;}\n",
      " </style>\n",
      "</head>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Search the html tag's direct children for the head tag    \n",
    "for tag in page_content.html.find_all('head', recursive = False):\n",
    "    print(tag.prettify())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exercise: Get Headers and Paragraphs\n",
    "\n",
    "Print only the text from all the `<h2>` and `<p>` tags inside the `<div>` tags that have a `class=\"section\"` attribute in the `sampe.html` file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Student Hub\n",
      "Student Hub is our real time collaboration platform where you can work with peers and mentors. You will find Community rooms with other students and alumni.\n",
      "\n",
      "Knowledge\n",
      "Search or ask questions in Knowledge\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Import BeautifulSoup\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Open the HTML file and create a BeautifulSoup Object\n",
    "with open('./sample.html') as f:\n",
    "    page_content = BeautifulSoup(f, 'lxml')\n",
    "\n",
    "# Print only the text from all the <h2> and <p> tags inside the <div> tags that have a class=\"section\" attribute\n",
    "for section in page_content.find_all('div', class_='section'):\n",
    "    header = section.h2.get_text()\n",
    "    print(header)\n",
    "    paragraph = section.p.get_text()\n",
    "    print(paragraph)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Requests Library\n",
    "The `requests` library allows us to send web HTTP requests and get a website's HTML data. It's an easy-to-use library with a lot of features ranging from passing parameters in URLs to sending custom headers and SSL verification. We can add headers, form data, multi-part files, and parameters with simple Python dictionaries. We can then access the response data using the same request. Once the `requests` library gets us the HTML data, we can use Beautifulsoup to extract the data we want. \n",
    "\n",
    "The `requests.get(website)` function is used to get the source code from our `website`. This `requests.get()` function returns a `Response` object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import BeautifulSoup\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Import requests\n",
    "import requests\n",
    "\n",
    "# Create a Response object\n",
    "r = requests.get('https://en.wikipedia.org/wiki/The_World%27s_Billionaires')\n",
    "\n",
    "# Get HTML data\n",
    "html_data = r.text\n",
    "\n",
    "# print(html_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `.text` method returns a string, therefore, `html_data` is a string containing the HTML data from our website."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.status_code # server's response"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `requests` library allows us to provide arguments to a web request by using the `params` keyword argument in the `.get()` function. The `params` keyword argument consists of a dictionary of strings, with the parameters that we want. For example, suppose we wanted to get `50` tweets from our Twitter page https://twitter.com/AIForTrading1. To do this we need to set `count=50` to our web request. This can be done by using the following code:\n",
    "\n",
    "```python\n",
    "requests.get('https://twitter.com/AIForTrading1', params = {'count':'50'})\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CSV Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "courses=[{'website':'Coursera','course':'Django'},{'website':'Udemy','course':'Docker'},{'website':'Udacity','course':'AWS'}]\n",
    "\n",
    "import csv\n",
    "with open('Courses.csv','w') as file:\n",
    "    csv_writer=csv.DictWriter(file,fieldnames=['website','course'])\n",
    "    csv_writer.writeheader()\n",
    "    csv_writer.writerows(courses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coursera | Django\n",
      "Udemy | Docker\n",
      "Udacity | AWS\n"
     ]
    }
   ],
   "source": [
    "with open('Courses.csv','r') as file:\n",
    "    csv_read=csv.DictReader(file)\n",
    "    for row in csv_read:\n",
    "        print('{} | {}'.format(row['website'],row['course']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Example**: https://realpython.com/python-csv/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Zipfile"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "An example code snippet to read and extract files from an `example.zip` file:\n",
    "\n",
    "```python\n",
    "from zipfile import ZipFile\n",
    "\n",
    "filepath = \"my/path/to/example.zip\"\n",
    "with ZipFile(filepath, mode='r') as f:\n",
    "    name_list = f.namelist()\n",
    "    extract_path = f.extract(name_list[0], path=\"my/output/path/\")\n",
    "```\n",
    "\n",
    "\n",
    "* `f.namelist()` returns the list of files inside the opened .zip file\n",
    "\n",
    "\n",
    "* `f.extract(filename, path=None)` extracts a specific file to a specified directory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other Python Standard Libraries\n",
    "* https://towardsdatascience.com/the-python-standard-library-modules-you-should-know-as-a-data-scientist-47e1117ca6c8\n",
    "\n",
    "### shutil Module\n",
    "The **shutil** module offers a number of high-level operations on files and collections of files. In particular, it provides functions that support file copying and removal. It comes under Python's standard utility modules. \n",
    "\n",
    "* **disk_usage()** method is used to get disk usage statistics of the given path. This method returns a named tuple with the attributes total, used, and free (space in bytes).\n",
    "\n",
    "### psutil Module\n",
    "**psutil (Python system and process utilities)** is a cross-platform library for retrieving information on the processes currently running and system utilization (CPU, memory, disks, network, sensors) in Python. It's useful mainly for system monitoring, profiling, limiting process resources, and managing running processes. \n",
    "\n",
    "* **cpu_percent()** returns a float showing the current system-wide CPU use as a percentage. When the interval is 0.0 or None (default), the function compares process times to system CPU times elapsed since the last call, returning immediately (non-blocking). That means that the first time it's called it will return a meaningful 0.0 value. When the interval is > 0.0, the function compares process times to system CPU times elapsed before and after the interval (blocking).\n",
    "\n",
    "### Socket Module\n",
    "Using the **gethostbyname()** in the **Socket** module  , we can check whether the local host is correctly configured. This function translates a host name to IPv4 address format. For example, on passing the parameter `localhost` to the function gethostbyname, the result should be 127.0.0.1.\n",
    "\n",
    "The `Request` library can be used to determine whether the computer can make successful calls to the internet. A request is when we ping a website for information, which returns the website's status code. This status code is an integer value. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Everything ok\n"
     ]
    }
   ],
   "source": [
    "import shutil\n",
    "import psutil\n",
    "import requests\n",
    "import socket\n",
    "from network import *\n",
    "\n",
    "def check_disk_usage(disk):\n",
    "    \"\"\"Verifies that there's enough free space on disk\"\"\"\n",
    "    du = shutil.disk_usage(disk)\n",
    "    free = du.free / du.total * 100\n",
    "    return free > 20\n",
    "\n",
    "def check_cpu_usage():\n",
    "    \"\"\"Verifies that there's enough unused CPU\"\"\"\n",
    "    usage = psutil.cpu_percent(1) # averaging over a period of 1 second\n",
    "    return usage < 75\n",
    "\n",
    "def check_localhost():\n",
    "    localhost=socket.gethostbyname('localhost')\n",
    "    if localhost=='127.0.0.1':\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "def check_connectivity():\n",
    "    request=requests.get(\"http://www.google.com\")\n",
    "    if request.status_code==200:\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "# If there's not enough disk, or not enough CPU, print an error\n",
    "if not check_disk_usage('/') or not check_cpu_usage():\n",
    "    print(\"ERROR!\")\n",
    "elif check_localhost() and check_connectivity():\n",
    "    print(\"Everything ok\")\n",
    "else:\n",
    "    print(\"Network checks failed\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Examples\n",
    "### 1) Check for prime number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2, 3, 5, 7, 11, 13, 17, 19]"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from math import sqrt\n",
    "def is_prime(x):\n",
    "    if x<2:\n",
    "        return False\n",
    "    for i in range(2,int(sqrt(x))+1):\n",
    "        if x%i==0:\n",
    "            return False\n",
    "    return True\n",
    "\n",
    "# List Comprehension\n",
    "[a for a in range(20) if is_prime(a)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2) Getting Prime Multiples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{2: (4, 8),\n",
       " 3: (9, 27),\n",
       " 5: (25, 125),\n",
       " 7: (49, 343),\n",
       " 11: (121, 1331),\n",
       " 13: (169, 2197),\n",
       " 17: (289, 4913),\n",
       " 19: (361, 6859)}"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Dictionary Comprehension\n",
    "{a:(a*a,a*a*a) for a in range(20) if is_prime(a)} # Prime Divisors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3) Sum of first 100 prime numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Object: <itertools.islice object at 0x0000019272404598>\n",
      "\n",
      "First 100 Prime numbers: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211, 223, 227, 229, 233, 239, 241, 251, 257, 263, 269, 271, 277, 281, 283, 293, 307, 311, 313, 317, 331, 337, 347, 349, 353, 359, 367, 373, 379, 383, 389, 397, 401, 409, 419, 421, 431, 433, 439, 443, 449, 457, 461, 463, 467, 479, 487, 491, 499, 503, 509, 521, 523, 541]\n",
      "\n",
      "Sum: 24133\n"
     ]
    }
   ],
   "source": [
    "from itertools import islice, count\n",
    "hundred_primes=islice((x for x in count() if is_prime(x)),100) # object\n",
    "print('Object:',hundred_primes)\n",
    "print('\\nFirst 100 Prime numbers:',list(hundred_primes))\n",
    "print('\\nSum:',sum(islice((x for x in count() if is_prime(x)),100)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4) Organizing Music Directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "MUSIC_PATH='D:\\COMFY Music' # Change the music directory path\n",
    "os.chdir(MUSIC_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To print directory tree and all the files within that directory tree we can use os.walk which is a generator and has a tuple of 3 values:\n",
    "* directories(dirpath)\n",
    "* directories within that path(dirname)\n",
    "* files within that path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<generator object walk at 0x00000192594E1ED0>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.walk(MUSIC_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files in Chicane are: \n",
      "Files in Chicane\\Chicane - Giants (2010) [MP3 VBR] are: \n",
      "\t01. Chicane - Barefoot.mp3\n",
      "\t02. Chicane - Middledistancerunner (feat. Adam Young).mp3\n",
      "\t03. Chicane - Come Back.mp3\n",
      "\t04. Chicane - What am I doing here (Part 1).mp3\n",
      "\t05. Chicane - Giants.mp3\n",
      "\t06. Chicane - Poppiholla (5AM).mp3\n",
      "\t07. Chicane - So far out to sea.mp3\n",
      "\t08. Chicane - Where do I start.mp3\n",
      "\t09. Chicane - From where I stand.mp3\n",
      "\t10. Chicane - Hiding all the Stars (feat. Tash Andrews).mp3\n",
      "\t11. Chicane - What am I doing here (Part 2) (feat. Lemar).mp3\n",
      "\t12. Chicane - Titles.mp3\n",
      "\t13. Chicane - Middledistancerunner (DC Rework Edit).mp3\n",
      "\tAlbumArtSmall.jpg\n",
      "\tAlbumArt_{1A79DE65-25C2-45F5-A06C-990A99229F73}_Large.jpg\n",
      "\tAlbumArt_{1A79DE65-25C2-45F5-A06C-990A99229F73}_Small.jpg\n",
      "\tAlbumArt_{B5020207-474E-4720-7757-433E2A8EC400}_Large.jpg\n",
      "\tAlbumArt_{B5020207-474E-4720-7757-433E2A8EC400}_Small.jpg\n",
      "\tAlbumArt_{B5020207-474E-4720-A6C2-A52136139F00}_Large.jpg\n",
      "\tAlbumArt_{B5020207-474E-4720-A6C2-A52136139F00}_Small.jpg\n",
      "\tdesktop.ini\n",
      "\tFolder.jpg\n",
      "Files in Chicane\\Chicane - Thousand Mile Stare (2011) (Inspiron) are: \n",
      "\t01 Windbreak.mp3\n",
      "\t02 Playing Fields.mp3\n",
      "\t03 Goldfish.mp3\n",
      "\t04 Going Deep.mp3\n",
      "\t05 Flotsum and Jetsum .mp3\n",
      "\t06 Fin De Jours .mp3\n",
      "\t07 Thousand Mile Stare.mp3\n",
      "\t08 Deep (Moogmonkey Rework).mp3\n",
      "\t09 Super Mouflon.mp3\n",
      "\tAlbumArtSmall.jpg\n",
      "\tCover.jpg\n",
      "\tFolder.jpg\n"
     ]
    }
   ],
   "source": [
    "for dirpath, dirnames, filenames in os.walk('Chicane'): # 'Chicane' is a sample directory\n",
    "    print('Files in %s are: ' % dirpath)\n",
    "    for file in filenames:\n",
    "        print('\\t' + file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['J A M W I N E  #43 - A New Beginning',\n",
       " 'J A M W I N E  #44 - Bit by Bit',\n",
       " \"J A M W I N E  #45 - And she said 'Tantrums'\",\n",
       " 'J A M W I N E  #46 - Inventing Stones',\n",
       " 'J A M W I N E  #47 - Seventy Seven Streets',\n",
       " 'J A M W I N E  #48 - ZO HO LO',\n",
       " 'J A M W I N E  #49 - Mercurial',\n",
       " 'J A M W I N E  #50 - Doting Love',\n",
       " 'J A M W I N E  #51 - Power of solitude',\n",
       " 'J A M W I N E  #52 - Raining Capsules',\n",
       " 'J A M W I N E  #53 - Hysterical Dementia',\n",
       " 'J A M W I N E  #54 - A boon of spoons',\n",
       " 'J A M W I N E  #55 - Glean Gingerly',\n",
       " 'J A M W I N E #56 - Recapitulating Whines',\n",
       " 'J A M W I N E #57 - Silver Beano',\n",
       " 'J A M W I N E #58 - Cerulean Welkin',\n",
       " 'J A M W I N E #59 - B_Side',\n",
       " 'J A M W I N E #60 - Yellow Road',\n",
       " 'J A M W I N E #61 - Bald Punter',\n",
       " 'J A M W I N E #62 - Tempus Fugit',\n",
       " 'J A M W I N E #63 - Impending Castling',\n",
       " 'J A M W I N E #64 - Jittery Sentry',\n",
       " 'J A M W I N E #65 - Prolific Forebears',\n",
       " 'J A M W I N E #66 - Precedence',\n",
       " 'J A M W I N E #67 - Amorphous Meadow',\n",
       " 'J A M W I N E #68 - Mandala',\n",
       " 'J A M W I N E #69 - Transgressions',\n",
       " 'J A M W I N E #70 - Propinquity',\n",
       " 'J A M W I N E #71 - Renaissance',\n",
       " 'J A M W I N E #72 - Defenestration',\n",
       " 'J A M W I N E #73 - Encumbrance',\n",
       " 'J A M W I N E #74 - Dainty Fragments',\n",
       " 'J A M W I N E #75 - Coup de foudre',\n",
       " 'J A M W I N E #76 - Menos es más',\n",
       " 'J A M W I N E #77 - Jovial Pauper',\n",
       " 'J A M W I N E #78 - Entourage',\n",
       " 'J A M W I N E #79 - Moxie',\n",
       " 'J A M W I N E #80 - Satiate',\n",
       " 'J A M W I N E #81 -  Altruistic',\n",
       " 'J A M W I N E #82 -  Sanity',\n",
       " 'J A M W I N E #83 -  Encore',\n",
       " 'J A M W I N E #84 -  Cerebrate',\n",
       " 'J A M W I N E #85 -  Frozen Sun',\n",
       " 'Talk Twenty #01 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #02 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #03 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #04 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #05 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #06 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #07 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #08 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #09 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #10 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #11 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #12 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #13 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #14 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #15 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #16 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #17 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #18 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #19 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #20 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #21 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #22 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #23 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #24 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #25 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #26 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #27 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #28 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #29 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #30 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #31 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #32 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #33 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #34 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #35 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #36 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #37 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #38 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #39 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #40 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #41 - The Comfy Music with JAMWINE',\n",
       " 'Talk Twenty #42 - The Comfy Music with JAMWINE']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_folders=[]\n",
    "for file in os.listdir():\n",
    "    if ('J A M W I N E' in file) or ('JAMWINE' in file): # 'JAMWINE' is a string to be searched in the directory names\n",
    "        selected_folders.append(file)\n",
    "selected_folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_song(filename):\n",
    "    if filename.endswith('.mp3'):\n",
    "        return True \n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "music_list=[]\n",
    "for album in selected_folders:\n",
    "    for file in os.listdir(album):\n",
    "        if is_song(file):\n",
    "            music_list.append([album,file])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Album</th>\n",
       "      <th>Song</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>J A M W I N E  #43 - A New Beginning</td>\n",
       "      <td>01. CVBZ - Be Somebody.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>J A M W I N E  #43 - A New Beginning</td>\n",
       "      <td>02. Autograf - Simple (James Carter &amp; TELYKast...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>J A M W I N E  #43 - A New Beginning</td>\n",
       "      <td>03. Kaskade - Nobody Like You.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>J A M W I N E  #43 - A New Beginning</td>\n",
       "      <td>04. Khwezi - Colours.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>J A M W I N E  #43 - A New Beginning</td>\n",
       "      <td>05. Sam Martin - Bring Me Home (Papa Ya Remix)...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Album  \\\n",
       "0  J A M W I N E  #43 - A New Beginning   \n",
       "1  J A M W I N E  #43 - A New Beginning   \n",
       "2  J A M W I N E  #43 - A New Beginning   \n",
       "3  J A M W I N E  #43 - A New Beginning   \n",
       "4  J A M W I N E  #43 - A New Beginning   \n",
       "\n",
       "                                                Song  \n",
       "0                         01. CVBZ - Be Somebody.mp3  \n",
       "1  02. Autograf - Simple (James Carter & TELYKast...  \n",
       "2                  03. Kaskade - Nobody Like You.mp3  \n",
       "3                           04. Khwezi - Colours.mp3  \n",
       "4  05. Sam Martin - Bring Me Home (Papa Ya Remix)...  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "songs_df=pd.DataFrame(music_list,columns=['Album','Song'])\n",
    "songs_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# songs_df.to_csv('Comfy Music.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Song</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Album</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Talk Twenty #37 - The Comfy Music with JAMWINE</th>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Song\n",
       "Album                                               \n",
       "Talk Twenty #37 - The Comfy Music with JAMWINE  19.0"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs_count=songs_df.groupby('Album').count()\n",
    "\n",
    "# Determining Albums having less than 20 songs\n",
    "songs_count[songs_count<20].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Song</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Album</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>J A M W I N E #59 - B_Side</th>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Talk Twenty #04 - The Comfy Music with JAMWINE</th>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Talk Twenty #17 - The Comfy Music with JAMWINE</th>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Song\n",
       "Album                                               \n",
       "J A M W I N E #59 - B_Side                      21.0\n",
       "Talk Twenty #04 - The Comfy Music with JAMWINE  21.0\n",
       "Talk Twenty #17 - The Comfy Music with JAMWINE  22.0"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Determining Albums having greater than 20 songs\n",
    "songs_count[songs_count>20].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(BASE_PATH) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5) Logs Analysis (Detecting ERROR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "logs='''\n",
    "July 31 00:06:21 mycomputername kernel[96041]: WARN Failed to start network connection\n",
    "July 31 00:09:53 mycomputername updater[46711]: WARN Computer needs to be turned off and on again\n",
    "July 31 00:12:36 mycomputername kernel[48462]: INFO Successfully connected\n",
    "July 31 00:13:52 mycomputername updater[43530]: ERROR Error running Python2.exe: Segmentation Fault (core dumped)\n",
    "July 31 00:16:13 mycomputername NetworkManager[63902]: WARN Failed to start application install\n",
    "July 31 00:26:45 mycomputername CRON[83063]: INFO I'm sorry Dave. I'm afraid I can't do that\n",
    "July 31 00:27:56 mycomputername cacheclient[75746]: WARN PC Load Letter\n",
    "July 31 00:33:31 mycomputername system[25588]: ERROR Out of yellow ink, specifically, even though you want grayscale\n",
    "July 31 00:36:55 mycomputername updater[73786]: WARN Packet loss\n",
    "July 31 00:37:38 mycomputername dhcpclient[87602]: INFO Googling the answer\n",
    "July 31 00:37:48 mycomputername utility[21449]: ERROR The cake is a lie!\n",
    "July 31 00:44:50 mycomputername kernel[63793]: ERROR Failed process [13966]\n",
    "July 31 00:45:23 mycomputername cacheclient[55644]: ERROR Unable to download more RAM\n",
    "July 31 00:51:23 mycomputername cacheclient[85917]: ERROR Failed process [13966]\n",
    "July 31 01:06:51 mycomputername cacheclient[47853]: WARN Computer needs to be turned off and on again\n",
    "July 31 01:07:17 mycomputername system[78132]: INFO Packets sent successfully\n",
    "July 31 01:19:34 mycomputername CRON[94028]: WARN Failed to start network connection\n",
    "July 31 01:24:05 mycomputername utility[14905]: INFO Healthy resource usage\n",
    "July 31 01:29:04 mycomputername kernel[57436]: INFO Generating Logs\n",
    "July 31 01:36:27 mycomputername process[81163]: ERROR Unable to download more RAM\n",
    "July 31 01:40:38 mycomputername kernel[35287]: ERROR ID: 10t\n",
    "July 31 01:42:02 mycomputername jam_tag=psim[27947]: INFO I'm sorry Dave. I'm afraid I can't do that\n",
    "July 31 01:42:28 mycomputername kernel[16004]: WARN Computer needs to be turned off and on again\n",
    "July 31 01:45:14 mycomputername cacheclient[28721]: INFO Googling the answer\n",
    "July 31 02:01:20 mycomputername NetworkManager[55118]: ERROR Unable to perform package upgrade\n",
    "July 31 02:12:54 mycomputername kernel[30641]: WARN Low on memory\n",
    "July 31 02:23:37 mycomputername utility[37367]: INFO Starting sync\n",
    "July 31 02:25:52 mycomputername system[41921]: WARN Failed to start CPU thread[39016]\n",
    "July 31 02:34:37 mycomputername kernel[32280]: INFO Loading...\n",
    "July 31 02:36:44 mycomputername NetworkManager[90289]: WARN Failed to start CPU thread[39016]\n",
    "July 31 02:39:01 mycomputername CRON[89330]: ERROR Unable to perform package upgrade\n",
    "July 31 02:45:39 mycomputername utility[57387]: INFO Access permitted\n",
    "July 31 02:58:44 mycomputername process[44707]: WARN Computer needs to be turned off and on again\n",
    "July 31 02:59:35 mycomputername system[55024]: WARN Packet loss\n",
    "July 31 03:09:30 mycomputername kernel[40705]: ERROR The cake is a lie!\n",
    "July 31 03:23:16 mycomputername cacheclient[57185]: INFO Checking process [16121]\n",
    "July 31 03:26:56 mycomputername cacheclient[90154]: INFO Healthy resource usage\n",
    "July 31 03:28:52 mycomputername CRON[55441]: INFO Loading...\n",
    "July 31 03:29:34 mycomputername dhcpclient[69232]: ERROR Unable to download more RAM\n",
    "July 31 03:34:41 mycomputername NetworkManager[14120]: ERROR 404 error not found\n",
    "July 31 03:36:26 mycomputername dhcpclient[79731]: ERROR The cake is a lie!\n",
    "July 31 03:38:24 mycomputername CRON[92141]: INFO Access permitted\n",
    "July 31 03:40:00 mycomputername dhcpclient[40114]: INFO Starting sync\n",
    "July 31 03:42:45 mycomputername utility[53726]: INFO I'm sorry Dave. I'm afraid I can't do that\n",
    "July 31 03:47:07 mycomputername NetworkManager[63805]: WARN Please reboot user\n",
    "July 31 04:09:16 mycomputername CRON[52593]: WARN PC Load Letter\n",
    "July 31 04:11:32 mycomputername CRON[51253]: ERROR: Failed to start CRON job due to script syntax error. Inform the CRON job owner!\n",
    "July 31 04:11:32 mycomputername jam_tag=psim[84082]: ERROR ID: 10t\n",
    "July 31 04:12:05 mycomputername utility[63418]: INFO Successfully connected\n",
    "July 31 04:14:22 mycomputername utility[53225]: ERROR I am error\n",
    "July 31 04:31:00 mycomputername NetworkManager[23060]: ERROR Out of yellow ink, specifically, even though you want grayscale\n",
    "July 31 04:36:49 mycomputername dhcpclient[89091]: ERROR Failed process [13966]\n",
    "July 31 04:37:34 mycomputername cacheclient[63496]: ERROR Sorting a list\n",
    "July 31 04:45:37 mycomputername CRON[78322]: INFO Checking process [16121]\n",
    "July 31 04:51:20 mycomputername kernel[66473]: WARN System overheating\n",
    "July 31 05:00:28 mycomputername NetworkManager[76363]: ERROR The cake is a lie!\n",
    "July 31 05:10:52 mycomputername jam_tag=psim[26712]: ERROR Process failed\n",
    "July 31 05:14:16 mycomputername utility[91499]: ERROR Sorting a list\n",
    "July 31 05:15:02 mycomputername system[76042]: WARN System overheating\n",
    "July 31 05:18:19 mycomputername kernel[59870]: INFO Starting sync\n",
    "July 31 05:27:34 mycomputername NetworkManager[67593]: INFO Writing Logs\n",
    "July 31 05:32:16 mycomputername process[21939]: ERROR Failed process [13966]\n",
    "July 31 05:34:23 mycomputername NetworkManager[96505]: INFO Plenty of disk space left\n",
    "July 31 05:36:31 mycomputername utility[15156]: INFO Packets sent successfully\n",
    "July 31 05:43:29 mycomputername dhcpclient[35484]: ERROR Encapsulating packets\n",
    "July 31 05:44:07 mycomputername cacheclient[97637]: ERROR Operation completed successfully\n",
    "July 31 05:58:25 mycomputername process[99181]: ERROR Process failed\n",
    "July 31 05:59:27 mycomputername utility[74219]: WARN Failed to start CPU thread[39016]\n",
    "July 31 06:20:09 mycomputername jam_tag=psim[34744]: INFO Loading...\n",
    "July 31 06:31:02 mycomputername jam_tag=psim[40033]: ERROR Syntax issue\n",
    "July 31 06:38:58 mycomputername CRON[31832]: INFO Memory allocated\n",
    "July 31 06:38:58 mycomputername updater[10115]: ERROR Unable to perform package upgrade\n",
    "July 31 06:43:49 mycomputername jam_tag=psim[59857]: INFO Successfully connected\n",
    "July 31 06:46:06 mycomputername CRON[12372]: ERROR Unable to perform package upgrade\n",
    "July 31 06:48:38 mycomputername NetworkManager[95699]: WARN Failed to start network connection\n",
    "July 31 07:01:16 mycomputername system[39619]: INFO Plenty of disk space left\n",
    "July 31 07:08:07 mycomputername jam_tag=psim[39984]: INFO Memory allocated\n",
    "July 31 07:08:10 mycomputername cacheclient[17912]: INFO Generating Logs\n",
    "July 31 07:10:12 mycomputername process[60665]: INFO Failed to start process[85253]\n",
    "July 31 07:11:12 mycomputername dhcpclient[98633]: WARN System overheating\n",
    "July 31 07:24:38 mycomputername jam_tag=psim[83107]: ERROR ID: 10t\n",
    "July 31 07:34:37 mycomputername process[22428]: INFO Failed to start process[85253]\n",
    "July 31 07:34:52 mycomputername updater[42267]: ERROR I am error\n",
    "July 31 07:35:27 mycomputername jam_tag=psim[96969]: INFO Loading...\n",
    "July 31 07:37:13 mycomputername updater[11461]: ERROR Unable to perform package upgrade\n",
    "July 31 07:46:27 mycomputername jam_tag=psim[21363]: INFO Packets sent successfully\n",
    "July 31 07:50:05 mycomputername system[68973]: INFO Compiling code\n",
    "July 31 08:02:17 mycomputername kernel[84055]: INFO Checking process [16121]\n",
    "July 31 08:14:45 mycomputername process[84233]: ERROR Out of ink\n",
    "July 31 08:21:11 mycomputername system[78476]: ERROR Error running Python2.exe: Segmentation Fault (core dumped)\n",
    "July 31 08:29:59 mycomputername utility[73121]: ERROR I am error\n",
    "July 31 08:33:24 mycomputername system[55880]: INFO Defragmenting hard drive\n",
    "July 31 08:39:13 mycomputername utility[78233]: ERROR The cake is a lie!\n",
    "July 31 08:49:55 mycomputername kernel[88866]: INFO Starting sync\n",
    "July 31 08:55:07 mycomputername system[24562]: ERROR Process failed\n",
    "July 31 08:56:17 mycomputername jam_tag=psim[67538]: WARN Packet loss\n",
    "July 31 09:01:06 mycomputername NetworkManager[89092]: INFO I'm sorry Dave. I'm afraid I can't do that\n",
    "July 31 09:26:46 mycomputername utility[53969]: WARN Low on memory\n",
    "July 31 09:29:37 mycomputername CRON[89099]: INFO Packets sent successfully\n",
    "July 31 09:37:31 mycomputername updater[46513]: ERROR Out of yellow ink, specifically, even though you want grayscale\n",
    "July 31 09:45:20 mycomputername utility[95200]: ERROR Out of yellow ink, specifically, even though you want grayscale\n",
    "July 31 09:47:30 mycomputername system[61238]: ERROR I am error\n",
    "July 31 09:51:01 mycomputername dhcpclient[83799]: WARN Packet loss\n",
    "July 31 09:54:04 mycomputername system[18585]: ERROR 404 error not found\n",
    "July 31 09:54:06 mycomputername updater[84188]: ERROR 404 error not found\n",
    "July 31 09:57:08 mycomputername dhcpclient[26102]: WARN Failed to start application install\n",
    "July 31 10:21:57 mycomputername NetworkManager[62856]: WARN Failed to start network connection\n",
    "July 31 10:25:03 mycomputername utility[89273]: ERROR lp0 on fire\n",
    "July 31 10:26:12 mycomputername utility[23086]: INFO Access permitted\n",
    "July 31 10:30:37 mycomputername system[48235]: WARN Please reboot user\n",
    "July 31 10:41:32 mycomputername NetworkManager[70251]: ERROR Sorting a list\n",
    "July 31 10:53:36 mycomputername system[78903]: ERROR Encapsulating packets\n",
    "July 31 10:56:12 mycomputername jam_tag=psim[97572]: WARN Failed to start network connection\n",
    "July 31 10:56:29 mycomputername updater[55938]: ERROR Unable to download more RAM\n",
    "July 31 10:58:29 mycomputername utility[78692]: INFO Writing Logs\n",
    "July 31 11:05:09 mycomputername cacheclient[34385]: WARN System overheating\n",
    "July 31 11:14:39 mycomputername process[22102]: WARN System overheating\n",
    "July 31 11:18:34 mycomputername process[32468]: ERROR 418: I'm a teapot\n",
    "July 31 11:19:56 mycomputername system[15816]: ERROR I am error\n",
    "July 31 11:20:11 mycomputername kernel[77339]: ERROR Out of ink\n",
    "July 31 11:25:08 mycomputername utility[65264]: INFO Checking process [16121]\n",
    "July 31 11:27:52 mycomputername CRON[65812]: ERROR ID: 10t\n",
    "July 31 11:43:32 mycomputername CRON[72993]: WARN Failed to start network connection\n",
    "July 31 11:50:34 mycomputername CRON[51748]: ERROR ID: 10t\n",
    "July 31 12:00:29 mycomputername NetworkManager[96959]: ERROR Operation completed successfully\n",
    "July 31 12:04:22 mycomputername updater[51112]: ERROR lp0 on fire\n",
    "July 31 12:07:01 mycomputername updater[47209]: ERROR 418: I'm a teapot\n",
    "July 31 12:15:19 mycomputername updater[76427]: WARN Low on memory\n",
    "July 31 12:21:53 mycomputername system[33216]: WARN PC Load Letter\n",
    "July 31 12:42:41 mycomputername system[38483]: ERROR AssertionError 'False' is not 'True'\n",
    "July 31 12:47:53 mycomputername dhcpclient[17238]: WARN PC Load Letter\n",
    "July 31 12:48:04 mycomputername system[68448]: ERROR lp0 on fire\n",
    "July 31 12:54:55 mycomputername updater[93309]: WARN Please send help I am stuck inside the internet\n",
    "July 31 13:00:34 mycomputername system[62593]: ERROR Out of yellow ink, specifically, even though you want grayscale\n",
    "July 31 13:04:58 mycomputername cacheclient[63249]: INFO Access permitted\n",
    "July 31 13:08:06 mycomputername cacheclient[32689]: INFO Loading...\n",
    "July 31 13:14:54 mycomputername system[84324]: ERROR 404 error not found\n",
    "July 31 13:23:12 mycomputername system[56046]: ERROR Unable to perform package upgrade\n",
    "July 31 13:36:17 mycomputername CRON[53567]: ERROR You seriously weren't expecting to find anything useful in here, were you?\n",
    "July 31 13:39:39 mycomputername process[94615]: ERROR AssertionError 'False' is not 'True'\n",
    "July 31 14:34:36 mycomputername dhcpclient[37575]: ERROR 404 error not found\n",
    "July 31 14:44:34 mycomputername kernel[34400]: INFO Successfully connected\n",
    "July 31 15:18:00 mycomputername process[56145]: WARN Please send help I am stuck inside the internet\n",
    "July 31 15:21:55 mycomputername CRON[38897]: INFO Failed to start process[85253]\n",
    "July 31 15:23:54 mycomputername utility[20760]: ERROR lp0 on fire\n",
    "July 31 15:45:53 mycomputername jam_tag=psim[18544]: INFO Hello world\n",
    "July 31 15:51:37 mycomputername kernel[79314]: WARN Failed to start CPU thread[39016]\n",
    "July 31 16:02:05 mycomputername cacheclient[80348]: INFO Starting sync\n",
    "July 31 16:07:01 mycomputername process[11843]: ERROR Process failed\n",
    "July 31 16:20:37 mycomputername dhcpclient[36371]: ERROR Unable to download more RAM\n",
    "July 31 16:24:34 mycomputername updater[83956]: INFO Access permitted\n",
    "July 31 16:30:59 mycomputername updater[61327]: INFO Starting sync\n",
    "July 31 16:36:57 mycomputername system[27219]: ERROR Process failed\n",
    "July 31 16:39:10 mycomputername jam_tag=psim[76787]: INFO Checking process [16121]\n",
    "July 31 16:43:19 mycomputername utility[31859]: ERROR Process failed\n",
    "July 31 16:45:49 mycomputername process[29993]: WARN Failed to start network connection\n",
    "July 31 17:13:27 mycomputername system[77845]: WARN Failed to start CPU thread[39016]\n",
    "July 31 17:25:11 mycomputername jam_tag=psim[70693]: ERROR Syntax issue\n",
    "July 31 17:55:16 mycomputername jam_tag=psim[58396]: ERROR 404 error not found\n",
    "July 31 17:55:32 mycomputername system[56762]: WARN Low on memory\n",
    "July 31 18:03:28 mycomputername process[79703]: ERROR lp0 on fire\n",
    "July 31 18:04:46 mycomputername updater[79077]: ERROR Operation completed successfully\n",
    "July 31 18:51:04 mycomputername cacheclient[83250]: ERROR Sorting a list\n",
    "July 31 18:51:46 mycomputername cacheclient[57597]: INFO Access permitted\n",
    "July 31 18:56:23 mycomputername kernel[87834]: WARN Failed to start network connection\n",
    "July 31 19:04:43 mycomputername kernel[36465]: WARN Packet loss\n",
    "July 31 19:24:44 mycomputername cacheclient[47526]: INFO Googling the answer\n",
    "July 31 19:44:47 mycomputername system[50340]: ERROR lp0 on fire\n",
    "July 31 19:48:46 mycomputername CRON[88206]: ERROR Failed process [13966]\n",
    "July 31 19:51:07 mycomputername NetworkManager[38455]: INFO Access permitted\n",
    "July 31 19:55:34 mycomputername jam_tag=psim[21289]: INFO Plenty of disk space left\n",
    "July 31 19:56:47 mycomputername CRON[61924]: ERROR Out of yellow ink, specifically, even though you want grayscale\n",
    "July 31 20:02:26 mycomputername cacheclient[50070]: ERROR Encapsulating packets\n",
    "July 31 20:05:36 mycomputername updater[23489]: INFO Healthy resource usage\n",
    "July 31 20:21:06 mycomputername utility[67718]: INFO Defragmenting hard drive\n",
    "July 31 20:38:05 mycomputername kernel[30846]: ERROR AssertionError 'False' is not 'True'\n",
    "July 31 20:42:44 mycomputername CRON[95019]: INFO Access permitted\n",
    "July 31 20:48:06 mycomputername kernel[51099]: WARN PC Load Letter\n",
    "July 31 20:57:02 mycomputername CRON[23996]: INFO Googling the answer\n",
    "July 31 21:05:47 mycomputername utility[67348]: ERROR The cake is a lie!\n",
    "July 31 21:06:46 mycomputername kernel[80105]: ERROR lp0 on fire\n",
    "July 31 21:26:11 mycomputername process[18312]: WARN Computer needs to be turned off and on again\n",
    "July 31 21:28:55 mycomputername jam_tag=psim[89637]: INFO Packets sent successfully\n",
    "July 31 21:48:30 mycomputername kernel[33330]: ERROR Syntax issue\n",
    "July 31 22:05:31 mycomputername process[37921]: ERROR Process failed\n",
    "July 31 22:07:54 mycomputername cacheclient[12017]: ERROR Process failed\n",
    "July 31 22:14:37 mycomputername utility[78832]: INFO Healthy resource usage\n",
    "July 31 22:22:34 mycomputername updater[78750]: ERROR ID: 10t\n",
    "July 31 22:41:00 mycomputername jam_tag=psim[40956]: ERROR The cake is a lie!\n",
    "July 31 22:42:51 mycomputername NetworkManager[28895]: INFO I'm sorry Dave. I'm afraid I can't do that\n",
    "July 31 23:15:22 mycomputername jam_tag=psim[75013]: ERROR Failed process [13966]\n",
    "July 31 23:17:05 mycomputername system[88316]: ERROR 418: I'm a teapot\n",
    "July 31 23:20:09 mycomputername system[42918]: ERROR Out of ink\n",
    "July 31 23:28:49 mycomputername utility[40452]: INFO Checking process [16121]\n",
    "July 31 23:29:25 mycomputername utility[22941]: WARN Failed to start CPU thread[39016]\n",
    "July 31 23:30:46 mycomputername process[99294]: WARN Computer needs to be turned off and on again\n",
    "July 31 23:35:49 mycomputername dhcpclient[10726]: INFO Googling the answer\n",
    "July 31 23:36:29 mycomputername updater[86053]: ERROR AssertionError 'False' is not 'True'\n",
    "July 31 23:40:05 mycomputername utility[88068]: INFO Generating Logs\n",
    "July 31 23:48:17 mycomputername CRON[28813]: WARN Please send help I am stuck inside the internet\n",
    "July 31 23:53:58 mycomputername CRON[59985]: INFO Successfully connected\n",
    "'''\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What is the error? CRON ERROR\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['July 31 02:39:01 mycomputername CRON[89330]: ERROR Unable to perform package upgrade',\n",
       " 'July 31 04:11:32 mycomputername CRON[51253]: ERROR: Failed to start CRON job due to script syntax error. Inform the CRON job owner!',\n",
       " 'July 31 06:46:06 mycomputername CRON[12372]: ERROR Unable to perform package upgrade',\n",
       " 'July 31 11:27:52 mycomputername CRON[65812]: ERROR ID: 10t',\n",
       " 'July 31 11:50:34 mycomputername CRON[51748]: ERROR ID: 10t',\n",
       " \"July 31 13:36:17 mycomputername CRON[53567]: ERROR You seriously weren't expecting to find anything useful in here, were you?\",\n",
       " 'July 31 19:48:46 mycomputername CRON[88206]: ERROR Failed process [13966]',\n",
       " 'July 31 19:56:47 mycomputername CRON[61924]: ERROR Out of yellow ink, specifically, even though you want grayscale']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "def error_search(logs):\n",
    "    error = input(\"What is the error? \")\n",
    "    returned_errors = []\n",
    "    for log in logs.split('\\n')[1:-1]:\n",
    "        error_patterns = [\"error\"]\n",
    "        for i in range(len(error.split(' '))):\n",
    "            error_patterns.append(r\"{}\".format(error.split(' ')[i].lower()))\n",
    "        if all(re.search(error_pattern, log.lower()) for error_pattern in error_patterns):\n",
    "            returned_errors.append(log)\n",
    "    return returned_errors\n",
    "       \n",
    "if __name__ == \"__main__\":\n",
    "    returned_errors = error_search(logs)\n",
    "    \n",
    "returned_errors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Readings\n",
    "* [Python Features](https://betterprogramming.pub/4-lesser-known-but-useful-python-features-you-should-try-e75a45905af3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}