Segmenting-and-clustering-neighborhood-Toronto2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from geopy.geocoders import Nominatim\n",
    "import folium\n",
    "import json\n",
    "from pandas.io.json import json_normalize # tranform JSON file into a pandas dataframe\n",
    "from sklearn.cluster import KMeans\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.colors as colors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M\n",
      "200\n",
      "{'Date': 'Fri, 29 May 2020 05:36:04 GMT', 'Vary': 'Accept-Encoding,Cookie,Authorization', 'Server': 'ATS/8.0.7', 'Content-Type': 'text/html; charset=UTF-8', 'X-Content-Type-Options': 'nosniff', 'P3P': 'CP=\"See https://en.wikipedia.org/wiki/Special:CentralAutoLogin/P3P for more info.\"', 'Content-language': 'en', 'Last-Modified': 'Sat, 23 May 2020 19:32:34 GMT', 'Content-Encoding': 'gzip', 'Age': '11946', 'X-Cache': 'cp5008 miss, cp5011 hit/46', 'X-Cache-Status': 'hit-front', 'Server-Timing': 'cache;desc=\"hit-front\"', 'Strict-Transport-Security': 'max-age=106384710; includeSubDomains; preload', 'Set-Cookie': 'WMF-Last-Access=29-May-2020;Path=/;HttpOnly;secure;Expires=Tue, 30 Jun 2020 00:00:00 GMT, WMF-Last-Access-Global=29-May-2020;Path=/;Domain=.wikipedia.org;HttpOnly;secure;Expires=Tue, 30 Jun 2020 00:00:00 GMT, GeoIP=IN:UP:Lucknow:26.84:80.92:v4; Path=/; secure; Domain=.wikipedia.org', 'X-Client-IP': '2409:4063:2103:cf7e:9c70:961b:6e8b:a779', 'Cache-Control': 'private, s-maxage=0, max-age=0, must-revalidate', 'Accept-Ranges': 'bytes', 'Content-Length': '12988', 'Connection': 'keep-alive'}\n"
     ]
    }
   ],
   "source": [
    "url = 'https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M'\n",
    "result = requests.get(url)\n",
    "print(url)\n",
    "print(result.status_code)\n",
    "print(result.headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(result.content, 'html.parser')\n",
    "table = soup.find('table')\n",
    "trs = table.find_all('tr')\n",
    "rows = []\n",
    "for tr in trs:\n",
    "    i = tr.find_all('td')\n",
    "    if i:\n",
    "        rows.append(i)\n",
    "        \n",
    "lst = []\n",
    "for row in rows:\n",
    "    postalcode = row[0].text.rstrip()\n",
    "    borough = row[1].text.rstrip()\n",
    "    neighborhood = row[2].text.rstrip()\n",
    "    if borough != 'Not assigned':\n",
    "        if neighborhood == 'Not assigned':\n",
    "            neighborhood = borough\n",
    "        lst.append([postalcode, borough, neighborhood])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['M3A', 'North York', 'Parkwoods'],\n",
       " ['M4A', 'North York', 'Victoria Village'],\n",
       " ['M5A', 'Downtown Toronto', 'Regent Park, Harbourfront'],\n",
       " ['M6A', 'North York', 'Lawrence Manor, Lawrence Heights'],\n",
       " ['M7A', 'Downtown Toronto', \"Queen's Park, Ontario Provincial Government\"],\n",
       " ['M9A', 'Etobicoke', 'Islington Avenue, Humber Valley Village'],\n",
       " ['M1B', 'Scarborough', 'Malvern, Rouge'],\n",
       " ['M3B', 'North York', 'Don Mills'],\n",
       " ['M4B', 'East York', 'Parkview Hill, Woodbine Gardens'],\n",
       " ['M5B', 'Downtown Toronto', 'Garden District, Ryerson'],\n",
       " ['M6B', 'North York', 'Glencairn'],\n",
       " ['M9B',\n",
       "  'Etobicoke',\n",
       "  'West Deane Park, Princess Gardens, Martin Grove, Islington, Cloverdale'],\n",
       " ['M1C', 'Scarborough', 'Rouge Hill, Port Union, Highland Creek'],\n",
       " ['M3C', 'North York', 'Don Mills'],\n",
       " ['M4C', 'East York', 'Woodbine Heights'],\n",
       " ['M5C', 'Downtown Toronto', 'St. James Town'],\n",
       " ['M6C', 'York', 'Humewood-Cedarvale'],\n",
       " ['M9C',\n",
       "  'Etobicoke',\n",
       "  'Eringate, Bloordale Gardens, Old Burnhamthorpe, Markland Wood'],\n",
       " ['M1E', 'Scarborough', 'Guildwood, Morningside, West Hill'],\n",
       " ['M4E', 'East Toronto', 'The Beaches'],\n",
       " ['M5E', 'Downtown Toronto', 'Berczy Park'],\n",
       " ['M6E', 'York', 'Caledonia-Fairbanks'],\n",
       " ['M1G', 'Scarborough', 'Woburn'],\n",
       " ['M4G', 'East York', 'Leaside'],\n",
       " ['M5G', 'Downtown Toronto', 'Central Bay Street'],\n",
       " ['M6G', 'Downtown Toronto', 'Christie'],\n",
       " ['M1H', 'Scarborough', 'Cedarbrae'],\n",
       " ['M2H', 'North York', 'Hillcrest Village'],\n",
       " ['M3H', 'North York', 'Bathurst Manor, Wilson Heights, Downsview North'],\n",
       " ['M4H', 'East York', 'Thorncliffe Park'],\n",
       " ['M5H', 'Downtown Toronto', 'Richmond, Adelaide, King'],\n",
       " ['M6H', 'West Toronto', 'Dufferin, Dovercourt Village'],\n",
       " ['M1J', 'Scarborough', 'Scarborough Village'],\n",
       " ['M2J', 'North York', 'Fairview, Henry Farm, Oriole'],\n",
       " ['M3J', 'North York', 'Northwood Park, York University'],\n",
       " ['M4J', 'East York', 'East Toronto, Broadview North (Old East York)'],\n",
       " ['M5J',\n",
       "  'Downtown Toronto',\n",
       "  'Harbourfront East, Union Station, Toronto Islands'],\n",
       " ['M6J', 'West Toronto', 'Little Portugal, Trinity'],\n",
       " ['M1K', 'Scarborough', 'Kennedy Park, Ionview, East Birchmount Park'],\n",
       " ['M2K', 'North York', 'Bayview Village'],\n",
       " ['M3K', 'North York', 'Downsview'],\n",
       " ['M4K', 'East Toronto', 'The Danforth West, Riverdale'],\n",
       " ['M5K', 'Downtown Toronto', 'Toronto Dominion Centre, Design Exchange'],\n",
       " ['M6K', 'West Toronto', 'Brockton, Parkdale Village, Exhibition Place'],\n",
       " ['M1L', 'Scarborough', 'Golden Mile, Clairlea, Oakridge'],\n",
       " ['M2L', 'North York', 'York Mills, Silver Hills'],\n",
       " ['M3L', 'North York', 'Downsview'],\n",
       " ['M4L', 'East Toronto', 'India Bazaar, The Beaches West'],\n",
       " ['M5L', 'Downtown Toronto', 'Commerce Court, Victoria Hotel'],\n",
       " ['M6L', 'North York', 'North Park, Maple Leaf Park, Upwood Park'],\n",
       " ['M9L', 'North York', 'Humber Summit'],\n",
       " ['M1M', 'Scarborough', 'Cliffside, Cliffcrest, Scarborough Village West'],\n",
       " ['M2M', 'North York', 'Willowdale, Newtonbrook'],\n",
       " ['M3M', 'North York', 'Downsview'],\n",
       " ['M4M', 'East Toronto', 'Studio District'],\n",
       " ['M5M', 'North York', 'Bedford Park, Lawrence Manor East'],\n",
       " ['M6M', 'York', 'Del Ray, Mount Dennis, Keelsdale and Silverthorn'],\n",
       " ['M9M', 'North York', 'Humberlea, Emery'],\n",
       " ['M1N', 'Scarborough', 'Birch Cliff, Cliffside West'],\n",
       " ['M2N', 'North York', 'Willowdale, Willowdale East'],\n",
       " ['M3N', 'North York', 'Downsview'],\n",
       " ['M4N', 'Central Toronto', 'Lawrence Park'],\n",
       " ['M5N', 'Central Toronto', 'Roselawn'],\n",
       " ['M6N', 'York', 'Runnymede, The Junction North'],\n",
       " ['M9N', 'York', 'Weston'],\n",
       " ['M1P',\n",
       "  'Scarborough',\n",
       "  'Dorset Park, Wexford Heights, Scarborough Town Centre'],\n",
       " ['M2P', 'North York', 'York Mills West'],\n",
       " ['M4P', 'Central Toronto', 'Davisville North'],\n",
       " ['M5P', 'Central Toronto', 'Forest Hill North & West, Forest Hill Road Park'],\n",
       " ['M6P', 'West Toronto', 'High Park, The Junction South'],\n",
       " ['M9P', 'Etobicoke', 'Westmount'],\n",
       " ['M1R', 'Scarborough', 'Wexford, Maryvale'],\n",
       " ['M2R', 'North York', 'Willowdale, Willowdale West'],\n",
       " ['M4R', 'Central Toronto', 'North Toronto West,  Lawrence Park'],\n",
       " ['M5R', 'Central Toronto', 'The Annex, North Midtown, Yorkville'],\n",
       " ['M6R', 'West Toronto', 'Parkdale, Roncesvalles'],\n",
       " ['M7R', 'Mississauga', 'Canada Post Gateway Processing Centre'],\n",
       " ['M9R',\n",
       "  'Etobicoke',\n",
       "  'Kingsview Village, St. Phillips, Martin Grove Gardens, Richview Gardens'],\n",
       " ['M1S', 'Scarborough', 'Agincourt'],\n",
       " ['M4S', 'Central Toronto', 'Davisville'],\n",
       " ['M5S', 'Downtown Toronto', 'University of Toronto, Harbord'],\n",
       " ['M6S', 'West Toronto', 'Runnymede, Swansea'],\n",
       " ['M1T', 'Scarborough', \"Clarks Corners, Tam O'Shanter, Sullivan\"],\n",
       " ['M4T', 'Central Toronto', 'Moore Park, Summerhill East'],\n",
       " ['M5T', 'Downtown Toronto', 'Kensington Market, Chinatown, Grange Park'],\n",
       " ['M1V',\n",
       "  'Scarborough',\n",
       "  \"Milliken, Agincourt North, Steeles East, L'Amoreaux East\"],\n",
       " ['M4V',\n",
       "  'Central Toronto',\n",
       "  'Summerhill West, Rathnelly, South Hill, Forest Hill SE, Deer Park'],\n",
       " ['M5V',\n",
       "  'Downtown Toronto',\n",
       "  'CN Tower, King and Spadina, Railway Lands, Harbourfront West, Bathurst Quay, South Niagara, Island airport'],\n",
       " ['M8V', 'Etobicoke', 'New Toronto, Mimico South, Humber Bay Shores'],\n",
       " ['M9V',\n",
       "  'Etobicoke',\n",
       "  'South Steeles, Silverstone, Humbergate, Jamestown, Mount Olive, Beaumond Heights, Thistletown, Albion Gardens'],\n",
       " ['M1W', 'Scarborough', \"Steeles West, L'Amoreaux West\"],\n",
       " ['M4W', 'Downtown Toronto', 'Rosedale'],\n",
       " ['M5W', 'Downtown Toronto', 'Stn A PO Boxes'],\n",
       " ['M8W', 'Etobicoke', 'Alderwood, Long Branch'],\n",
       " ['M9W', 'Etobicoke', 'Northwest, West Humber - Clairville'],\n",
       " ['M1X', 'Scarborough', 'Upper Rouge'],\n",
       " ['M4X', 'Downtown Toronto', 'St. James Town, Cabbagetown'],\n",
       " ['M5X', 'Downtown Toronto', 'First Canadian Place, Underground city'],\n",
       " ['M8X', 'Etobicoke', 'The Kingsway, Montgomery Road, Old Mill North'],\n",
       " ['M4Y', 'Downtown Toronto', 'Church and Wellesley'],\n",
       " ['M7Y',\n",
       "  'East Toronto',\n",
       "  'Business reply mail Processing Centre, South Central Letter Processing Plant Toronto'],\n",
       " ['M8Y',\n",
       "  'Etobicoke',\n",
       "  \"Old Mill South, King's Mill Park, Sunnylea, Humber Bay, Mimico NE, The Queensway East, Royal York South East, Kingsway Park South East\"],\n",
       " ['M8Z',\n",
       "  'Etobicoke',\n",
       "  'Mimico NW, The Queensway West, South of Bloor, Kingsway Park South West, Royal York South West']]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(103, 3)\n"
     ]
    }
   ],
   "source": [
    "cols = ['PostalCode', 'Borough', 'Neighborhood']\n",
    "df = pd.DataFrame(lst, columns=cols)\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PostalCode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighborhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [PostalCode, Borough, Neighborhood]\n",
       "Index: []"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.duplicated(['PostalCode'], keep=False)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(103, 3)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PostalCode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighborhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M3A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Parkwoods</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M4A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Victoria Village</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M5A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Regent Park, Harbourfront</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M6A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Lawrence Manor, Lawrence Heights</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M7A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Queen's Park, Ontario Provincial Government</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  PostalCode           Borough                                 Neighborhood\n",
       "0        M3A        North York                                    Parkwoods\n",
       "1        M4A        North York                             Victoria Village\n",
       "2        M5A  Downtown Toronto                    Regent Park, Harbourfront\n",
       "3        M6A        North York             Lawrence Manor, Lawrence Heights\n",
       "4        M7A  Downtown Toronto  Queen's Park, Ontario Provincial Government"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.groupby('PostalCode').agg(\n",
    "    {\n",
    "        'Borough':'first', \n",
    "        'Neighborhood': ', '.join,}\n",
    "    ).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PostalCode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighborhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M1B</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Malvern, Rouge</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M1C</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Rouge Hill, Port Union, Highland Creek</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M1E</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Guildwood, Morningside, West Hill</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M1G</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Woburn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M1H</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Cedarbrae</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  PostalCode      Borough                            Neighborhood\n",
       "0        M1B  Scarborough                          Malvern, Rouge\n",
       "1        M1C  Scarborough  Rouge Hill, Port Union, Highland Creek\n",
       "2        M1E  Scarborough       Guildwood, Morningside, West Hill\n",
       "3        M1G  Scarborough                                  Woburn\n",
       "4        M1H  Scarborough                               Cedarbrae"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PostalCode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighborhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>M5A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Regent Park, Harbourfront</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PostalCode           Borough               Neighborhood\n",
       "53        M5A  Downtown Toronto  Regent Park, Harbourfront"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[df['PostalCode'] == 'M5A']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfgeo = pd.read_csv(\"Geospatial_Coordinates.csv\")\n",
    "dfgeo.rename(columns={'Postal Code': 'PostalCode'}, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.merge(df, dfgeo, on=\"PostalCode\", how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PostalCode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>M5G</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Central Bay Street</td>\n",
       "      <td>43.657952</td>\n",
       "      <td>-79.387383</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PostalCode           Borough        Neighborhood   Latitude  Longitude\n",
       "57        M5G  Downtown Toronto  Central Bay Street  43.657952 -79.387383"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.loc[df2['PostalCode'] == 'M5G']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PostalCode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>M9V</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>South Steeles, Silverstone, Humbergate, Jamest...</td>\n",
       "      <td>43.739416</td>\n",
       "      <td>-79.588437</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    PostalCode    Borough                                       Neighborhood  \\\n",
       "101        M9V  Etobicoke  South Steeles, Silverstone, Humbergate, Jamest...   \n",
       "\n",
       "      Latitude  Longitude  \n",
       "101  43.739416 -79.588437  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.loc[df2['PostalCode'] == 'M9V']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLIENT_ID = '4EXOHY0UN2FIDIKP252TJ5XDDJVJCJRHE20KBGD5OHEE330J'# Foursquare Id\n",
    "CLIENT_SECRET = '5XBFB1YNTZPQ2JWZIU2BNTF3XCV3CW42T2E45CCKL55QF3N0' # Foursquare Secret\n",
    "VERSION = '20180605' # API version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.set_index('PostalCode', inplace = True) \n",
    "neighborhood_latitude = df2.loc['M9V']['Latitude']\n",
    "neighborhood_longitude = df2.loc['M9V']['Longitude']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://api.foursquare.com/v2/venues/explore?&client_id=4EXOHY0UN2FIDIKP252TJ5XDDJVJCJRHE20KBGD5OHEE330J&client_secret=5XBFB1YNTZPQ2JWZIU2BNTF3XCV3CW42T2E45CCKL55QF3N0&v=20180605&ll=43.739416399999996,-79.5884369&radius=500&limit=100'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LIMIT = 100 # limit of number of venues returned by Foursquare API\n",
    "radius = 500 # define radius\n",
    "# create URL\n",
    "url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(\n",
    "    CLIENT_ID, \n",
    "    CLIENT_SECRET, \n",
    "    VERSION, \n",
    "    neighborhood_latitude, \n",
    "    neighborhood_longitude, \n",
    "    radius, \n",
    "    LIMIT)\n",
    "url # display URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = requests.get(url).json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function that extracts the category of the venue\n",
    "def get_category_type(row):\n",
    "    try:\n",
    "        categories_list = row['categories']\n",
    "    except:\n",
    "        categories_list = row['venue.categories']\n",
    "        \n",
    "    if len(categories_list) == 0:\n",
    "        return None\n",
    "    else:\n",
    "        return categories_list[0]['name']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "venues = results['response']['groups'][0]['items']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\anaconda\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: pandas.io.json.json_normalize is deprecated, use pandas.json_normalize instead\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>referralId</th>\n",
       "      <th>reasons.count</th>\n",
       "      <th>reasons.items</th>\n",
       "      <th>venue.id</th>\n",
       "      <th>venue.name</th>\n",
       "      <th>venue.location.address</th>\n",
       "      <th>venue.location.crossStreet</th>\n",
       "      <th>venue.location.lat</th>\n",
       "      <th>venue.location.lng</th>\n",
       "      <th>venue.location.labeledLatLngs</th>\n",
       "      <th>venue.location.distance</th>\n",
       "      <th>venue.location.postalCode</th>\n",
       "      <th>venue.location.cc</th>\n",
       "      <th>venue.location.city</th>\n",
       "      <th>venue.location.state</th>\n",
       "      <th>venue.location.country</th>\n",
       "      <th>venue.location.formattedAddress</th>\n",
       "      <th>venue.categories</th>\n",
       "      <th>venue.photos.count</th>\n",
       "      <th>venue.photos.groups</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e-0-4c633939e1621b8d48842553-0</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'summary': 'This spot is popular', 'type': '...</td>\n",
       "      <td>4c633939e1621b8d48842553</td>\n",
       "      <td>Subway</td>\n",
       "      <td>6210 Finch Ave West, Store 103</td>\n",
       "      <td>at Albion Rd.</td>\n",
       "      <td>43.742645</td>\n",
       "      <td>-79.589643</td>\n",
       "      <td>[{'label': 'display', 'lat': 43.74264512142215...</td>\n",
       "      <td>372</td>\n",
       "      <td>M9V 0A1</td>\n",
       "      <td>CA</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>ON</td>\n",
       "      <td>Canada</td>\n",
       "      <td>[6210 Finch Ave West, Store 103 (at Albion Rd....</td>\n",
       "      <td>[{'id': '4bf58dd8d48988d1c5941735', 'name': 'S...</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e-0-4be58dc4cf200f479154133c-1</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'summary': 'This spot is popular', 'type': '...</td>\n",
       "      <td>4be58dc4cf200f479154133c</td>\n",
       "      <td>Shoppers Drug Mart</td>\n",
       "      <td>1530 Albion Rd</td>\n",
       "      <td>Albion Mall</td>\n",
       "      <td>43.741685</td>\n",
       "      <td>-79.584487</td>\n",
       "      <td>[{'label': 'display', 'lat': 43.741685, 'lng':...</td>\n",
       "      <td>405</td>\n",
       "      <td>M9V 1B4</td>\n",
       "      <td>CA</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>ON</td>\n",
       "      <td>Canada</td>\n",
       "      <td>[1530 Albion Rd (Albion Mall), Etobicoke ON M9...</td>\n",
       "      <td>[{'id': '4bf58dd8d48988d10f951735', 'name': 'P...</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e-0-4be70e26cf200f47e334153c-2</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'summary': 'This spot is popular', 'type': '...</td>\n",
       "      <td>4be70e26cf200f47e334153c</td>\n",
       "      <td>Popeyes Louisiana Kitchen</td>\n",
       "      <td>1530 Albion Rd.</td>\n",
       "      <td>at Kipling Ave. (Albion Centre)</td>\n",
       "      <td>43.741202</td>\n",
       "      <td>-79.584545</td>\n",
       "      <td>[{'label': 'display', 'lat': 43.74120165509377...</td>\n",
       "      <td>370</td>\n",
       "      <td>M9V 1B4</td>\n",
       "      <td>CA</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>ON</td>\n",
       "      <td>Canada</td>\n",
       "      <td>[1530 Albion Rd. (at Kipling Ave. (Albion Cent...</td>\n",
       "      <td>[{'id': '4d4ae6fc7a7b7dea34424761', 'name': 'F...</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e-0-5112b872e4b0c0a78d7bcd27-3</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'summary': 'This spot is popular', 'type': '...</td>\n",
       "      <td>5112b872e4b0c0a78d7bcd27</td>\n",
       "      <td>Sunny Foodmart</td>\n",
       "      <td>1620 Albion road</td>\n",
       "      <td>Albion Road and Finch Ave</td>\n",
       "      <td>43.741840</td>\n",
       "      <td>-79.590561</td>\n",
       "      <td>[{'label': 'display', 'lat': 43.74184023292111...</td>\n",
       "      <td>319</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CA</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>ON</td>\n",
       "      <td>Canada</td>\n",
       "      <td>[1620 Albion road (Albion Road and Finch Ave),...</td>\n",
       "      <td>[{'id': '4bf58dd8d48988d118951735', 'name': 'G...</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e-0-4cd4738cdfb4a1cd4337535c-4</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'summary': 'This spot is popular', 'type': '...</td>\n",
       "      <td>4cd4738cdfb4a1cd4337535c</td>\n",
       "      <td>The Beer Store</td>\n",
       "      <td>1530 Albion Rd</td>\n",
       "      <td>Near Finch Ave. W.</td>\n",
       "      <td>43.741694</td>\n",
       "      <td>-79.584373</td>\n",
       "      <td>[{'label': 'display', 'lat': 43.7416936, 'lng'...</td>\n",
       "      <td>413</td>\n",
       "      <td>M9V 1B4</td>\n",
       "      <td>CA</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>ON</td>\n",
       "      <td>Canada</td>\n",
       "      <td>[1530 Albion Rd (Near Finch Ave. W.), Etobicok...</td>\n",
       "      <td>[{'id': '5370f356bcbc57f1066c94c2', 'name': 'B...</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>e-0-4b04a05bf964a520c45522e3-5</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'summary': 'This spot is popular', 'type': '...</td>\n",
       "      <td>4b04a05bf964a520c45522e3</td>\n",
       "      <td>Sheriff's No Frills</td>\n",
       "      <td>1530 Albion Rd</td>\n",
       "      <td>at Finch Ave. W.</td>\n",
       "      <td>43.741696</td>\n",
       "      <td>-79.584379</td>\n",
       "      <td>[{'label': 'display', 'lat': 43.741696, 'lng':...</td>\n",
       "      <td>413</td>\n",
       "      <td>M9V 1B4</td>\n",
       "      <td>CA</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>ON</td>\n",
       "      <td>Canada</td>\n",
       "      <td>[1530 Albion Rd (at Finch Ave. W.), Etobicoke ...</td>\n",
       "      <td>[{'id': '4bf58dd8d48988d118951735', 'name': 'G...</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>e-0-4d8ba6910c4e41bdaaf7667f-6</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'summary': 'This spot is popular', 'type': '...</td>\n",
       "      <td>4d8ba6910c4e41bdaaf7667f</td>\n",
       "      <td>Pizza Pizza</td>\n",
       "      <td>1530 Albion Road, Unit T25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>43.741569</td>\n",
       "      <td>-79.584489</td>\n",
       "      <td>[{'label': 'display', 'lat': 43.74156896801906...</td>\n",
       "      <td>397</td>\n",
       "      <td>M9V 1B4</td>\n",
       "      <td>CA</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>ON</td>\n",
       "      <td>Canada</td>\n",
       "      <td>[1530 Albion Road, Unit T25, Etobicoke ON M9V ...</td>\n",
       "      <td>[{'id': '4bf58dd8d48988d1ca941735', 'name': 'P...</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>e-0-4c1951d6834e2d7f2d3a2a80-7</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'summary': 'This spot is popular', 'type': '...</td>\n",
       "      <td>4c1951d6834e2d7f2d3a2a80</td>\n",
       "      <td>McDonald's</td>\n",
       "      <td>1530 Albion Road, Unit F-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>43.741757</td>\n",
       "      <td>-79.584230</td>\n",
       "      <td>[{'label': 'display', 'lat': 43.7417571, 'lng'...</td>\n",
       "      <td>427</td>\n",
       "      <td>M9V 1B4</td>\n",
       "      <td>CA</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>ON</td>\n",
       "      <td>Canada</td>\n",
       "      <td>[1530 Albion Road, Unit F-1, Toronto ON M9V 1B...</td>\n",
       "      <td>[{'id': '4bf58dd8d48988d16e941735', 'name': 'F...</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>e-0-4f035c5c5c5c51dd31b52e55-8</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'summary': 'This spot is popular', 'type': '...</td>\n",
       "      <td>4f035c5c5c5c51dd31b52e55</td>\n",
       "      <td>Dollarama</td>\n",
       "      <td>1620 Albion Rd, Finch Albion Centre</td>\n",
       "      <td>NaN</td>\n",
       "      <td>43.742018</td>\n",
       "      <td>-79.591076</td>\n",
       "      <td>[{'label': 'display', 'lat': 43.742018, 'lng':...</td>\n",
       "      <td>359</td>\n",
       "      <td>M9V 4B4</td>\n",
       "      <td>CA</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>ON</td>\n",
       "      <td>Canada</td>\n",
       "      <td>[1620 Albion Rd, Finch Albion Centre, Etobicok...</td>\n",
       "      <td>[{'id': '52dea92d3cf9994f4e043dbb', 'name': 'D...</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>e-0-54e521a9498e559c968e8083-9</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'summary': 'This spot is popular', 'type': '...</td>\n",
       "      <td>54e521a9498e559c968e8083</td>\n",
       "      <td>NORI SUSHI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>43.742775</td>\n",
       "      <td>-79.586985</td>\n",
       "      <td>[{'label': 'display', 'lat': 43.742775, 'lng':...</td>\n",
       "      <td>391</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Canada</td>\n",
       "      <td>[Canada]</td>\n",
       "      <td>[{'id': '4bf58dd8d48988d111941735', 'name': 'J...</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>e-0-4ce971bff8653704e40db7c4-10</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'summary': 'This spot is popular', 'type': '...</td>\n",
       "      <td>4ce971bff8653704e40db7c4</td>\n",
       "      <td>LCBO</td>\n",
       "      <td>Albion Mall</td>\n",
       "      <td>1530 Albion Rd</td>\n",
       "      <td>43.741508</td>\n",
       "      <td>-79.584501</td>\n",
       "      <td>[{'label': 'display', 'lat': 43.74150752194671...</td>\n",
       "      <td>392</td>\n",
       "      <td>M9V 1B4</td>\n",
       "      <td>CA</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>ON</td>\n",
       "      <td>Canada</td>\n",
       "      <td>[Albion Mall (1530 Albion Rd), Toronto ON M9V ...</td>\n",
       "      <td>[{'id': '4bf58dd8d48988d186941735', 'name': 'L...</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         referralId  reasons.count  \\\n",
       "0    e-0-4c633939e1621b8d48842553-0              0   \n",
       "1    e-0-4be58dc4cf200f479154133c-1              0   \n",
       "2    e-0-4be70e26cf200f47e334153c-2              0   \n",
       "3    e-0-5112b872e4b0c0a78d7bcd27-3              0   \n",
       "4    e-0-4cd4738cdfb4a1cd4337535c-4              0   \n",
       "5    e-0-4b04a05bf964a520c45522e3-5              0   \n",
       "6    e-0-4d8ba6910c4e41bdaaf7667f-6              0   \n",
       "7    e-0-4c1951d6834e2d7f2d3a2a80-7              0   \n",
       "8    e-0-4f035c5c5c5c51dd31b52e55-8              0   \n",
       "9    e-0-54e521a9498e559c968e8083-9              0   \n",
       "10  e-0-4ce971bff8653704e40db7c4-10              0   \n",
       "\n",
       "                                        reasons.items  \\\n",
       "0   [{'summary': 'This spot is popular', 'type': '...   \n",
       "1   [{'summary': 'This spot is popular', 'type': '...   \n",
       "2   [{'summary': 'This spot is popular', 'type': '...   \n",
       "3   [{'summary': 'This spot is popular', 'type': '...   \n",
       "4   [{'summary': 'This spot is popular', 'type': '...   \n",
       "5   [{'summary': 'This spot is popular', 'type': '...   \n",
       "6   [{'summary': 'This spot is popular', 'type': '...   \n",
       "7   [{'summary': 'This spot is popular', 'type': '...   \n",
       "8   [{'summary': 'This spot is popular', 'type': '...   \n",
       "9   [{'summary': 'This spot is popular', 'type': '...   \n",
       "10  [{'summary': 'This spot is popular', 'type': '...   \n",
       "\n",
       "                    venue.id                 venue.name  \\\n",
       "0   4c633939e1621b8d48842553                     Subway   \n",
       "1   4be58dc4cf200f479154133c         Shoppers Drug Mart   \n",
       "2   4be70e26cf200f47e334153c  Popeyes Louisiana Kitchen   \n",
       "3   5112b872e4b0c0a78d7bcd27             Sunny Foodmart   \n",
       "4   4cd4738cdfb4a1cd4337535c             The Beer Store   \n",
       "5   4b04a05bf964a520c45522e3        Sheriff's No Frills   \n",
       "6   4d8ba6910c4e41bdaaf7667f                Pizza Pizza   \n",
       "7   4c1951d6834e2d7f2d3a2a80                 McDonald's   \n",
       "8   4f035c5c5c5c51dd31b52e55                  Dollarama   \n",
       "9   54e521a9498e559c968e8083                 NORI SUSHI   \n",
       "10  4ce971bff8653704e40db7c4                       LCBO   \n",
       "\n",
       "                 venue.location.address       venue.location.crossStreet  \\\n",
       "0        6210 Finch Ave West, Store 103                    at Albion Rd.   \n",
       "1                        1530 Albion Rd                      Albion Mall   \n",
       "2                       1530 Albion Rd.  at Kipling Ave. (Albion Centre)   \n",
       "3                      1620 Albion road        Albion Road and Finch Ave   \n",
       "4                        1530 Albion Rd               Near Finch Ave. W.   \n",
       "5                        1530 Albion Rd                 at Finch Ave. W.   \n",
       "6            1530 Albion Road, Unit T25                              NaN   \n",
       "7            1530 Albion Road, Unit F-1                              NaN   \n",
       "8   1620 Albion Rd, Finch Albion Centre                              NaN   \n",
       "9                                   NaN                              NaN   \n",
       "10                          Albion Mall                   1530 Albion Rd   \n",
       "\n",
       "    venue.location.lat  venue.location.lng  \\\n",
       "0            43.742645          -79.589643   \n",
       "1            43.741685          -79.584487   \n",
       "2            43.741202          -79.584545   \n",
       "3            43.741840          -79.590561   \n",
       "4            43.741694          -79.584373   \n",
       "5            43.741696          -79.584379   \n",
       "6            43.741569          -79.584489   \n",
       "7            43.741757          -79.584230   \n",
       "8            43.742018          -79.591076   \n",
       "9            43.742775          -79.586985   \n",
       "10           43.741508          -79.584501   \n",
       "\n",
       "                        venue.location.labeledLatLngs  \\\n",
       "0   [{'label': 'display', 'lat': 43.74264512142215...   \n",
       "1   [{'label': 'display', 'lat': 43.741685, 'lng':...   \n",
       "2   [{'label': 'display', 'lat': 43.74120165509377...   \n",
       "3   [{'label': 'display', 'lat': 43.74184023292111...   \n",
       "4   [{'label': 'display', 'lat': 43.7416936, 'lng'...   \n",
       "5   [{'label': 'display', 'lat': 43.741696, 'lng':...   \n",
       "6   [{'label': 'display', 'lat': 43.74156896801906...   \n",
       "7   [{'label': 'display', 'lat': 43.7417571, 'lng'...   \n",
       "8   [{'label': 'display', 'lat': 43.742018, 'lng':...   \n",
       "9   [{'label': 'display', 'lat': 43.742775, 'lng':...   \n",
       "10  [{'label': 'display', 'lat': 43.74150752194671...   \n",
       "\n",
       "    venue.location.distance venue.location.postalCode venue.location.cc  \\\n",
       "0                       372                   M9V 0A1                CA   \n",
       "1                       405                   M9V 1B4                CA   \n",
       "2                       370                   M9V 1B4                CA   \n",
       "3                       319                       NaN                CA   \n",
       "4                       413                   M9V 1B4                CA   \n",
       "5                       413                   M9V 1B4                CA   \n",
       "6                       397                   M9V 1B4                CA   \n",
       "7                       427                   M9V 1B4                CA   \n",
       "8                       359                   M9V 4B4                CA   \n",
       "9                       391                       NaN                CA   \n",
       "10                      392                   M9V 1B4                CA   \n",
       "\n",
       "   venue.location.city venue.location.state venue.location.country  \\\n",
       "0              Toronto                   ON                 Canada   \n",
       "1            Etobicoke                   ON                 Canada   \n",
       "2            Etobicoke                   ON                 Canada   \n",
       "3              Toronto                   ON                 Canada   \n",
       "4            Etobicoke                   ON                 Canada   \n",
       "5            Etobicoke                   ON                 Canada   \n",
       "6            Etobicoke                   ON                 Canada   \n",
       "7              Toronto                   ON                 Canada   \n",
       "8            Etobicoke                   ON                 Canada   \n",
       "9                  NaN                  NaN                 Canada   \n",
       "10             Toronto                   ON                 Canada   \n",
       "\n",
       "                      venue.location.formattedAddress  \\\n",
       "0   [6210 Finch Ave West, Store 103 (at Albion Rd....   \n",
       "1   [1530 Albion Rd (Albion Mall), Etobicoke ON M9...   \n",
       "2   [1530 Albion Rd. (at Kipling Ave. (Albion Cent...   \n",
       "3   [1620 Albion road (Albion Road and Finch Ave),...   \n",
       "4   [1530 Albion Rd (Near Finch Ave. W.), Etobicok...   \n",
       "5   [1530 Albion Rd (at Finch Ave. W.), Etobicoke ...   \n",
       "6   [1530 Albion Road, Unit T25, Etobicoke ON M9V ...   \n",
       "7   [1530 Albion Road, Unit F-1, Toronto ON M9V 1B...   \n",
       "8   [1620 Albion Rd, Finch Albion Centre, Etobicok...   \n",
       "9                                            [Canada]   \n",
       "10  [Albion Mall (1530 Albion Rd), Toronto ON M9V ...   \n",
       "\n",
       "                                     venue.categories  venue.photos.count  \\\n",
       "0   [{'id': '4bf58dd8d48988d1c5941735', 'name': 'S...                   0   \n",
       "1   [{'id': '4bf58dd8d48988d10f951735', 'name': 'P...                   0   \n",
       "2   [{'id': '4d4ae6fc7a7b7dea34424761', 'name': 'F...                   0   \n",
       "3   [{'id': '4bf58dd8d48988d118951735', 'name': 'G...                   0   \n",
       "4   [{'id': '5370f356bcbc57f1066c94c2', 'name': 'B...                   0   \n",
       "5   [{'id': '4bf58dd8d48988d118951735', 'name': 'G...                   0   \n",
       "6   [{'id': '4bf58dd8d48988d1ca941735', 'name': 'P...                   0   \n",
       "7   [{'id': '4bf58dd8d48988d16e941735', 'name': 'F...                   0   \n",
       "8   [{'id': '52dea92d3cf9994f4e043dbb', 'name': 'D...                   0   \n",
       "9   [{'id': '4bf58dd8d48988d111941735', 'name': 'J...                   0   \n",
       "10  [{'id': '4bf58dd8d48988d186941735', 'name': 'L...                   0   \n",
       "\n",
       "   venue.photos.groups  \n",
       "0                   []  \n",
       "1                   []  \n",
       "2                   []  \n",
       "3                   []  \n",
       "4                   []  \n",
       "5                   []  \n",
       "6                   []  \n",
       "7                   []  \n",
       "8                   []  \n",
       "9                   []  \n",
       "10                  []  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nearby_venues = json_normalize(venues) # flatten JSON\n",
    "nearby_venues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>categories</th>\n",
       "      <th>lat</th>\n",
       "      <th>lng</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Subway</td>\n",
       "      <td>Sandwich Place</td>\n",
       "      <td>43.742645</td>\n",
       "      <td>-79.589643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Shoppers Drug Mart</td>\n",
       "      <td>Pharmacy</td>\n",
       "      <td>43.741685</td>\n",
       "      <td>-79.584487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Popeyes Louisiana Kitchen</td>\n",
       "      <td>Fried Chicken Joint</td>\n",
       "      <td>43.741202</td>\n",
       "      <td>-79.584545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sunny Foodmart</td>\n",
       "      <td>Grocery Store</td>\n",
       "      <td>43.741840</td>\n",
       "      <td>-79.590561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The Beer Store</td>\n",
       "      <td>Beer Store</td>\n",
       "      <td>43.741694</td>\n",
       "      <td>-79.584373</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        name           categories        lat        lng\n",
       "0                     Subway       Sandwich Place  43.742645 -79.589643\n",
       "1         Shoppers Drug Mart             Pharmacy  43.741685 -79.584487\n",
       "2  Popeyes Louisiana Kitchen  Fried Chicken Joint  43.741202 -79.584545\n",
       "3             Sunny Foodmart        Grocery Store  43.741840 -79.590561\n",
       "4             The Beer Store           Beer Store  43.741694 -79.584373"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_columns = ['venue.name', 'venue.categories', 'venue.location.lat', 'venue.location.lng']\n",
    "nearby_venues = nearby_venues.loc[:, filtered_columns]\n",
    "\n",
    "# filter the category for each row\n",
    "nearby_venues['venue.categories'] = nearby_venues.apply(get_category_type, axis=1)\n",
    "\n",
    "# clean columns\n",
    "nearby_venues.columns = [col.split(\".\")[-1] for col in nearby_venues.columns]\n",
    "\n",
    "nearby_venues.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11 venues were returned by Foursquare.\n"
     ]
    }
   ],
   "source": [
    "print('{} venues were returned by Foursquare.'.format(nearby_venues.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getNearbyVenues(names, latitudes, longitudes, radius=500):\n",
    "    \n",
    "    venues_list=[]\n",
    "    for name, lat, lng in zip(names, latitudes, longitudes):\n",
    "        print(name)\n",
    "            \n",
    "        # create the API request URL\n",
    "        url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(\n",
    "            CLIENT_ID, \n",
    "            CLIENT_SECRET, \n",
    "            VERSION, \n",
    "            lat, \n",
    "            lng, \n",
    "            radius, \n",
    "            LIMIT)\n",
    "            \n",
    "        # make the GET request\n",
    "        results = requests.get(url).json()[\"response\"]['groups'][0]['items']\n",
    "        \n",
    "        # return only relevant information for each nearby venue\n",
    "        venues_list.append([(\n",
    "            name, \n",
    "            lat, \n",
    "            lng, \n",
    "            v['venue']['name'], \n",
    "            v['venue']['location']['lat'], \n",
    "            v['venue']['location']['lng'],  \n",
    "            v['venue']['categories'][0]['name']) for v in results])\n",
    "\n",
    "    nearby_venues = pd.DataFrame([item for venue_list in venues_list for item in venue_list])\n",
    "    nearby_venues.columns = ['Neighborhood', \n",
    "                  'Neighborhood Latitude', \n",
    "                  'Neighborhood Longitude', \n",
    "                  'Venue', \n",
    "                  'Venue Latitude', \n",
    "                  'Venue Longitude', \n",
    "                  'Venue Category']\n",
    "    \n",
    "    return(nearby_venues)                   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PostalCode</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>M1B</th>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Malvern, Rouge</td>\n",
       "      <td>43.806686</td>\n",
       "      <td>-79.194353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M1C</th>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Rouge Hill, Port Union, Highland Creek</td>\n",
       "      <td>43.784535</td>\n",
       "      <td>-79.160497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M1E</th>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Guildwood, Morningside, West Hill</td>\n",
       "      <td>43.763573</td>\n",
       "      <td>-79.188711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M1G</th>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Woburn</td>\n",
       "      <td>43.770992</td>\n",
       "      <td>-79.216917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M1H</th>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Cedarbrae</td>\n",
       "      <td>43.773136</td>\n",
       "      <td>-79.239476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M9N</th>\n",
       "      <td>York</td>\n",
       "      <td>Weston</td>\n",
       "      <td>43.706876</td>\n",
       "      <td>-79.518188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M9P</th>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>Westmount</td>\n",
       "      <td>43.696319</td>\n",
       "      <td>-79.532242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M9R</th>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>Kingsview Village, St. Phillips, Martin Grove ...</td>\n",
       "      <td>43.688905</td>\n",
       "      <td>-79.554724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M9V</th>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>South Steeles, Silverstone, Humbergate, Jamest...</td>\n",
       "      <td>43.739416</td>\n",
       "      <td>-79.588437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M9W</th>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>Northwest, West Humber - Clairville</td>\n",
       "      <td>43.706748</td>\n",
       "      <td>-79.594054</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>103 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                Borough                                       Neighborhood  \\\n",
       "PostalCode                                                                   \n",
       "M1B         Scarborough                                     Malvern, Rouge   \n",
       "M1C         Scarborough             Rouge Hill, Port Union, Highland Creek   \n",
       "M1E         Scarborough                  Guildwood, Morningside, West Hill   \n",
       "M1G         Scarborough                                             Woburn   \n",
       "M1H         Scarborough                                          Cedarbrae   \n",
       "...                 ...                                                ...   \n",
       "M9N                York                                             Weston   \n",
       "M9P           Etobicoke                                          Westmount   \n",
       "M9R           Etobicoke  Kingsview Village, St. Phillips, Martin Grove ...   \n",
       "M9V           Etobicoke  South Steeles, Silverstone, Humbergate, Jamest...   \n",
       "M9W           Etobicoke                Northwest, West Humber - Clairville   \n",
       "\n",
       "             Latitude  Longitude  \n",
       "PostalCode                        \n",
       "M1B         43.806686 -79.194353  \n",
       "M1C         43.784535 -79.160497  \n",
       "M1E         43.763573 -79.188711  \n",
       "M1G         43.770992 -79.216917  \n",
       "M1H         43.773136 -79.239476  \n",
       "...               ...        ...  \n",
       "M9N         43.706876 -79.518188  \n",
       "M9P         43.696319 -79.532242  \n",
       "M9R         43.688905 -79.554724  \n",
       "M9V         43.739416 -79.588437  \n",
       "M9W         43.706748 -79.594054  \n",
       "\n",
       "[103 rows x 4 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}