Explore_JobOffers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NAV - Job Offers (from Difi Datahotell API)\n",
    "\n",
    "* [NO] Ledige stillinger meldt til NAV (fra 2002 til 2017)\n",
    "* [EN] Available job offers reported by NAV (from 2002 to 2017)\n",
    "\n",
    "Dataset e.g.: https://data.norge.no/data/arbeids-og-velferdsetaten-nav/ledige-stillinger-meldt-til-nav-2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data2017 = requests.get('https://hotell.difi.no/api/json/nav/ledige-stillinger/2017?page=1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['entries', 'page', 'pages', 'posts'])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobOffers = data2017.json()\n",
    "jobOffers.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1304"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number fo pages\n",
    "jobOffers['pages']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "130377"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of job offers\n",
    "jobOffers['posts']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'antall_stillinger': '1',\n",
       " 'arbeidssted_fylke': 'Østfold',\n",
       " 'arbeidssted_fylkesnummer': '01',\n",
       " 'arbeidssted_kommune': 'Halden',\n",
       " 'arbeidssted_kommunenummer': '0101',\n",
       " 'arbeidssted_land': 'Norge',\n",
       " 'arbeidssted_landkode': 'NO',\n",
       " 'isco_versjon': 'ISCO-08',\n",
       " 'nav_enhet_kode': '0101',\n",
       " 'offisiell_statistikk_flagg': '1',\n",
       " 'registrert_dato': '12.12.2016',\n",
       " 'sistepubl_dato': '15.01.2017',\n",
       " 'statistikk_aar_mnd': '201701',\n",
       " 'stilling_kilde': 'Reg av arb.giver på nav.no',\n",
       " 'stillingsnummer': '0101201612000013',\n",
       " 'stillingstittel': 'Pedagogisk leder',\n",
       " 'virksomhet_navn': 'YNGLINGEN BARNEHAGE',\n",
       " 'virksomhet_organisasjonsnr': '974122545',\n",
       " 'yrke': 'Førskolelærere',\n",
       " 'yrke_grovgruppe': 'Undervisning',\n",
       " 'yrkesbetegnelse': 'Barnehagelærer',\n",
       " 'yrkeskode': '2342'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# First job offer\n",
    "jobOffers['entries'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data into MongoDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pymongo import MongoClient\n",
    "client = MongoClient()\n",
    "db = client['nav']\n",
    "# if 'job-offers' in db.collection_names(include_system_collections=False):\n",
    "#     db['job-offers'].delete_many({})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "year: 2002 (1498 pages)        \n",
      "year: 2003 (1300 pages)        \n",
      "year: 2004 (1328 pages)        \n",
      "year: 2005 (1577 pages)        \n",
      "year: 2006 (2148 pages)        \n",
      "year: 2007 (2555 pages)        \n",
      "year: 2008 (2305 pages)        \n",
      "year: 2009 (1678 pages)        \n",
      "year: 2010 (1582 pages)        \n",
      "year: 2011 (1678 pages)        \n",
      "year: 2012 (1601 pages)        \n",
      "year: 2013 (1433 pages)        \n",
      "year: 2013 (259/1433 pages)\r"
     ]
    }
   ],
   "source": [
    "years = [i for i in range(2002,2018)]\n",
    "url_base = \"https://hotell.difi.no/api/json/nav/ledige-stillinger/\"\n",
    "\n",
    "last_year = years[0]\n",
    "for year in years:\n",
    "    req_page1 = requests.get('https://hotell.difi.no/api/json/nav/ledige-stillinger/'+str(year)+'?page=1')\n",
    "    pages = req_page1.json()['pages']\n",
    "    print(\"year: \"+str(year)+\" (\"+str(pages)+\" pages)        \", end='\\n')\n",
    "    lastDownload = db['downloads'].find_one({'owner':'NAV', 'dataset':'job-offers', 'year': year})\n",
    "    startPage = 1\n",
    "    if lastDownload != None:\n",
    "        if lastDownload['last_downloaded_page'] == pages:\n",
    "            continue\n",
    "        startPage = lastDownload['last_downloaded_page']+1\n",
    "    for page in range(startPage,pages+1):\n",
    "        if year > last_year:\n",
    "            last_year = year\n",
    "        print(\"year: \"+str(year)+\" (\"+str(page)+\"/\"+str(pages)+\" pages)\", end='\\r')\n",
    "        url = 'https://hotell.difi.no/api/json/nav/ledige-stillinger/'+str(year)+'?page='+str(page)\n",
    "        req = requests.get(url)\n",
    "        jobOffers = req.json()['entries']\n",
    "        if any(jobOffers):\n",
    "            result = db['job-offers'].insert_many(jobOffers)\n",
    "        db['downloads'].replace_one({'owner':'NAV', 'dataset':'job-offers', 'year': year},\n",
    "                                    {'owner':'NAV', 'dataset':'job-offers', 'year': year,\n",
    "                                     'last_downloaded_page': page}, upsert=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## visualize distribution of job offers on the map\n",
    "\n",
    "* per commune or district\n",
    "* per group of profession (`yrke_grovgruppe`)\n",
    "* per designation of profession (`yrkesbetegnelse`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## visualize distribution of job offers over time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}