training_deep_neural_networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": "# Setup\n\nFirst, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures. We also check that Python 3.5 or later is installed (although Python 2.x may work, it is deprecated so we strongly recommend you use Python 3 instead), as well as Scikit-Learn >= 0.20 and TensorFlow >= 2.0.",
   "metadata": {
    "cell_id": "00004-7875112a-b4e8-4a02-9818-e3cf9eabf23b",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 165.171875
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00005-2446b0e4-56c5-4b42-aaef-6b3f801dbe78",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d25857d2",
    "execution_start": 1659354408513,
    "execution_millis": 5084,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 922
   },
   "source": "# Python >= 3.5 is required\nimport sys\nassert sys.version_info >= (3, 5)\n\n# Scikit-Learn >= 0.20 is required\nimport sklearn\nassert sklearn.__version__ >= \"0.20\"\n\ntry:\n    # %tensorflow_version only exists in Colab.\n    %tensorflow_version 2.x\nexcept Exception:\n    pass\n\n# TensorFlow ≥2.0 is required\nimport tensorflow as tf\nfrom tensorflow import keras\nassert tf.__version__ >= \"2.0\"\n\n%load_ext tensorboard\n\n# Common imports\nimport numpy as np\nimport os\n\n# to make this notebook's output stable across runs\nnp.random.seed(42)\n\n# To plot pretty figures\n%matplotlib inline\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nmpl.rc('axes', labelsize=14)\nmpl.rc('xtick', labelsize=12)\nmpl.rc('ytick', labelsize=12)\n\n# Where to save the figures\nPROJECT_ROOT_DIR = \".\"\nCHAPTER_ID = \"deep\"\nIMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\nos.makedirs(IMAGES_PATH, exist_ok=True)\n\ndef save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n    path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n    print(\"Saving figure\", fig_id)\n    if tight_layout:\n        plt.tight_layout()\n    plt.savefig(path, format=fig_extension, dpi=resolution)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "# Vanishing/Exploding Gradients Problem",
   "metadata": {
    "cell_id": "00006-49025a73-dac7-4b25-83c2-2c6dc3b69520",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 82
   }
  },
  {
   "cell_type": "code",
   "source": "def logit(z):\n    return 1 / (1 + np.exp(-z))",
   "metadata": {
    "cell_id": "00007-7aebea08-667d-421d-bf80-73a1d1df6a7d",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "7b88bd2a",
    "execution_start": 1659354413601,
    "execution_millis": 3,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00008-47799d1b-c6d9-44ca-adb3-6bf317f226d8",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "3471c462",
    "execution_start": 1659354413614,
    "execution_millis": 738,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 708.1875,
    "deepnote_output_heights": [
     null,
     282
    ]
   },
   "source": "z = np.linspace(-5, 5, 200)\n\nplt.plot([-5, 5], [0, 0], 'k-')\nplt.plot([-5, 5], [1, 1], 'k--')\nplt.plot([0, 0], [-0.2, 1.2], 'k-')\nplt.plot([-5, 5], [-3/4, 7/4], 'g--')\nplt.plot(z, logit(z), \"b-\", linewidth=2)\nprops = dict(facecolor='black', shrink=0.1)\nplt.annotate('Saturating', xytext=(3.5, 0.7), xy=(5, 1), arrowprops=props, fontsize=14, ha=\"center\")\nplt.annotate('Saturating', xytext=(-3.5, 0.3), xy=(-5, 0), arrowprops=props, fontsize=14, ha=\"center\")\nplt.annotate('Linear', xytext=(2, 0.2), xy=(0, 0.5), arrowprops=props, fontsize=14, ha=\"center\")\nplt.grid(True)\nplt.title(\"Sigmoid activation function\", fontsize=14)\nplt.axis([-5, 5, -0.2, 1.2])\n\nsave_fig(\"sigmoid_saturation_plot\")\nplt.show()",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Saving figure sigmoid_saturation_plot\n",
     "output_type": "stream"
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABOz0lEQVR4nO3dd3wUxfvA8c+kVyC00ItSQ5WilC8QmgiINFGREgSlWSkiiiiIDZSmWOAnCoJIRynSFEJXCJgAoUSRamgBQgjpufn9sUdMuRTgkrskz/v12ldyu3M7z20u99zszs4orTVCCCGEvXGwdQBCCCGEJZKghBBC2CVJUEIIIeySJCghhBB2SRKUEEIIuyQJSgghhF2SBCXui1IqUCk1x9ZxQM5iUUodVUpNyqOQUte7QCm1Pg/q8VdKaaVUyTyoa6hS6pxSymSLY5oulkFKqWhbxiCsT8l9UCIzSqlSwGSgC1AWiASOAh9rrbeayxQHErXWt2wV5x05iUUpdRRYqbWelEsx+APbgVJa64hU64ti/L9FWrGuM8AcrfWnqda5AMWByzoX/7mVUj7AFWA0sBK4pbXOkwShlNJAH631ylTr3AFvrfWVvIhB5A0nWwcg7NoqwAMYAvwNlAbaACXuFNBaX7dNaBnZUyzpaa1v5lE9CcClPKiqMsbnx3qt9cU8qC9LWutYINbWcQgr01rLIkuGBSgGaKBDNuUCMb7F33nsC6zF+LA4CzyH0eqalKqMBkYAPwMxQBjQFqgAbAZuA8FAo3R19QKOAPHAeWAC5rMAmcRS2lzHnVgGp4/Fwut50PycS+Y4DgGPpyvjAnxo3mc88A/wClDF/NpSLwvMz1mA8WEOMBS4DDim2+8SYG1O4jC/1jR1mdf7mx+XvIvjdgZ4G5gLRAEXgNezOEaDLLzOKsAk4KiFstGpHk8y/w2eAU4Bt4CfUsdrLheQKubLwMJUsaau94yleszrhmF8sUow/3wh3XZt/lusMB/jf4D+tv7fk+W/Ra5BicxEm5cnlFJud/G8hRjfrtsB3YH+5sfpvQ0sBRoAQebf5wNfAg8B4Rgf6gAopRpjfJCsBuoB44E3gZeyiGUBUA3oAPQABmJ8kGbFC9gIdDTHtgpYrZSqle41DsQ4vVUbo4UZifHh39tcpg7GadFXLdSxAihqruPO6/PCOF6LcxhHL4xE8p65nrKWXsxdHLdRGAmhETAVmKaUam5pn8Ay4DHz7w+b6z6fSVlLqgBPAz2BRzH+3h+kinkYRrL8DqiPcYr5qHlzU/PPF8z13nmchlKqJzAHmAXUBWYDXyqluqUr+g7GF4EG5tf1rVKq0l28FpGbbJ0hZbHfBePD9joQB+wDPgUeSVcmEHOrBaiJ8a20WartFYFkMragPkr1uK553ehU6/xJ1RIAfgC2pat7EnAhk1hqmJ/fMtX2yuljyeFx+B142/x7dfN+H8ukbJq4U61fgLkFZX68GliU6nF/4CbglpM4zI/PAGOzqj+Hx+0M8GO6Mn+lrstCLE3M9VRJt9+ctKDigKKp1k0A/k71+ALGdc7M6tbAk9nUswf41sLfYHcW70MnjBa9tKLsZJEWlMiU1noVUA7ohvFtvgXwu1LqrUyeUgswYbSI7uzjPEZrKL3DqX6/bP55xMK60uaftTE+dFLbDZRXShWxsP/a5lj2p4rlbCaxpFBKeSqlpimljimlbph7hjUB7nyrfsi83+1Z7ScHFgM9lFIe5sf9gFVa67gcxpFTOT1uh9OVCee/Y29tZ3Xaa3IpdSmlSgPlgd/us47MXrdfunUpr1trnQRcJfdet7hLkqBElrTWcVrrrVrr97TWLTBOw00y9xa7H4mpq8liXU7eo1n1VrvbnmyfAn2AiRgdQhpiJLn7fb3pbQCSgO7mD+UO/Hd6L6/iSH1sEi1su9vPBxOg0q1ztlDOGnXdq/TvB1vGIrIhfwhxt45hnAqxdF3qBMZ7qvGdFUqpChitsPt1HGiZbt3/ME5VWepWfieWh1PFUikHsfwP+F5rvUprfRjjdNODqbYHm/fbNpPnJ5h/OmZVidY6HuPaUD+M6zGXME5R5jSOO3VlWQ93f9zux1XAVymVOkk1vJsdaKOb+L9A+yyKJXLvr/vY3cQjbEsSlLBIKVVCKbVNKdVfKVVfKVVVKdUHGAf8prWOSv8crfVJjF54XyulmimlGmJc6I7h7lsy6U0H2iilJimlaiil+gFjgGmWCptj2QTMVUo1N8eygOy7IocBPZVSjZRS9TBaNSnJWGsdBiwHvlFK9TYfl1ZKqQHmImcxXmtXpVQpc+eHzCwGOgHDMa4BmXIah9kZoJVSqnwWN+be1XG7T4EY92C9pZR6UCk1BHjyHvbzAfCaUmqUOeaGSqkxqbafAdorpcqY78ey5BNggFLqRaVUdaXUyxhfBnLjdYtcIglKZCYa46L8q8AOIBSja/USjG/8mRmE8W0/EKO7+Q8YN3TG3U8wWutDGKe8emO+Wdi8ZDVyxCDgNLANWGeO/Uw2VY02x7sL47rb7+bfUxto3tdnGC21BRi98tBa/wu8i/Ehezmb+HZhtBb8SHt6L6dxvIPRCeUURuslg3s8bvdEa30c4/aBoRjXdjpivGfudj9fAS9i9NQ7ivFFo06qImMwWrDngT8z2cdPwMsYvROPYbyPR2qt191tPMJ2ZCQJkavM3+zDgb7mThdCCJEjMpKEsCqlVDvAG6NHXmmMlkQExrdgIYTIMaud4lNKvaSUClJKxSulFmRRLkApdVApFaWUumDuSiuJsuBwBt7HSFDrMK4/tdZa37ZpVEKIfMdqp/iUUr0wupl2Aty11oMyKTcC47zyH0ApjOsUK7TWH1slECGEEAWC1VouWuvVAEqpJhhjqmVW7qtUD/9VSv1A5l12hRBCFFL2cGqtNUYPMYuUUkMxegXh7u7euGLFinkVV46ZTCYcHKRDZHbkOOXM+fPn0VpTqZIMCZcTtnxfJekknPLRFQp7/R8MCwuL0FqXSr/epkdWKTUYY/iW5zMro7WeB8wDaNKkiQ4KCsqsqM0EBgbi7+9v6zDsnhynnPH39ycyMpLg4GBbh5Iv5OX7Kio+iufXPs/UDlOp6lM1T+q0Jnv9H1RKnbW03mapVCnVA/gI6KxTTewmhBD2KC4pjh5Le7DmxBrCroXZOpxCwSYtKKXUY8D/AV211keyKy+EELaUbEqm3+p+bD+zncU9F9OpWidbh1QoWC1BmbuKO2GMkeVonkMoyTxCcOpy7TBGF+iptd6fcU9CCGE/tNaM3DCS1cdXM6vTLPrV72frkAoNa57iextjnLPxGHPbxAJvK6UqKaWiU00CNhFjWJhfzOujlVIbrRiHEEJYTXRCNIcuHeKt/73Fq80szT8pcos1u5lPwpiMzBKvVOWkS7kQIl/QWuPt6s2OQTtwd3K3dTiFjv31NxRCCDuw5MgSui7pyu2E23g4e5B2FhGRFyRBCSFEOpv+3kTATwHEJMbg6JDd1FMit0iCEkKIVP648Ae9l/embum6/PzMz7g5WZqbU+QFSVBCCGF2/OpxuizpQlmvsmzqt4mibkVtHVKhJglKCCHMEpITqFS0ElsGbMHXy9fW4RR6+WcQKSGEyCVxSXG4ObnRoEwDDg09JB0i7IS0oIQQhVp0QjT+C/yZuG0igCQnOyIJSghRaCUkJ/Dk8ic5EH6AxuUa2zockY6c4hNCFEombWLQT4PYfGoz33T7hh61etg6JJGOtKCEEIXS6M2j+fHoj3zU/iOGNBpi63CEBdKCEkIUSi0qtsDNyY03Wr5h61BEJiRBCSEKlYu3LlLWuyxP1XmKp+o8ZetwRBbkFJ8QotBYdWwVD3z2ANtPb7d1KCIHJEEJIQqF7ae38+zqZ2lUthGPVHjE1uGIHJAEJYQo8A5dPET3pd2pXrw66/quw8PZw9YhiRyQBCWEKNAu3rrIY4sfw8fdh839N1PcvbitQxI5JJ0khBAFWhmvMrzyyCv08etD+SLlbR2OuAuSoIQQBVJkXCTXYq7xYPEHebv127YOR9wDOcUnhChwYhNj6fZjN9oubEtcUpytwxH3SFpQQogCJcmUxNMrn2bPuT0sfXKpTDiYj0mCEkIUGFprXlj3AuvC1vFlly/lRtx8Tk7xCSEKjC8PfMmC4AVMajOJEU1H2DoccZ+kBSWEKDAGNRyEo4MjwxoPs3Uowgqs2oJSSr2klApSSsUrpRZkU3aUUuqSUipKKfWtUsrVmrEIIQqPTX9vIio+Ck8XT4Y3GS6TDhYQ1j7FFw68D3ybVSGlVCdgPNAeqAw8AEy2cixCiEJg37V9PL7kcd7Z/o6tQxFWprTW1t+pUu8DFbTWgzLZvgQ4o7V+y/y4PfCD1rpMVvv19vbWjRunnfXyqaeeYuTIkcTExNClS5cMzxk0aBCDBg0iIiKCJ598MsP2ESNG8PTTT3P+/HkGDBiQYfuYMWPo1q0bJ0+eZNiwjKcN3n77bZycnChWrBivvfZahu0ffvghLVq0YO/evbz11lsZts+aNYuGDRvy66+/8v7772fYPnfuXGrWrMm6deuYPn16hu2LFi2iYsWKLFu2jK+++irD9pUrV1KyZEkWLFjAggULMmz/5Zdf8PDw4Msvv2T58uUZtgcGBgLw6aefsn79+jTb3N3d2bhxIwBTpkzht99+S7O9RIkSrFq1CoA333yTjRs3UqxYsZTtFSpUYPHixQC89tprBAcHp3l+jRo1mDdvHgBDhw4lLCwszfaGDRsya9YsAPr378+FCxfSbG/evDkfffQRAL179+batWtptrdv356JE41pvjt37kxsbGya7Y8//jhjx44FwN/fn/Ry670XHBxMUlISP/74Y7bvvQ4dOhAcHFxo33u7z+3Gf74/HtEe1A+uj1OycdUi/Xtv3759aZ5fWN97kZGRFCtWzCqfe9Z87+3YseOg1rpJ+nK2ugZVB/g51eMQwFcpVUJrneYvqZQaCgwFcHZ2JjIyMs2OwsLCCAwMJC4uLsM2gBMnThAYGMjNmzctbg8NDSUwMJArV65Y3H7kyBG8vb05d+6cxe0hISHUrFmTv//+2+L2Q4cOkZCQwNGjRy1uDwoKIjIykpCQEIvb//jjDy5evMiRI0csbt+3bx+nTp0iNDTU4vY9e/ZQtGhRTpw4YXH7zp07cXNzIywszOL2Ox8Sp06dyrA9NjY2Zfvp06czbDeZTCnbz507R3Jycpoyzs7OKdsvXLiQ4fnh4eEp28PDwzNsv3DhQsr2y5cvZ9h+7ty5lO1Xr14lKioqzfbTp0+nbL9+/Trx8fFptp86dSplu6Vjk1vvvaSkJLTWOXrvOTk5Fdr33rfrv+XVkFfxSPKg0q5KRCdEp2xP/95L//zC+t678z94v597wcEhJCa6Ehp6jsuXi5Cc7IHJ5IbW7phMrvzf/93i559PcPp0AmFh3dDaFZPJ2Ka1KyNHeuDicoUrV6py/vzHQPMMdYDtWlCngBe11pvMj52BBKCq1vpMZvtt0qSJDgoKsnq89yswMNDitxyRlhynnPH39ycyMjLDt3rxH601j3zzCP/e+pfpftN55rFnbB1SvhAYGEibNv7ExsL162mXa9eMnzduwK1bxhIV9d/vqddFR4N1U4eyqxZUNFAk1eM7v9+yQSxCiHxGKcXyPsu5nXCbq8eu2jocm4uLg8uX4dKl/36m/j0iwkhAly41Jzoa0jXY7om7O3h7/7d4eBjr0i+ZrU+9dOpkuQ5bJahQoAFw58RzA+By+tN7QgiRWlR8FP938P8Y1XwUVYpVASDwWKBNY8ptcXFw/ryxnDuXdjl/Hi5ehJs3c7o3o7O0iwuUKAHFi2dcfHygSJG0ySf9Y29vcLqP7JGQkMAPP/zAk08OwCmLHVk1QSmlnMz7dAQclVJuQJLWOild0e+BBUqpHzB6/r0NLLBmLEKIgiUuKY4eS3uw8+xO/Kv407hc4+yflA9obbRw/vor7fLPP0YSunIl+304OYGvL5QpYyx3fr/zs2RJIyGFhe2ja9fmuLuDrXri//PPP3Tr1o1jx47RoUMHKlasmGlZa7eg3gbeTfW4PzBZKfUtcAzw01qf01pvUkpNA7YD7sCqdM8TQogUyaZk+q/uz/Yz2/m+x/f5MjlpDWfPwtGjcOSI8TMszEhGWbWAnJygQgWoVCnjUrEilC1rtHoccnDT0I0b8XjYcK7GZcuWMWTIEGJjY/Hw8Mj2fjWrJiit9SRgUiabvdKVnQHMsGb9QoiCR2vNi7+8yKrjq5jx6AwGNMjYLdre3L4Nf/5pLHeS0dGjRicDS7y9oXp1Y6lRw/j54INQubLRAnJ0zNv4rS0uLo6RI0eybNkyYmJiUtY7ZJNVZagjIYRdOx5xnAXBCxjfcjyjmo+ydTgZJCYayWf/fjhwwPgZGgomU8aypUtDvXpQt66x1KplJKPSpW13yi23nThxgscff5zw8PA093tprfO2BSWEENbmV8qPP4f9Sa2StWwdCmCcjtuzB3buhF274NAhoyNDao6O0LAhNG5sJKQ7Sal0aZuEbDMLFy5k5MiRxMbGYumWJmlBCSHypaVHl5KQnMDABgOpXaq2zeK4dQu2b4fAQNixA4KDM7aOqlWDhx+Gpk2Nnw0bYtNrPbYWHR3NkCFDWL9+fZpTeqlJC0oIkS9t/nszA9YMoGXFlvSv3x8HlXczA5lMcPgwbNoEmzcbraXExP+2OznBI49A69bG0qyZ0T1bGA4fPszjjz/O1atXiUvftExHEpQQIl/Z/+9+ei/vTZ1Sdfj5mZ/zJDnFxcFvv8GaNbB+vXGD6x0ODkYS6tgR2rQxfvf0zPWQ8qUVK1YwYMCADEM3WaK1llN8Qoj840TECbr80AVfL1829d9EUbeiuVZXVBT88ouRlH75xRi+547y5Y3RDR57DNq3lxZSTvn4+FC8eHGioqK4fft2tuUlQQkh8o1Nf2/CycGJLf23UMYry8kN7kl8PGzcCIsXGy2l1F/0GzaEnj2hRw+jU0NB7VWXmzp06MC5c+f4/vvvefPNN4mOjpZrUEKIguG1Zq8xoP4ASniUsNo+tYa9e2HRIli+3BgMFYwE1KrVf0mpalWrVVmoOTk5MXjwYI4fP87nn3+eZVlpQQkh7NrthNs8vfJp3mnzDg+Xf9hqyenqVViwAObOhVOn/lvfoAH07w99+xqn8oT1RURE8MUXX6S5FuXi4oKzs3PKqb+ctKDyrmuMEEKkk5CcQO/lvdn490bCb4Xf9/60Nu5PevZZY3igceOM5FS+vPH74cNGN/GxYyU55ab3338fU7q++A4ODrz11lsUK1YMDw8PkpOTs21BSYISQtiESZt47ufn2HxqM/Men0ePWj3ueV+xsfD118bNsG3awI8/Gl3DH3/cuNZ09ixMnWpcWxK56/Lly8ybNy9D6ykgIIC33nqL8PBwpkyZQr169XBxcclyX3KKTwiR57TWjNo0iiVHlvBR+48Y0mjIPe0nIgIWLqzMU08Zp/TAGDz1+eeNpVIlKwYtcmTKlCkkJyenWefo6MikSZMAcHd3Z/To0YwePTrbfUmCEkLkuSRTEmdunmFUs1G80fKNu37+qVMwYwZ89x3Exhq9G5o0MU7d9eoFzs7WjljkxMWLF5k/fz4JCQkp61xcXBg8eDBlytx9r0xJUEKIPJVsSsbZ0ZlVT63CQTlke6E8tX/+gffeM3rk3bnE8cgj1/j44xK0aSNdw21t0qRJGa49OTo6MnHixHvan1yDEkLkmdXHV9P0/5pyOfoyTg5OOR4l4uxZeOEFqFkTFi40RncYNMgYRfzjj4/g7y/JydYuXLjA999/n6b15OrqytChQ/H19b2nfUqCEkLkie2nt9N3VV/cnNzwcvHK/gnAv//CyJHGlBTffGO0mgYNgpMnjdN7derkbswi5959990M154cHBx4++2373mfcopPCJHr/rz4J92Xdqda8Wqsf3Y9ni5ZD2Z3+zZ88glMm2b00FMK+vWDd94xJvQT9uXcuXMsWbKExFSj6rq6uvLiiy9SsmTJe96vJCghRK76+/rfPPbDY/i4+7C5/2aKu2c+sJ3JBEuWwPjxRusJjE4PU6aAn18eBSzu2sSJEy323HvzzTfva79yik8IkavcnNyoXbI2W/pvoUKRCpmW27vXGCl8wAAjOTVqZMy/tGqVJCd7dubMGZYvX56m9eTm5sYrr7xC8fscZVdaUEKIXBGdEI27kzsVilRge8D2THvrRUTAmDHw/ffG47Jl4cMPYeBAozOEsG8TJkwgKSkpzTpHR0fGjRt33/uWP78QwupiE2Pp8kMXAn4KACxPTKe10SOvVi0jObm6wttvQ1iY0RFCkpP9O3XqFKtXr06ToNzd3Rk1ahQ+Pj73vX9pQQkhrCrJlMQzq55h97ndLH1yqcUyf/0Fw4fDtm3G43btjKGKqlfPw0DFfXvzzTfTnNoDo/U0duxYq+xfvqMIIaxGa83QdUNZe3Itc7rM4ak6T6XZnpgIH3xgjIm3bRuUKGG0on79VZJTfhMWFsa6devSdI5wd3fn9ddfp2hR60w0adUEpZQqrpRao5S6rZQ6q5R6NpNyrkqpr5VSl5VS15VS65RSMrawEPncO9vf4bvg73i3zbuMbDoyzbaTJ6FlS+M0Xnw8BATAiRPGtSa5yTb/sdR6cnJyYtSoUVarw9qn+L4AEgBfoCGwQSkVorUOTVfuVaA5UB+4CcwDPgd6WTkeIUQeeqzaYyQkJ/Bum3dT1plM8OWXxnQXsbFQsSJ8+y106GDDQMV9uXXrFj/99FOaYY08PDwYP3483t7eVqvHai0opZQn0BuYqLWO1lrvBtYCAywUrwps1lpf1lrHAcsAuSdciHzq1HVjRsCWlVoytePUlE4R//4Ljz0GL79sJKeBA+HIEUlO+Z23tzc7duygadOmeHoaN107OTnxyiuvWLUea7agagBJWuuwVOtCgDYWys4HZiulygGRQD9go6WdKqWGAkMBfH19CQwMtGLI1hEdHW2XcdkbOU45ExkZSXJycr45Vvuu7WNi6ETerPUm7Uu3T1m/fXspZsyoQXS0M0WKJDJ69EnatIngzz+tW7+8r3LO2sdq2rRpBAcH8/XXX9OpUyeCgoKstm/AuKhpjQVoBVxKt+4FINBC2aLAUkADScCfQPHs6mjcuLG2R9u3b7d1CPmCHKecadOmjW7QoIGtw8iR3Wd3a/f33XXjuY11VFyU1lrr2FitR4zQ2uhIrnWXLlpfvJh7Mcj7Kufs9VgBQdrCZ741O0lEA0XSrSsC3LJQ9gvAFSgBeAKryaQFJYSwT0evHOXxHx+nYtGKbOy3EW9Xb06dghYt4KuvwMUF5swxZrS9h6mAhLBqggoDnJRSqTuLNgDSd5AAowPFAq31da11PEYHiYeVUvc+qqAQIs/cir9Fp8Wd8HD2YEv/LZTyLMWqVcbwRH/+CQ88YAxd9OKL0kNP3DurJSit9W2MltB7SilPpVRLoDuwyELxA8BApVRRpZQzMBII11pHWCseIUTu8Xb15v2277O5/2bKelTm1VfhySchKsoY3PXQIWjc2NZRivzO2jfqjgTcgSvAj8AIrXWoUqqVUio6VbmxQBzwF3AV6AL0tHIsQggruxV/i6Bw40L4cw89Ryldl3bt4LPPjGnWZ8+GlSvBSvdpikLOqvdBaa2vAz0srN8FeKV6fA2j554QIp+IT4qnx7IeBIUHcfrV05w5XpwePeD8eShfHlavhocftnWUIj1/f3/q1q3LnDlzbB3KXZOhjoQQ2Uo2JdN/TX+2nd7G550/Z+va4vzvf0Zyat4cgoIKVnK6evUqI0eOpEqVKri6uuLr60v79u3ZunVrjp4fGBiIUoqIiLy7arFgwQK8vDLOVLx69Wo++uijPIvDmmSwWCFElrTWvPjLi6w8tpJPOkwnbMVAPvjA2Pbcc0aPPVdX28Zobb179yYmJob58+dTrVo1rly5wo4dO7h27Vqex5KQkICLi8s9P/9+52SyJWlBCSGytDx0OXMPzmVUo7fZNW00H3xgTIUxaxbMn1/wklNkZCS7du3i448/pn379lSuXJmmTZsyduxYnnnmGQAWL15M06ZN8fb2pnTp0vTp04d/zVMAnzlzhrZt2wJQqlQplFIMGjQIME63vfTSS2nqGzRoEI8//njKY39/f0aMGMHYsWMpVaoULVu2BGDGjBnUr18fT09Pypcvz/PPP09kZCRgtNiee+45bt++jVIKpRSTJk2yWGeVKlV4//33GTZsGEWKFKFChQp88sknaWIKCwujTZs2uLm5UbNmTX755Re8vLxYsGCBVY5xTkmCEkJk6Um/J/m81Qp2Tn6PtWvBxwc2bYJXXy2YXci9vLzw8vJi7dq1xMXFWSyTkJDA5MmTCQkJYf369URERNC3b18AKlasyKpVqwAIDQ3l4sWLzJ49+65iWLx4MVprdu3axffmmRwdHByYNWsWoaGhLFmyhP379/Pyyy8D0KJFC2bNmoWHhwcXL17k4sWLWU55MXPmTOrVq8ehQ4d44403GDduHPv27QPAZDLRs2dPnJyc+P3331mwYAGTJ08mPj7+rl6DNcgpPiGERevD1tOwTEOiwyswfciTnDlj3N+0cSPUqGHr6HKPk5MTCxYs4IUXXmDevHk89NBDtGzZkj59+vDII48AMHjw4JTyDzzwAF999RW1a9fmwoULVKhQIeW0WunSpSlZ8u5v76xatSrTp09Ps+61115L+b1KlSpMmzaN7t27s3DhQlxcXChatChKKcrk4K7oRx99NKVV9fLLL/PZZ5/x22+/0bx5c7Zu3crJkyfZsmUL5csbk0zMnDkzpSWXl6QFJYTIYMupLfRa1ovnPp9PixZw5gw0bQr79hXs5HRH7969CQ8PZ926dXTu3Jm9e/fSrFkzPvzwQwAOHTpE9+7dqVy5Mt7e3jRp0gSAc+fOWaX+xhZuItu2bRsdO3akQoUKeHt706tXLxISErh06dJd779+/fppHpcrV44rV64AcOLECcqVK5eSnACaNm2Kgw2mOJYEJYRIY/+/++m1rBdlz73KrinvcOMGdOsG27dD6dK2ji7vuLm50bFjR9555x327t3LkCFDmDRpEjdv3qRTp054eHiwaNEiDhw4wKZNmwDj1F9WHBwc7oxHmiL9nEpAygjhd5w9e5auXbtSu3ZtVqxYwcGDB/n2229zVKclzs7OaR4rpdJMnWEvJEEJIVKciDhBlx+64HpgPOe++YT4eMXIkbBmDaT7zCx0/Pz8SEpKIjg4mIiICD788ENat25NrVq1Ulofd9zpdZd6tlkwOk1cvHgxzbqQkJBs6w4KCiIhIYGZM2fSvHlzatSoQXh4eIY609d3L2rVqkV4eHia/QcFBdkkgUmCEkKkGLvldWK3vMX1n94GYOpUY8BXR0cbB5aHrl27Rrt27Vi8eDGHDx/m9OnTrFixgmnTptG+fXv8/PxwdXVlzpw5/PPPP2zYsIGJEyem2UflypVRSrFhwwauXr1KdLQxkE67du3YuHEja9eu5eTJk4wePZrz589nG1P16tUxmUzMmjWL06dP8+OPPzJr1qw0ZapUqUJcXBxbt24lIiKCmJiYe3r9HTt2pGbNmgQEBBASEsLvv//O6NGjcXJySpnnK69IghJCAMbMt+V2rSTmt9E4OsLChcYsuAWxp15WvLy8aNasGbNnz6ZNmzbUqVOHt956i2effZZly5ZRqlQpFi5cyE8//YSfnx+TJ09mxowZafZRvnx5Jk+ezIQJE/D19U3pkDB48OCUpWXLlnh7e9OzZ/ajvNWvX5/Zs2czY8YM/Pz8+Oabb/j000/TlGnRogXDhw+nb9++lCpVimnTpt3T63dwcGDNmjXEx8fz8MMPExAQwIQJE1BK4ebmdk/7vGeW5uCw10Xmg8rf5DjlTF7PBxUdH63f2PS27vtsogatXVy0XrMmz6q/b/K+yrl7PVbBwcEa0EFBQdYNyIxM5oOSbuZCFGKJyYn0/KEvW6c+Dyed8PSEn3+G9u2zf64ouNasWYOnpyfVq1fnzJkzjB49mgYNGtCoUaM8jUMSlBCFlEmb6L90OFvfew3OtMPHx7jHyXyrjyjEbt26xRtvvMH58+fx8fHB39+fmTNn5vk1KElQQhRCWmteXP0my98cDOdbUrYsbNkCdevaOjJhDwYOHMjAgQNtHYYkKCEKo5P/XuKbUX3gfBMqVtRs36548EFbRyVEWpKghChkbtyAgb3KknS+LJUrG8mpalVbRyVERtLNXIhC5Pt966nzSDgHDkDVqrBjhyQnYb+kBSVEIfHzoT0M6lUBfakcDzxoInC7AxUr2joqITInCUqIQmB76GF6dS2CvlSPB6slsyPQkVRjgQphl+QUnxAF3KHT//BoJ43pUj0erJ7Irp2SnET+IC0oIQqwW7fgmZ5FSfr3ASpVSWBnoAtly9o6KiFyRlpQQhRQ0dGarl3hr5ASVKpsYtcOF8qVs3VUQuSctKCEKICuR8VSvflxrh9rRPnysO03BypVsnVUQtwdq7aglFLFlVJrlFK3lVJnlVLPZlG2kVJqp1IqWil1WSn1qjVjEaKwiolLonabUK4fa0TRErH89htyE67Il6zdgvoCSAB8gYbABqVUiNY6NHUhpVRJYBMwClgJuAAVrByLEIVOQoLGr20IV4Kb4FUslj073KlZ09ZRCXFvrNaCUkp5Ar2BiVrraK31bmAtMMBC8dHAZq31D1rreK31La31cWvFIkRhlJwMDTsd5uzvjXHzimXXdnfq1LF1VELcO2u2oGoASVrrsFTrQoA2Fso2A44opfYC1YA/gBe11ufSF1RKDQWGAvj6+hIYGGjFkK0jOjraLuOyN3KcciYyMpLk5OS7OlZawyef1OB4YAOc3GKYMe0YkZHRFIbDLe+rnMtvx8qaCcoLiEq37ibgbaFsBaAR0BE4AkwDfgRapi+otZ4HzANo0qSJ9vf3t17EVhIYGIg9xmVv5DjlTLFixYiMjLyrYzVmbBIbNzrh7q7ZvMWNVv9rknsB2hl5X+VcfjtW1kxQ0UCRdOuKALcslI0F1mitDwAopSYDEUqpolrrm1aMSYgC77nXj7Ngem2cnDSrVyta/a+QzdEuCixr9uILA5yUUtVTrWsAhFooexjQqR5rC2WEENkYP+1vFnxaG5SJud/G8thjto5ICOuxWoLSWt8GVgPvKaU8lVItge7AIgvFvwN6KqUaKqWcgYnAbmk9CZFzM789x9TxxlDkU2fcZvAADxtHJIR1WXskiZGAO3AF45rSCK11qFKqlVIq+k4hrfU24C1gg7lsNSDTe6aEEGktWXuZ0UN9QTsy6q0bjHvN0qVeIfI3q94HpbW+DvSwsH4XRieK1Ou+Ar6yZv1CFAYHDsCwfqUhWfHs8xFMf7+krUMSIlfIWHxC5CNBIbfp3FkTHa3o1w8WzS2Jkj4RooCSsfiEyCf++iee/7W7Tfx1T7p01Xz3ncJBvmKKAkze3kLkA5cuJ9O4VQTx10tT46HLrFiucHa2dVRC5C5JUELYuagoTf3/XeBWeHnKVrvCH9t88ZAOe6IQkAQlhB2Li4NGbc9x9e/KFCsXwaFdpSlWzNZRCZE3JEEJYaeSkqBvXzh1qDKexW8RtLMEZcrYOioh8o4kKCHskNbQZ+A1fvoJihWDvdu9efBB6a4nChfpxSeEHTp7cwSHfyyBs2si69c7U7++rSMSIu9JC0oIO/PX5Z7cPDsMHBL5YVk8LTOM8S9E4SAJSgg7MmXmRcJPvAqY+OL/ounT3Svb5whRUEmCEsJOLF+ZxDtjSgPgW/UDRg72sXFEQtiWJCgh7MC2bTCgnxNoR8pW/5oyRVbZOiQhbE46SQhhY7v2xfH4E04kJDjx0ktw+PBSblqYeOarr77i9u3b+Pn5Ubt2bSpXroyDjHUkCjBJUELY0JHQRNo/Gk/ibTe6PxnN7NletGtnuey2bdtYs2YNnp6eJCUlkZiYSIUKFahTpw5NmjShTp06+Pn5Ua1aNVxcXPL2hQiRCyRBCWEjZ86aaNYmisToEtRtcY4VSyplOfjr1KlTWb9+PVFRUSnrTp8+zenTp9m4cSOenp6YTCZiY2Px9fWlVq1aNGnShFGjRlFG7vAV+ZCcHxDCBi5f1jRqGUHMtRJUrneeP7ZWynbw1wceeIC+ffvibKFgcnIyUVFRREdHk5ycTHh4ONu2bWP69OlERkbmzosQIpdJghIij928Cf9rH82Nf0tT8oF/+XNHhRwP/vrBBx/g6OiYo7IeHh589NFH1KpV6z6iFcJ2JEEJkYdiY6FbN/g71JuylW9zeE9ZfHxyPoRR2bJlGT58OG5ublmWc3JyolGjRowZM+Z+QxbCZiRBCZFHEhOhdZfL7NoF5cvD3kBPypa5+3/BiRMnZtt7z9nZGR8fH27fvn2v4Qphc5KghMgDJhN06XOZoEBfnL1usmULVKlyb/sqXrw4r7/+Ou7u7pmWiY2NZcuWLdSsWZP9+/ffW0VC2JgkKCFymdbw7PNX+fVnXxxcb7PhF42f3/3tc+zYsdl2JY+Pj+fixYv4+/szZcoUkpOT769SIfKYJCghctkr466z7LtS4BTHkhW36diq2H3v08vLi3fffRdPT8806z0s9LaIjY3l448/pkWLFvz777/3XbcQecWqCUopVVwptUYpdVspdVYp9Ww25V2UUseVUhesGYcQ9mLmTJjzaXFwSGLOtxE83a201fY9cuTINKf5PDw8GD9+PB4eHiiVtuNFTEwMhw4donbt2qxevdpqMQiRm6zdgvoCSAB8gX7AV0qpOlmUfx24auUYhLALCxbA6NHG7x/NvsKLAypYdf+urq58/PHHeHp64uHhwdSpU5k4cSLBwcHUqlUrwzWqpKQkbt26xYABAwgICCAmJsaq8QhhbVZLUEopT6A3MFFrHa213g2sBQZkUr4q0B/4yFoxCGEvvv8hgcFDTADMmgXjXyqXK/UEBARQokQJWrRowYsvvghA9erVCQ4OZtiwYRY7UsTExLB8+XJq1apFcHBwrsQlhDUorbV1dqTUQ8AerbVHqnVjgTZa624Wyq8H5gM3gMVaa4tfL5VSQ4GhAL6+vo2XLl1qlXitKTo6Gi8vmbcnO4XlOO3YWZxJk/3A5ES7pwOZOPzunv/aa6+RnJzM559/nqPyV69excvLy2IyOnjwIJMnTyY2NpakpKQM211dXXnuuefo06dPvh14trC8r6zBXo9V27ZtD2qtm2TYoLW2ygK0Ai6lW/cCEGihbE9go/l3f+BCTupo3Lixtkfbt2+3dQj5QmE4TuvXm7SDU6IGrR8N2H9P+2jTpo1u0KCB1WK6evWq7tChg/b09NRAhsXDw0O3atVKX7p0yWp15qXC8L6yFns9VkCQtvCZb82vTNFAkXTrigC3Uq8wnwqcBrxixbqFsLlff4XuPZMwJTnRrM8eNn3X1NYhAVCyZEm2bNnC1KlTM+1AsW/fPmrWrMkvv/xioyiFyMiaCSoMcFJKVU+1rgEQmq5cdaAKsEspdQlYDZRVSl1SSlWxYjxC5JmdO+GJJzTJic74dQlkz9IWqJyPYJTrlFK8+OKLHDhwgAceeMBiB4qbN2/y5JNPMnz4cOLi4mwUqRD/sVqC0lrfxkg27ymlPJVSLYHuwKJ0RY8CFYGG5uV54LL59/PWikeIvLJvH3TtCrGxin4BcQT/3AoHBzvKTqn4+flx9OhRBg4caPGaVWxsLN9//z316tXj2LFjNohQiP9Y+6roSMAduAL8CIzQWocqpVoppaIBtNZJWutLdxbgOmAyP5Zb3UW+EhQEHR5NJDoanu1nYuF8N5ydcjbauK24ubnx9ddfs2LFCooWLYqTU9pp4WJjYzl16hRNmzbliy++uHPdWIg8Z9UEpbW+rrXuobX21FpX0lovMa/fpbW22HVEax2oM+nBJ4Q9Cw6Gdh0SiYl2pljjrcyZe5sczoRhF7p27cqJEydo1qxZhhEptNbExMQwbtw4Hn30USIiImwUpSjM8me/UmGRv78/L730kq3DKBQOHoQ2bZO4ddMZz7q/cvTXBvh4ets6rLtWpkwZduzYwaRJkzK9Z2rHjh3UrFmTbdu22SBCUZgV+gR19epVRo4cSZUqVXB1dcXX15f27duzdevWHD0/MDCQtm3b5uk3zAULFli8l2H16tV89JHc95zb9u+Htu1MREU64VpnE0FbH6R8MesNYZTXHBwcGDt2LHv27KFixYoZ5ppKTEzk+vXrPP7444waNYqEhAQbRSoKm0KfoHr37s3+/fuZP38+YWFhrF+/ns6dO3Pt2rU8j+V+//GLFy+Ot3f++xafn/z+O3TsCLeiHHCvv5G9GytQq0xVW4dlFQ899BDHjx+nT58+mQ46O2/ePBo2bMhff/1lgwhFoWPp5ih7Xax9o+6NGzc0oLdu3ZppmUWLFukmTZpoLy8vXapUKf3kk0/qCxcuaK21Pn36dIabHgMCArTWxs2WL774Ypp9BQQE6K5du6Y8btOmjR4+fLgeM2aMLlmypG7SpInWWuvp06frevXqaQ8PD12uXDk9ZMgQfePGDa21caNd+jrfffddi3VWrlxZT5kyRQ8dOlR7e3vr8uXL62nTpqWJ6eTJk7p169ba1dVV16hRQ2/YsEF7enrq77777l4OaZbs9SbBnNq9W2tvb5MGrfv00To6Nj5X6rH2jbr3YuXKldrb21s7OjpmeL8ppbSHh4f+9ttvtclksmmcWuf/91VestdjRR7cqJvveHl54eXlxdq1azO97yMhIYHJkycTEhLC+vXriYiIoG/fvgBUrFiRVatWARAaGsrFixeZPXv2XcWwePFitNbs2rWL77//HjBOucyaNYvQ0FCWLFnC/v37efnllwFo0aIFs2bNwsPDg4sXL3Lx4kXGjh2b6f5nzpxJvXr1OHToEG+88Qbjxo1j3759AJhMJnr27ImTkxO///47CxYsYPLkycTHx9/VaygMdu2CTp00t24pHup4kiVLwNMt6/mY8rPevXtz7NgxGjVqlKE1pc0dKF566SW6d+9OZGSkbYIUBZ+lrGWvS24MdbRy5Urt4+OjXV1ddbNmzfSYMWP077//nmn548ePa0CfP39ea/1fi+bq1atpyuW0BVWvXr1sY9y4caN2cXHRycnJWmutv/vuO+3p6ZmhnKUW1DPPPJOmTLVq1fSUKVO01lpv2rRJOzo6prQItdZ6z549GpAWVCqbNmnt7m60nKi3SH8b9H2u1mcPLag7kpKS9Hvvvafd3d0tDpPk6uqqS5UqpXft2mWzGPPr+8oW7PVYIS0oy3r37k14eDjr1q2jc+fO7N27l2bNmvHhhx8CcOjQIbp3707lypXx9vamSRNjPMNz585Zpf7GjRtnWLdt2zY6duxIhQoV8Pb2plevXiQkJHDp0qW73n/9+vXTPC5XrhxXrlwB4MSJE5QrV47y5cunbG/atGm+HTQ0N6xcCd26aWJjFTT8jqlfXOa5xhYH6C+QHB0dmThxIoGBgZQtWxZXV9c02+Pj47l69SqPPvoob731lsUBaYW4V/JJhHHjYseOHXnnnXfYu3cvQ4YMYdKkSdy8eZNOnTrh4eHBokWLOHDgAJs2bQKy79Dg4OCQ4QbHxMTEDOXS339y9uxZunbtSu3atVmxYgUHDx7k22+/zVGdljg7O6d5rJTCZDLd9X4Ko/nz4emnITFRQbOZjP34BONajbF1WDbx8MMPc/LkSZ544olMO1DMnj2bJk2acObMmbwPUBRIkqAs8PPzIykpieDgYCIiIvjwww9p3bo1tWrVSml93OHiYlyHSE5OOwhGqVKluHjxYpp1ISEh2dYdFBREQkICM2fOpHnz5tSoUYPw8PAMdaav717UqlWL8PDwNPsPCgqSBAZMnw7PPw8mE3R+4Xeee+Mo0x792NZh2ZS3tzfLly9n7ty5eHp6Zmhpx8TEcPToUV5//XUbRSgKmkKdoK5du0a7du1YvHgxhw8f5vTp06xYsYJp06bRvn17/Pz8cHV1Zc6cOfzzzz9s2LCBiRMnptlH5cqVUUqxYcMGrl69SnR0NADt2rVj48aNrF27lpMnTzJ69GjOn89+qMHq1atjMpmYNWsWp0+f5scff2TWrFlpylSpUoW4uDi2bt1KRETEPc+M2rFjR2rWrElAQAAhISH8/vvvjB49GicnpwwjXhcWWsPbb8OdfiezZ8Mv85oxv/s3hfaYpNe/f3+OHDlC3bp1M7Sm3NzcmDZtmo0iEwVNoU5QXl5eNGvWjNmzZ9OmTRvq1KnDW2+9xbPPPsuyZcsoVaoUCxcu5KeffsLPz4/JkyczY8aMNPsoX748gwYNYsKECfj6+qaM5DB48OCUpWXLlnh7e9OzZ89sY6pfvz6zZ89mxowZ+Pn58c033/Dpp5+mKdOiRQuGDx9O3759KVWq1D1/IDg4OLBmzRri4+N5+OGHCQgIYMKECSilMtysWRgkJcGIEfDBB+DgaMLzqZE07Wn0eJTklFbVqlU5ePAgr7zySsoIFB4eHsydO5eqVQvGfWHCDljqOWGvi0xYmPuCg4M1oIOCgqy+b3s+Trduad21q9agtYtrsnbt97Su92U9fT3mep7HYk+9+HJi586dumTJkrpPnz42qd+e31f2xl6PFZn04nPKLoGJgm3NmjV4enpSvXp1zpw5w+jRo2nQoAGNGjWydWh55vJlY7qMgwehqE8y9H0CnxrH2NR/Dz7uPrYOz+61atWK8+fPZxgVXYj7Je+oQu7WrVu88cYbnD9/Hh8fH/z9/Zk5c2ahOaV14gR07gxnzkClKkkk9u1Ikk8oW/rvoZx3OVuHl28UxlPCIvdJgirkBg4cyMCBA20dhk3s3g1PPAE3bkDTprDmZ/jozzoMfmg61UtUz34HQohcJQlKFEqLFsELL0B8PHTpmsTn869R3teXOWXn2Do0IYRZoe7FJwqf5GR4/XUYONBITsNHJMMzvXhseSvikiyPxyjyTpUqVTL0WhWFl7SgRKERGQl9+8KmTeDkBLNmm/ij7HP8cngdX3f9GjcnuY6SFwYNGkRERATr16/PsO3AgQMZRlcRhVehaEGNHz+el19+mVOnTtk6FGEjJ0/CI48YyalECdi6VfPPg6+z6PAiprSdwrAmw2wdosAYgcXSUEp5TSZltA8FPkFduXKF2bNnM3fuXOrWrUvr1q359ddfbR2WyEMbNxrJKSwM6tWDAwcgzPv/mPH7DF55+BUmtJpg6xCFWfpTfEop5s2bR58+ffD09OSBBx5g8eLFaZ5z9epVnnnmGXx8fPDx8aFr165pJlQ8deoU3bt3p0yZMnh6etKoUaMMrbcqVaowadIkBg8eTLFixejXr1/uvlCRIwU+QX399deAMVBrXFwcu3bt4qmnnrJxVCIvJCUZwxZ16QI3b0KvXrB3L1StCr1r9+Y9//eY+Vjh6VKfX7333nt0796dkJAQnn76aQYPHpwym0BMTAyjR4/Gzc2NHTt2sG/fPsqWLUuHDh1ShgCLjo6mc+fObN26lZCQEHr37k2vXr04ceJEmnpmzJhBrVq1CAoKSpnNQNhWgU5QSUlJfPbZZ2kmI3RxcWHw4ME2jErkhYsXoUMH87BFDjBlCqxYAaGRf5CQnEAJjxJMbDMRB1Wg/wUKhAEDBtC/f3+qVavGlClTcHJyYufOnQAsXboUrTXfffcd9evXp1atWsydO5fo6OiUVlKDBg0YPnw49erVo1q1akyYMIFGjRqxcuXKNPW0adOGcePGUa1aNapXl9sM7EGB/u9ct25dhtlhHRwceOWVV2wUkcgLv/0GDRvCjh3g6wu//mq0pHaf30mbBW1489c3bR2iuAup5zRzcnKiVKlSKbMKHDx4kIsXL+Lt7Z0yQ3bRokW5ceNGyjXn27dvM27cOPz8/PDx8cHLy4ugoKAMc7rdmetN2A+r9uJTShUH5gOPAhHAm1rrJRbKvQ4EAJXN5b7UWn9izVgAPvroo5TRxe9o2bIllSpVsnZVwg4kJ8P778Pkycao5G3bwpIlUKYMhFwKoduP3ajqU5U3W0mCyk+ymtPMZDJRrVo1NmzYkOF5xYsXB2Ds2LFs2rSJTz/9lOrVq+Ph4cHAgQMzdISQ3oP2x9rdzL8AEgBfoCGwQSkVorUOTVdOAQOBw8CDwBal1Hmt9VJrBXL8+HGOHj2aZp2Xlxfjx4+3VhXCjpw+DYMGwc6doBS8846xODrCPzf+odPiThRxLcLm/psp6VHS1uEKK2nUqBGLFi2iZMmSFCtWzGKZ3bt3M3DgQHr37g1AXFwcp06dokaNGnkYqbgXVjvFp5TyBHoDE7XW0Vrr3cBaIMP82FrraVrrQ1rrJK31SeBnoKW1YgHjgmf6b0hFixalffv21qxG2JjW8O23UL++kZx8fWHzZqMV5ehojNb/1IqnSDQlsqX/FioVldazPYiKiiI4ODjNci8z8fbr14/ixYvTvXt3duzYwenTp9m5cydjxoxJ6clXo0YN1qxZw6FDhzhy5Aj9+/dPc11a2C9rtqBqAEla67BU60KANlk9SRldqFoBczPZPhQYCuDr60tgYGC2gcTExLBo0aI0s866urrSo0cPduzYke3z71Z0dHSO4irsrH2cbtxwZvr0muzZY7SIWre+yujRYTg7J5K6mhHlRpBQJoHLoZe5zGWr1Z9bIiMjSU5OLrDvqUuXLrFr1y4eeuihNOtbt26d0rpJ/dpDQ0MpWfK/Vm/6Mh988AFLliyhR48e3L59mxIlStCwYUOOHTvGv//+S58+ffjkk09o2bIlXl5ePPnkk/j5+XHp0qWUfViqtyDKd59VlubguJcFI8lcSrfuBSAwm+dNxkhkrtnVkdP5oD7//HPt6empgZTFzc1NR0ZG5nh+krthr3Os2BtrHqefftK6VClj/qYiRbT+/nutTab/tscmxurFIYu1KfXKfCK/zQdla/L/l3P2eqzIZD4oa/biiwaKpFtXBLiV2ROUUi9hXIvqqrWOz6zc3dBaM23aNG7fvp2yztHRkWeeeYaiRYtaowphQxcvwlNPQY8ecPUqtGsHR47AgAHGtSeAJFMSfVf1pf+a/vx56U+bxiuEuHfWTFBhgJNSKvUNBA2A9B0kAFBKDQbGA+211hesFURgYCA3btxIs87FxYUxY8ZYqwphAyYTfP011K5t3M/k4QGzZsHWrZC6U6bWmhHrR/DTiZ+Y/dhsGpUtPBMvClHQWO0alNb6tlJqNfCeUup5jF583YEW6csqpfoBHwJttdb/WCsGgKlTp2boWl67dm3q1q1rzWpEHjp6FIYNM0aBAGP22y++gMqVM5aduH0i3/z5DRNaTeCVR+R+NyHyM2vfqDsScAeuAD8CI7TWoUqpVkqp1FnjfaAEcEApFW1evr7fyi9cuJDhAqC3t7d0Lc+noqJg/Hh46CEjOZUpA8uXw7p1lpPT0StH+XDXh7zQ6AWmtJ2S9wELIazKqvdBaa2vAz0srN8FeKV6XNWa9d4xZ86cOx0vUjg6OtKjR4aQhB1LTobvvjNGf7hs7nQ3fDh89BFkcqsLAHVL12XncztpXqG5jK8nRAFQYOaDio+P56uvvkpz75Obmxsvv/xyhjvRhf3avh1GjYKQEONx8+Ywc6YxGnlmNv29Ca01nat35n+V/pc3gQohcl2BGYtv5cqVKcOf3KG1ZsSIETaKSNyN48ehZ0+jV15IiNHx4ccfYc+erJPTvvP76LWsF5N3TMakTZkXFELkOwWmBZW+c4RSio4dO1K2bFkbRiWy89df8N57xph5JhN4esKbb8Lo0eDunvVzQ6+E0nVJV8oXKc/Pz/wsI5MLUcAUiAT1559/Zpgt18PDgzfeeMNGEYnsnD5tTIHx/ffGNSdnZxg61Bg/LyffKc7dPEenxZ1wdXJlS/8t+Hr55n7QQog8VSAS1KeffpphbK3SpUvTsqVVh/cTVvDXX/Dpp8b4eUlJxnh5Q4YYHSKqVMn5fhYELyA6IZqdz+2kqk+u9LkRQthYvk9Q169fZ/Xq1WmuP3l6ejJu3DjpyWVH9u2Dd96pw+7dxgCvDg7G6A/vvAPVqt39/ia2nsiA+gMkOQlRgOX7k/bz58/PkIi01gwYkGEQdZHHTCb46Sdo2RJatIBdu0rh7Gy0mEJDjdN7d5OcEpITGPLzEMKuhaGUkuQkRAGXrxJUQkICK1asSJkl12QyMX36dGJjY1PKODk5ERAQIJOP2dCVKzB1KlSvbvTM27vXuH+pX7+znDkD33wDtWrd3T5N2sTANQP5Nvhb9v+7PzfCFkLYmXx1ii86Opq+ffvi6enJsGHDqFGjRppBYcFIUKNGjbJRhIWX1sYU619/DatXQ2Kisb5KFXjtNaPVFBR0mrJlLQwBke2+Na9ufJVlocuY1mEa/ev3t2rsQgj7lK8SlKOjI56enkRFRTF79my01iTe+SQ0a9SoEdWrV89kD8Lazp0zuogvXAgnThjrHBzgiSeM0R8efdToCHE/3t/5PnMOzGFs87G83vL1+w9aCJEv5LsEded6U/rZcsEYd++1117L46gKnxs3YOVKWLzYmMX2jrJl4YUX4PnnoWJF69SVkJzAln+2ENAggKkdp1pnp0KIfCFfJSgnJ6cMo0WklpSUREBAAJs2bWLMmDH4+fnlYXQF240bsH69cfrul1/gzvcDNzfo3h369YPHHjPuZ7IWrTUuji5s6b8FJwcnuRFXiEImX/3HOzo6Zjill1psbCyxsbEsXLiQ+vXrM3ny5DyMruAJD4cvv4SOHaF0aRg40OiVl5gIHTrAggXGYK5Ll0K3btZNTr/98xuP/fAYN+Nu4u7sjrOjjKcoRGGT71pQlk7tpefg4ICvry/9+8vF9LuRmGjcr7R5s7EcPPjfNkdHY5y8nj2NpXz53IvjYPhBeizrQZViVWR8PSEKsXyVoJRS2SYpd3d3/Pz82LJlC8WLF8/D6PIfreHkSQgMNBLSb7/BrVv/bXdzg06djIT0+ONQokTuxxR2LYzOP3SmhHsJNvffjI+7T+5XKoSwS/kqQYExSkRmCcrDw4MuXbqwePFiXF1d8zgy+2cyGbPT7thhdG7YudO4Zym1WrWMa0mdOkHr1sbU6nkl/FY4jy56FICtA7ZSzrtc3lUuhLA7+S5BeXt7c+PGjQzr3d3dGTVqFFOmTJEhjjBaR//+C/v3w4EDxs+gIGOW2tR8fY1E1LGjkZQqVbJNvABR8VF4uniy+unVVC8htwoIUdjluwRVrFgxzp07l2adu7s7X3/9NQMHDrRRVLZlMsHZs0brKCTkv4R06VLGshUrQps2RlJq08YY7cHW+TwhOQFnB2dqlazF4eGHcXS4zxunhBAFQr5LUKmvKyml8Pb2Zt26dbRu3dqGUeUNrY1ec8eOGcnoyBFjCQ2FVFNhpShWDJo2NZaHHzZ+lrOzs2aJyYn0WtaLB30eZHbn2ZKchBAp8l2CKlmyJADOzs6UKlWK7du3U6NGDRtHZT1aw7VrxrQUlpbUnRhS8/WFevWgbt3/klK1arZvHWXFpE0MWTuEDX9t4OuuX9s6HCGEncl3CcrX1xcHBwfq1q3Lli1bUhJWfhEfDxcuGEMEnT9v/Ey/pBteMI1ixYyODPXq/ZeQ6taFUqXy7CVYhdaa17e8zqLDi5jSdgrDmgyzdUhCCDuT7xJU06ZNuX79Ot99953d9NRLTlZcvWqcfrt06b+fln5P32vOEm9v49qQpaVECftuFeXUp3s/ZcbvM3j54ZeZ0GqCrcMRQtihfJegAgICCAgIsOo+tYaYGOP0WeolKgquXzeWa9f++z39cvNmmxzX5eho3ORaqZLlpWJFKFq0YCShrNQsWZPnGj7HrMdmSa9LIYRFVk1QSqniwHzgUSACeFNrvcRCOQV8DDxvXvUNMF5rrbPaf1ISnDkDsbEZl5gYy+vvbIuJMRJO+iR0Z8liiL8cvG6Nj4/C19e4FlSmjLFY+r1UKXDKd18LrCciJoKSHiV5ouYTPFHzCVuHI4SwY9b+qPwCSAB8gYbABqVUiNY6NF25oUAPoAGgga3AaSDLK+UhIVA1lyZRdXMzTq15e0ORIv/9LF487VKiRMZ1f/65g3bt/HMnsALkcORhus3uxuKei+leq7utwxFC2DmVTaMl5ztSyhO4AdTVWoeZ1y0C/tVaj09Xdi+wQGs9z/x4CPCC1rpZVnU4ODykXVw24uCQgKNjPA4Od5YEHBzi06278/jOtjgcHWNSFienWPPvt3F0jMXBIfmeX3tkZCTFihW75+cXBtGe0fzZ8E9cE1156M+HcE6UwV8zExwcTFJSEk2aNLF1KPmC/P/lnL0eqx07dhzUWmd4w1uzBVUDSLqTnMxCAEsXaOqYt6UuV8fSTpVSQzFaXDg7O1Or1mP3HajWxsCoWQyMfleSk5OJjIy0zs4KoHiPeP5u/jcOSQ5U2VWF27FZdFMUJCUlobWW91QOyf9fzuW3Y2XNBOUFpBtIh5uAdyZlb6Yr56WUUumvQ5lbWfMAmjRpooOCgqwXsZUEBgbi7+9v6zDsUlR8FI3nNaZIbBGm15nOoKmDbB2S3fP39ycyMpLg4GBbh5IvyP9fztnrscqso5Q1E1Q0UCTduiKApVtL05ctAkRn10lC5D/eLt4MbjiYtlXbEvd3nK3DEULkI9acsDAMcFJKpR7lswGQvoME5nUNclBO5FNxSXH8ff1vlFK82epNmlXI8vKiEEJkYLUEpbW+DawG3lNKeSqlWgLdgUUWin8PjFZKlVdKlQPGAAusFYuwrWRTMs+uepZm3zTjRmzGkeeFECInrD3l+0jAHbgC/AiM0FqHKqVaKaVSD2c6F1gHHAGOAhvM60Q+p7VmxIYRrDmxhnfavCMTDgoh7plV74PSWl/HuL8p/fpdGB0j7jzWwDjzIgqQidsn8n+H/o8JrSbwyiOv2DocIUQ+Zu0WlCjEVoSu4INdH/BCoxeY0naKrcMRQuRzhXjQHWFt3Wp2Y/qj03n1kVdlfD0hxH2TFpS4b7vP7eZG7A3cnNwY3Xy0TDoohLAKSVDivvx+4Xc6Le7EyxtftnUoQogCRhKUuGfHrh6j65KulPUqy/RHp9s6HCFEASMJStyTczfP0WlxJ1wcXdgyYAu+Xr62DkkIUcBIJwlxT4atH0ZUfBQ7B+3kAZ8HbB2OEKIAkgQl7sn8J+ZzNvIsDco0yL6wEELcAznFJ3IsITmBz/74jCRTEuW8y9G8YnNbhySEKMAkQYkcMWkTAT8F8OqmV9l+erutwxFCFAKSoES2tNa8uvFVlh5dytQOU+n4YEdbhySEKAQkQYlsfbDrA+YcmMOY5mN4vcXrtg5HCFFISIISWboQdYGPdn/EwAYDmdZxmgxhJITIM9KLT2SpQpEK/PH8H9QsURMHJd9nhBB5Rz5xhEXbTm9jbpAxRVfd0nVxdnS2cURCiMJGEpTI4GD4Qbov7c6cA3OIT4q3dThCiEJKEpRI469rf9H5h86UcC/Bpn6bcHVytXVIQohCShKUSBF+K5xHFz+KRrNlwBbKFylv65CEEIWYdJIQKTb/vZlrMdfYFrCNGiVq2DocIUQhJwlKpHjuoefoXL0zZbzK2DoUIYSQU3yFXWJyIv1X92fn2Z0AkpyEEHZDElQhprXmhXUv8MORHzh+9bitwxFCiDQkQRVib/z6BgtDFjLZfzLDmgyzdThCCJGGVRKUUqq4UmqNUuq2UuqsUurZLMq+rpQ6qpS6pZQ6rZSSwd1s4JM9n/DJ3k94semLTGw90dbhCCFEBtbqJPEFkAD4Ag2BDUqpEK11qIWyChgIHAYeBLYopc5rrZdaKRaRDa01f176k6frPM1nnT+T8fWEEHbpvhOUUsoT6A3U1VpHA7uVUmuBAcD49OW11tNSPTyplPoZaAlIgsoDJm3CQTmwuNdikkxJMr6eEMJuWaMFVQNI0lqHpVoXArTJ7onK+OreCpibRZmhwFDzw2il1Mn7iDW3lAQibB1EPiDHKedKKqXkWOWMvK9yzl6PVWVLK62RoLyAqHTrbgLeOXjuJIzrYN9lVkBrPQ+Yd6/B5QWlVJDWuomt47B3cpxyTo5Vzsmxyrn8dqyyPb+jlApUSulMlt1ANFAk3dOKALey2e9LGNeiumqtZURSIYQQaWTbgtJa+2e13XwNykkpVV1r/Zd5dQPAUgeJO88ZjHF9qrXW+kLOwxVCCFFY3PcVcq31bWA18J5SylMp1RLoDiyyVF4p1Q/4EOiotf7nfuu3E3Z9CtKOyHHKOTlWOSfHKufy1bFSWuv734lSxYFvgY7ANWC81nqJeVsrYKPW2sv8+DRQAUh9Wm+x1nr4fQcihBCiwLBKghJCCCGsTW6CEUIIYZckQQkhhLBLkqCsTClVXSkVp5RabOtY7JFSylUpNd88ZuMtpVSwUqqzreOyF3czrmVhJu+je5PfPp8kQVnfF8ABWwdhx5yA8xgjjRQF3gaWK6Wq2DIoO5J6XMt+wFdKqTq2Dckuyfvo3uSrzydJUFaklHoGiAR+s3EodktrfVtrPUlrfUZrbdJarwdOA41tHZutpRrXcqLWOlprvRu4M66lSEXeR3cvP34+SYKyEqVUEeA9YLStY8lPlFK+GOM5ZnpjdyGS2biW0oLKhryPspZfP58kQVnPFGC+jIyRc0opZ+AHYKHW+oSt47ED9zOuZaEl76McyZefT5KgciC78QiVUg2BDsBMG4dqczkYu/FOOQeM0UYSgJdsFrB9uadxLQszeR9lLz9/PllrwsICLQfjEb4GVAHOmSf/8wIclVJ+WutGuR2fPcnuWEHKNCvzMToCdNFaJ+Z2XPlEGHc5rmVhJu+jHPMnn34+yUgSVqCU8iDtN9+xGG+IEVrrqzYJyo4ppb7GmHm5g3mSS2GmlFoKaOB5jGP0C9Aik9mpCzV5H+VMfv58khaUFWitY4CYO4+VUtFAnL3/8W1BKVUZGIYxFuOlVNPND9Na/2CzwOzHSIxxLa9gjGs5QpJTRvI+yrn8/PkkLSghhBB2STpJCCGEsEuSoIQQQtglSVBCCCHskiQoIYQQdkkSlBBCCLskCUoIIYRdkgQlhBDCLkmCEkIIYZf+HyH7e8JeBFvXAAAAAElFTkSuQmCC\n"
     },
     "metadata": {
      "needs_background": "light",
      "image/png": {
       "width": 424,
       "height": 280
      }
     },
     "output_type": "display_data"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "## Xavier and He Initialization",
   "metadata": {
    "cell_id": "00009-3f8a9e1f-21cd-47e6-9c40-5ce75ed88c8d",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00010-94c04dde-6524-495e-9629-52c29a7c0055",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d7d7407",
    "execution_start": 1659354414334,
    "execution_millis": 18,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 703,
    "deepnote_output_heights": [
     611
    ]
   },
   "source": "[name for name in dir(keras.initializers) if not name.startswith(\"_\")]",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 4,
     "data": {
      "text/plain": "['Constant',\n 'GlorotNormal',\n 'GlorotUniform',\n 'HeNormal',\n 'HeUniform',\n 'Identity',\n 'Initializer',\n 'LecunNormal',\n 'LecunUniform',\n 'Ones',\n 'Orthogonal',\n 'RandomNormal',\n 'RandomUniform',\n 'TruncatedNormal',\n 'VarianceScaling',\n 'Zeros',\n 'constant',\n 'deserialize',\n 'get',\n 'glorot_normal',\n 'glorot_uniform',\n 'he_normal',\n 'he_uniform',\n 'identity',\n 'lecun_normal',\n 'lecun_uniform',\n 'ones',\n 'orthogonal',\n 'random_normal',\n 'random_uniform',\n 'serialize',\n 'truncated_normal',\n 'variance_scaling',\n 'zeros']"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00011-c8a4bca6-84cb-4837-982d-617cbcea8bdd",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "3fa1e2cb",
    "execution_start": 1659354414343,
    "execution_millis": 12,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 113.1875,
    "deepnote_output_heights": [
     21.1875
    ]
   },
   "source": "keras.layers.Dense(10, activation=\"relu\", kernel_initializer=\"he_normal\")",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 5,
     "data": {
      "text/plain": "<keras.layers.core.dense.Dense at 0x7f95629b3d00>"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00012-62b666bd-e358-4177-bf40-6050a772aeac",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "113986d0",
    "execution_start": 1659354414360,
    "execution_millis": 42,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 149.1875,
    "deepnote_output_heights": [
     21.1875
    ]
   },
   "source": "init = keras.initializers.VarianceScaling(scale=2., mode='fan_avg',\n                                          distribution='uniform')\nkeras.layers.Dense(10, activation=\"relu\", kernel_initializer=init)",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 6,
     "data": {
      "text/plain": "<keras.layers.core.dense.Dense at 0x7f95629c1670>"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "## Nonsaturating Activation Functions",
   "metadata": {
    "cell_id": "00013-9d3b79c6-c6a6-45d5-becb-3cabee3f4e1d",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "markdown",
   "source": "### Leaky ReLU",
   "metadata": {
    "cell_id": "00014-bff487f8-4ffa-445e-8c34-baf77b5cae20",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 62
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00015-b8c491f2-660a-4ca6-9d88-440983596b76",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "ecf9a150",
    "execution_start": 1659354414400,
    "execution_millis": 0,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "source": "def leaky_relu(z, alpha=0.01):\n    return np.maximum(alpha*z, z)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00016-19ed541e-1b50-4eb9-91e5-9509b68acd0a",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "67e6cbd6",
    "execution_start": 1659354414401,
    "execution_millis": 537,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 600.1875,
    "deepnote_output_heights": [
     null,
     282
    ]
   },
   "source": "plt.plot(z, leaky_relu(z, 0.05), \"b-\", linewidth=2)\nplt.plot([-5, 5], [0, 0], 'k-')\nplt.plot([0, 0], [-0.5, 4.2], 'k-')\nplt.grid(True)\nprops = dict(facecolor='black', shrink=0.1)\nplt.annotate('Leak', xytext=(-3.5, 0.5), xy=(-5, -0.2), arrowprops=props, fontsize=14, ha=\"center\")\nplt.title(\"Leaky ReLU activation function\", fontsize=14)\nplt.axis([-5, 5, -0.5, 4.2])\n\nsave_fig(\"leaky_relu_plot\")\nplt.show()",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Saving figure leaky_relu_plot\n",
     "output_type": "stream"
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqV0lEQVR4nO3de3xU1b3//9cHAkIICfCLoNQCxwtWwIIaqdZL02LVClYRUBQFvICXI2KPWKVeSsW7qFWs4gWKClUQUKza/hRPo4IWiYqnQgsVCwpyU0gg5kaS9f1jDTqEhMxMMtlzeT8fj3mwZ8/O3u/ZM8xn9t5r1jLnHCIiIommRdABRERE6qICJSIiCUkFSkREEpIKlIiIJCQVKBERSUgqUCIikpBUoCQiZubMbGjQOZKZmY02s5Jm2lazvF5mdoKZ/Z+ZVZpZQby310CWHqHnnRdkDmk6KlApwMxmmtkrQeeIhplNCn2YODOrMbMvzWy2mX0/yvUUmNkj9Ty21swm1LPtT2LNHmGuugrEHODgJt5Ofa/9gcCfm3Jb9XgI+Bg4BDinGbYH1Pu6f4F/3subK4fElwqUBGkV/gPlIOA84EhgbqCJ4sg5V+ac29JM29rknKtohk0dCvyvc+4L59y2ZthevZxz1aHnXRVkDmk6KlBpwMx6mdmrZrbTzLaY2XNmdkDY48ea2etm9pWZ7TCzxWZ2fAPrvCG0/Amhvxla6/Gfm9kuM+uyj9VUhT5QvnTOvQM8CRxnZtlh6znTzD4ws3Iz+4+Z3WFmrWPcFRExs5ZmNj20vTIz+7eZ/drMWtRabpSZ/cPMKsxss5k9HZq/NrTIC6EjqbWh+d+e4jOznqHHjqy1zrGh/dqqoRxmNgkYBQwMOxrNDz22xxGcmR1pZotC69kWOvLKCXt8ppm9YmbjzWyDmW03sz+aWWY9+6iHmTkgB5gR2t5oM8sPTefWXnb3qbewZQaY2VIzKzWzQjM7utY2jjOz/zWzb8ysODTd1cxmAj8B/jvsefeo6xSfmZ0c2kZ56DV6MPz9EzoSe9TM7gzt9y1mNqX2ay3B0IuQ4szsQOBt4BOgP3AKkAUsDPtP2B54FjgptMxy4DUz+//qWJ+Z2RRgHPAT59wS4DngklqLXgK84pzbHGHOA/CniKpDN8zsNGA28AjQO7TOocCdkayzEVoAG4BzgSOAm4DfABeH5b0ceBz4I/BD4Az8PgY4NvTvGPwR4u7733LOrQaWASNqPTQCmOuc2xVBjin4I85Foe0cCLxbe1tm1g74/4ES/Os7GPgxMKPWoicBffDvkfNCy42vvb6Q3afTSoFrQ9Nz6lm2PncBNwJHA18Ds83MQpn7An8DPgVOAI4LrT8jlOk9/L7f/by/qON5fw/4C/ARcBRwKXB+aLvhRgBV+H1ydej5nBflc5F4cM7pluQ3YCa+GNT12G3Am7XmdQQc0L+evzFgI3Bh2DyH/0/7R2A10D3ssTz8f/Dvha2/DBi0j8yT8IWoBP8h50K3h8KWeRu4pdbfnR36GwvdLwAeqWcba4EJ9Wz7kyj38d3AorD764G797G8A4bWmjcaKAm7fw2wLuy5dANqgB9HkaPO1z58+/hCWQy0D3s8P7TMoWHr+QJoGbbMk+HbqidPCTC6jvXmhs3rEZqXV2uZ08KWOSE076DQ/dnAe/vY7l6vex3buQP4N9Ci1mtQAWSGree9Wut5A3gq1v+PujXdTUdQqe8Y4GQzK9l947tvm4cAmFlnM3vczFabWTGwE+iM/8AMNwX/4XKic27d7pnOuULgH/jTTQAXANvw3173ZQ3QD3+EcRPwIf4IITz7TbWy/wloBxxAHJnZFaHTTltD2/0Vof1hZp2B7wFvNnIzzwNd8Ucu4L/d/8c59+1R0L5yROEI4P+cczvD5r2LL4a9wuatdM5Vh93/Ev8+iJf/q7UtwrZ3FPC/jVz/EcDfnXM1YfMWA63x187qyrE7Szyft0RIBSr1tQBexReC8NthwO7WX0/ji8Sv8Kc5+uGPEGpf63kDXxjOqGM7T+G/nYI/Ffd0rQ+7ulQ65z51zq1wzt2J/6D4Q63sv6uV+4eh7FsbWDfADvw1kto64I8o6mRm5wG/xx9VnBba7qPsvT8axfkGE2/w3Wm+Efgjh+bMET6cwa46Hov2M2J3MbCwea3qWTZ8e7tzNNdnUlM/b4mDjKADSNx9iL+Gsc756xp1ORG4xjn3KoD5hg0H1rHca8ACQhf/nXNPhz02G7jPzK7GX1MYHkPW24FVZjbVOfdBKPsPnHOfxrAu8K0Ej6lj/tGhx+pzIrDUOfdtM2YzO2T3tHNui5ltAAbgC0xddgEtI8g4C3jEzJ7At2IMb2yyzxwhlRFs55/AJWbWPuwo6sf4D+F/RpAxGru/OBwYNt0vhvV8BPxsH49H+rzPNbMWYUdRJ4b+dk0MmaSZ6VtC6sg2s361bj3wRyQ5wBwz+5GZHWxmp5jZE2bWPvS3q4ELzbf2OxZ/6qmyro04514BhgHTzGxk2Pwi4AXgfuBt59y/o30Czrk1wEJgcmjWbcAFZnabmfUxsx+Y2VAzu7fWn+bW8dy7Ag8Cp5nZLaHn1tvM7gCODz1Wn9XA0Wb2CzM7zMxuwbcaC3cHcK2Z/cp8i7x+ZnZd2ONrgQFmdoCZddzHtl7CH2FMB5Y533gimhxrgT5mdriZ5ZpZXUcrs/HX+Z4x35rvZHwDjwWNKP71+RR/CnlSaL+cCtwcw3ruA44KvU/7hp7fZWa2+/TmWqB/qOVebj2t7h7Fn0J91MyOMLOB+Gt4jzjnSmPIJM0t6ItgujX+hj8F5Oq4zQs9fhgwD9iOb7ywCpgKtA493hdYGnpsDXARvkXapLBt7HHRHzgztPzIsHknh5YbGUHmSdTRUAH/zd4RaigAnAq8g/+A3QEUAleHLV9Qz3OfUuvvt+FbihUAJzeQrTW+YGwHikLTtwJray13KbASX8w3ATNq7Z9/44+k1obmjSaskUTYss+EMl8TbQ5gf+B1/HVDB+TX83odib9mVhZa30wgp9Z76JVa26/zNaq1zB6NJMJew+Whbb0HDKTuRhL1NqQIzTsR31CmLPT8FwEHhh7rGVr37gY2PepZx8n493YFsBn/xWS/Wu+f2o0t9toXugVz2916SKTRQtdMHge6On1DFZFG0jUoaTTzP+Y8AN8C70kVJxFpCroGJU3h1/jThtv47vqRiEij6BSfiIgkJB1BiYhIQorbNajc3FzXo0ePeK2+Ub755hvatWsXdIykpf0Xm1WrVlFdXU2vXr0aXlj2ovdd7Orbd1u2wBdfgBn84AeQWWfXwPH3wQcffOWc27/2/LgVqB49elBYWBiv1TdKQUEB+fn5QcdIWtp/scnPz6eoqChh/18kOr3vYlfXvnvzTTjtND/9/PNw7rnNn2s3M1tX13yd4hMRSTOffeYLUnU1TJwYbHHaFxUoEZE0UlICZ58N27bBwIEwOYHb3apAiYikCedg9Gj4xz/g8MNh9mxoGUmPkQFRgRIRSRN33AHz50N2NixcCDl19fWfQFSgRETSwMKFcMstvsXec8/5I6hEF1WBCvWoXG5ms+IVSEREmtbatZlceKGfvvNOOKOuEd0SULRHUH8AlsUjiIiINL3t2+Hmm/tQUgLnnQc33BB0oshFXKDMbDi+y/vGDnMtIiLNoLoahg+HDRsy6dcPZszwp/iSRUQ/1DWzbPzgcT8DLtvHcmOBsQBdunShoKCgCSI2vZKSkoTNlgy0/2JTVFREdXW19l2M9L6L3rRpB/P6693Izq7ghhs+5P33K4KOFJVIe5KYDEx3zq23fZRf59wTwBMAeXl5LlF/9a1fpDeO9l9sOnToQFFRkfZdjPS+i87s2TBnDmRkwO9+t5Lhw48POlLUGixQZtYPOAU4Ku5pRESk0T74AC4Lnet66CHo1as42EAxiuQIKh8/lPLnoaOnLKClmfVyzh0dv2giIhKtzZt9TxHl5TBmDFx5Jbz1VtCpYhNJgXoCeD7s/gR8wboyHoFERCQ2lZUwZAisXw8//jE88khyNYqorcECFRq++9shvM2sBCh3zm2NZzAREYnONdfAkiXwve/5HiNatw46UeNEPdyGc25SHHKIiEgjTJsGjz8O++0HL70EBxwQdKLGU1dHIiJJ7p13YNw4P/3kk5CXF2yepqICJSKSxD7/3F93qqqC666Diy4KOlHTUYESEUlSpaW+xd7WrfDzn8PddwedqGmpQImIJCHn/G+dPvoIDjnED9ueEXWrgsSmAiUikoTuu88Pm5GV5YfS6NQp6ERNTwVKRCTJ/PWvcOONfvrZZ6F372DzxIsKlIhIElm92vdQ7hxMmuSvQaUqFSgRkSSxYwecdRYUF8PgwX6E3FSmAiUikgRqamDECPjXv6BPH3j6aWiR4p/gKf70RERSw623wiuvQMeOvqeI9u2DThR/KlAiIgnuhRfgjjv8EdPcub5ZeTpQgRIRSWAffwyjR/vpKVPglFMCjdOsVKBERBLUV1/5VnqlpTByJFx7bdCJmpcKlIhIAtq1C849F9auhWOP9T2VJ/PYTrFQgRIRSUDXXQd/+5sfNuPFF6FNm6ATNT8VKBGRBDNjBkyd6gccXLDAD0CYjlSgREQSyN//Dlde6acffRSOPz7YPEFSgRIRSRBffgnnnAOVlXD11XDppUEnCpYKlIhIAigv990XbdwI+fnwwANBJwqeCpSISMCcgyuugPffh+7d/Y9xW7UKOlXwVKBERAL28MO+b73MTN+N0f77B50oMahAiYgE6M03fZNygD/+Efr1CzROQlGBEhEJyGef+R/jVlfDb37jp+U7KlAiIgEoKfFjO23bBgMHwuTJQSdKPCpQIiLNrKbGdwD7ySdw+OEwe3bqj+0UC+0SEZFmdscdMH8+5OTAwoX+X9mbCpSISDNauNAPPmgGf/qTP4KSuqlAiYg0kxUr4MIL/fRdd8EZZwSbJ9GpQImINIPt2/3YTiUlMHw4/PrXQSdKfCpQIiJxVlXli9Knn8JRR8H06ek3tlMsVKBEROJs4kR4/XXIzfVjO2VmBp0oOahAiYjE0ezZMGUKZGTAvHm+rz2JjAqUiEicFBbCZZf56Ycfhp/8JNg8yUYFSkQkDjZv9sNnlJfDmDG+t3KJjgqUiEgTq6yEIUNg/Xo44QR45BE1ioiFCpSISBMbNw6WLIHvfc9fd2rdOuhEyUkFSkSkCU2bBk88AW3a+LGdDjgg6ETJSwVKRKSJvP22P3oCePJJyMsLNk+yU4ESEWkCn38OQ4f6H+Ved913XRpJ7CIqUGY2y8w2mtkOM1ttZpfFO5iISLIoLfXdGG3dCqeeCnffHXSi1BDpEdRdQA/nXDbwS+B2MzsmfrFERJKDc3DppfDRR3DIIfD88/5HudJ4ERUo59wK51zF7ruh2yFxSyUikiTuu88XpawsP5RGx45BJ0odEdd5M3sUGA20BT4CXqtjmbHAWIAuXbpQUFDQJCGbWklJScJmSwbaf7EpKiqiurpa+y5Gifi+W7q0ExMnHgkYN9zwD7Zu/ZoEiwgk5r6LhDnnIl/YrCVwPJAP3OOc21Xfsnl5ea6wsLDRAeOhoKCA/Pz8oGMkLe2/2OTn51NUVMTy5cuDjpKUEu19t3o19O8PxcXwu9/5QQgTVaLtu9rM7APn3F5tHqNqxeecq3bOLQYOAq5sqnAiIsmkuBjOOsv/e845cPPNQSdKTbE2M89A16BEJA3V1Pgm5P/6F/TpA08/DS30g524aHC3mllnMxtuZllm1tLMTgPOB96MfzwRkcRy663wyivQqZNvFJGVFXSi1BVJIwmHP503DV/Q1gHXOudejmcwEZFE88ILcMcd/ohpzhw4+OCgE6W2BguUc24roFFMRCStffwxjB7tp++/H045JdA4aUFnTkVEGvDVV75RRGkpjBoF48cHnSg9qECJiOzDrl0wbBisW+eblU+bprGdmosKlIjIPlx3HRQU+GEzFizww2hI81CBEhGpx4wZMHWqH3BwwQI/AKE0HxUoEZE6vPceXBnqjuCxx+D444PNk45UoEREatmwwfcQUVkJV18Nl1wSdKL0pAIlIhKmvNwXp02bID8fHngg6ETpSwVKRCTEObjiCnj/feje3f8wt1WroFOlLxUoEZGQhx7yfetlZvpujHJzg06U3lSgRESARYtgwgQ/PXMm9O0baBxBBUpEhM8+g/POg+pq+M1v/A9zJXgqUCKS1kpKfDdG27bBoEEweXLQiWQ3FSgRSVs1NTByJHzyCRx+OMyapbGdEoleChFJW7ffDi++CDk5vlFETk7QiSScCpSIpKWFC+G3v/Udvz73nD+CksSiAiUiaWfFCj9sO8Bdd8EvfhFsHqmbCpSIpJVt23yjiJISGD4cfv3roBNJfVSgRCRtVFXB+efDmjVw1FEwfbrGdkpkKlAikjYmToTXX4f994eXXvI9RkjiUoESkbQwaxZMmQIZGTBvHnTrFnQiaYgKlIikvMJCuOwyP/3ww3DyycHmkcioQIlIStu0CQYPhooKGDvW91YuyUEFSkRSVmUlDB0K69fDCSf44dvVKCJ5qECJSEpyzo+Gu2QJHHQQzJ8PrVsHnUqioQIlIilp2jR48klo08Z3Z9SlS9CJJFoqUCKSct5+G665xk8/+STk5QWbR2KjAiUiKWXdOn/dqarKD0C4u0sjST4qUCKSMkpLfYu9rVvh1FPh7ruDTiSNoQIlIinBObj0UvjoIzj0UHj+eWjZMuhU0hgqUCKSEu691xelrCzfjVHHjkEnksZSgRKRpPfaa76fPfBdGvXuHWweaRoqUCKS1Fatggsu8Kf4brvND6UhqUEFSkSSVnGxL0jFxXDOOXDTTUEnkqakAiUiSam6GkaM8EdQffrA009DC32ipRS9nCKSlG69FV59FTp1goULfeMISS0qUCKSdObOhTvv9M3I586Fgw8OOpHEgwqUiCSVjz+Giy/201OmwIABweaR+FGBEpGk8dVXvlFEaSmMGgXjxwedSOJJBUpEkkJVlTFsmO9rr39/31u5xnZKbQ0WKDPbz8ymm9k6M9tpZsvN7BfNEU5EZLdHHz2EggI44AA/fEabNkEnkniL5AgqA/gC+AmQA9wMzDWzHnHMJSLyrenT4cUXD6J1a1iwALp2DTqRNIeMhhZwzn0DTAqb9YqZ/Qc4Blgbn1giIt5778GVV/rpxx6D448PNo80nwYLVG1m1gXoCayo47GxwFiALl26UFBQ0Nh8cVFSUpKw2ZKB9l9sioqKqK6u1r6LwtatrbniimPYtWs/Bg36DwcfvA7tvugl6//ZqAqUmbUCZgNPO+f+Vftx59wTwBMAeXl5Lj8/vykyNrmCggISNVsy0P6LTYcOHSgqKtK+i1B5OZx8MmzbBj/9KYwf/7n2XYyS9f9sxK34zKwF8CxQCVwdt0Qikvacg8svh2XLoEcP/2PcjAwXdCxpZhEdQZmZAdOBLsAZzrldcU0lImntoYfgmWcgM9OP7ZSbG3QiCUKkp/geA44ATnHOlcUxj4ikuUWL4Lrr/PTMmdC3b6BxJECR/A6qO3A50A/YZGYloduIeIcTkfSyZg2cey7U1PihM4YNCzqRBCmSZubrAP1eW0TiqqQEzj4btm+HQYP84IOS3tTVkYgErqYGRo6ETz6BH/zAD9uusZ1EbwERCdztt/vui3Jy/NhOOTlBJ5JEoAIlIoF66SX47W99x6/PPw89ewadSBKFCpSIBGbFCrjoIj99991w+unB5pHEogIlIoHYts2P7VRSAuefD9dfH3QiSTQqUCLS7KqqYPhw36z8qKPgqac0tpPsTQVKRJrdjTfCG2/A/vv7a1CZmUEnkkSkAiUizerZZ+H++yEjA+bPh27dgk4kiUoFSkSaTWEhjBnjp6dOhZNOCjaPJDYVKBFpFps2+Z4iKipg7Fi44oqgE0miU4ESkbirqIAhQ2DDBjjhBH/0JNIQFSgRiSvnYNw4ePddOOggf92pdeugU0kyUIESkbiaNg2efBLatPHdGXXpEnQiSRYqUCISN2+9Bddc46efegry8oLNI8lFBUpE4mLdOhg61P8od8IEGKER5CRKKlAi0uRKS32Lva++glNP9f3siURLBUpEmpRzcMklsHw5HHqo76G8ZcugU0kyUoESkSZ1770wZw5kZfmxnTp2DDqRJCsVKBFpMq+9BhMn+unZs6FXr2DzSHJTgRKRJrFqlR82wzm47Tb45S+DTiTJTgVKRBqtuNiP7bRjh+8x4qabgk4kqUAFSkQapbraNyFftQqOPBJmzoQW+mSRJqC3kYg0yq23wquvQqdOfmynrKygE0mqUIESkZjNnQt33umbkc+dCwcfHHQiSSUqUCISk+XL4eKL/fT998OAAYHGkRSkAiUiUdu61fcUUVoKo0d/19+eSFNSgRKRqOzaBcOG+b72+veHxx4Ds6BTSSpSgRKRqPzP//heyg880A+f0aZN0IkkValAiUjEpk+HRx7xAw4uWABduwadSFKZCpSIROTdd+HKK/30tGlw3HHB5pHUpwIlIg1avx7OOcdff7rmmu9a74nEkwqUiOxTebkvTps3w09/ClOmBJ1I0oUKlIjUyzkYOxaWLYMePfyPcVu1CjqVpAsVKBGp1+9/D88+C5mZfmyn3NygE0k6UYESkTotWgQTJvjpmTPhhz8MNI6kIRUoEdnLmjVw7rlQU+OHzhg2LOhEko5UoERkDzt3+rGdtm+HM8/0gw+KBEEFSkS+VVMDo0bBihVwxBEwa5bGdpLgRPTWM7OrzazQzCrMbGacM4lIQCZP9t0X5eT4sZ2ys4NOJOksI8LlvgRuB04D2sYvjogE5aWXYNIkf8T0/PPQs2fQiSTdRVSgnHMLAMwsDzgorolEpNmtWAEXXeSn77oLTj892DwioGtQImlv2zbfKKKkBM4/H66/PuhEIl6kp/giYmZjgbEAXbp0oaCgoClX32RKSkoSNlsy0P6LTVFREdXV1Qm176qrjRtvPJI1azpx2GE7GTnyI956qyboWHXS+y52ybrvmrRAOeeeAJ4AyMvLc/n5+U25+iZTUFBAomZLBtp/senQoQNFRUUJte+uuw4KC2H//WHRovZ063Zy0JHqpfdd7JJ13+kUn0iaevZZeOAByMiA+fOhW7egE4nsKaIjKDPLCC3bEmhpZm2AKudcVTzDiUh8LFsGY8b46alT4aSTgs0jUpdIj6BuBsqAG4ELQ9M3xyuUiMTPpk0weDBUVMDll8MVVwSdSKRukTYznwRMimsSEYm7igoYMgQ2bIATT4SHHw46kUj9dA1KJE04B1df7YduP+ggmDcPWrcOOpVI/VSgRNLEY4/BU09Bmza+14guXYJOJLJvKlAiaeCtt2D8eD89fTocc0yweUQioQIlkuLWrYOhQ6GqyvcSccEFQScSiYwKlEgKKy2Fs8+Gr76C007z/eyJJAsVKJEU5RxccgksXw6HHgrPPQctWwadSiRyKlAiKeqee2DOHMjKgoULoWPHoBOJREcFSiQFvfoq/OY3fnr2bOjVK9g8IrFQgWomZsa8efOCjiFpYNUq3xDCOT9C7i9/GXQikdioQIWMHj2aQYMGBR1DpFGKi/3YTjt2+B4jbrop6EQisVOBEkkR1dUwYoQ/gjrySJg5E8yCTiUSOxWoCKxcuZKBAwfSvn17OnfuzPnnn8+mTZu+fXzZsmWceuqp5Obmkp2dzYknnsh77723z3Xec8895Obm8ve//z3e8SVN3HKLv/bUqZNvFJGVFXQikcZRgWrAxo0bOfnkk+nTpw/vv/8+ixYtoqSkhLPOOouaGj/y6M6dO7nooot45513eP/99+nXrx9nnHEGX3/99V7rc84xYcIEpk6dyltvvcVxxx3X3E9JUtCcOf43Ti1bwty58F//FXQikcZr0hF1U9Fjjz1G3759ueeee76d98wzz9CpUycKCwvp378/P/vZz/b4m6lTpzJ//nz+8pe/cOGFF347v7q6mksuuYQlS5awZMkSunfv3mzPQ1LX8uVw8cV++oEHYMCAQOOINBkVqAZ88MEHvP3222TVcb5kzZo19O/fny1btnDLLbfwt7/9jc2bN1NdXU1ZWRmff/75HstPmDCBjIwMli5dSufOnZvrKUgK27rVN4ooK4PRo2HcuKATiTQdFagG1NTUMHDgQKZMmbLXY11C3UGPGjWKzZs38+CDD9KjRw/2228/BgwYQGVl5R7L//znP+e5557jtddeY/To0c0RX1LYrl0wbBh8/jn86Ee+t3I1ipBUogLVgKOPPpq5c+fSvXt3WrVqVecyixcv5uGHH2bgwIEAbN68mY0bN+613BlnnME555zDsGHDMDNGjRoV1+yS2n71K99L+YEHwoIFfhgNkVSiRhJhduzYwfLly/e4DRw4kOLiYs477zyWLl3KZ599xqJFixg7diw7d+4EoGfPnsyaNYuVK1eybNkyhg8fTut6RoIbNGgQL7zwAldccQXPPPNMcz49SSFPPQV/+IMfcHDBAujaNehEIk1PR1Bh3nnnHY466qg95g0ZMoQlS5YwceJETj/9dMrLy+nWrRunnnoq++23HwAzZsxg7NixHHPMMXTt2pVJkyaxdevWerczaNAg5s6dy7nnngvAyJEj4/ekJOW8+y5cdZWfnjYN1BBUUpUKVMjMmTOZOXNmvY/vq5uivn37snTp0j3mXXTRRXvcd87tcf/MM8+krKws+qCS1tavh3PO8defrrnmu9Z7IqlIp/hEkkRZGQweDJs3w89+BnW02xFJKSpQIknAObj8cigshB49/A9z62mzI5IyVKBEksDvfw/PPguZmb4bo9zcoBOJxF/KF6hVq1YxY8aMoGOIxOyNN2DCBD/99NPwwx8Gm0ekuaRsIwnnHNOnT2f8+PHU1NTQsWNHBg8eHHQskaisWQPnnQc1NXDzzTB0aNCJRJpPSh5BFRUVcdZZZzF+/HhKS0spLy9n1KhRrF+/PuhoIhHbudN3Y7R9O5x5Jvzud0EnEmleKVeg3nvvPQ4//HBef/11SktLv51fWlrK4MGDv+2BXCSR1dTAyJGwYgUccQTMmgUtUu5/q8i+pcxbvrq6mkmTJjFgwAC2bNlCRUXFHo9nZGSwefNmysvLA0ooErnJk+Gll6BDB98oIjs76EQizS8lCtSGDRs4/vjjue++++r88WtmZiaDBw9m5cqVZGZmBpBQJHIvvgiTJvkjpueeg8MOCzqRSDCSvpHEwoULGTlyJKWlpVRVVe3xWIsWLWjbti2PP/44I0aMCCihSOQ++cSf2gO4+244/fRg84gEKWkLVFlZGePGjeNPf/pTvUdNBx98MC+//DL/peFFJQls2+YbRZSUwPnnf9e0XCRdJeUpvpUrV9KnT596i1Pbtm256qqr+PDDD1WcJClUVcHw4fDZZ3D00b63co3tJOkuqY6gnHNMmzaNCRMmUFZWtlcHrK1atSIrK4t58+btNQy7SCK74Qb/g9zOnf01KF0qFUmiArV9+3ZGjBjB22+/vUfz8d3atWtH//79mTt3LrnqB0aSyDPPwAMPQEYGzJsH3boFnUgkMSTFKb7FixfTs2dP3nzzTb755pu9Hm/bti2TJ0/mzTffVHGSpLJsGYwd66cfeQROOinYPCKJJKGPoKqqqpg0aRIPPPBAndea2rRpw/7778+f//xn+vbtG0BCkdht2uSHz6io8D2VX3550IlEEkugR1CVlZV8+OGHdT72xRdf8KMf/YgHH3yw3lZ6Q4YM4Z///KeKkySdigoYMgQ2bIATT4SHHw46kUjiCbRA3X///Rx77LEsW7Zsj/nz58+nd+/efPzxx3tdb2rRogVZWVnMmDGDWbNm0a5du+aMLNJozsF//7cfuv373/fXnVq3DjqVSOIJ7BTfjh07uPPOO6mpqeGss85i1apVZGRkcNVVVzF37tw6G0JkZmZy2GGHsXDhQrp37x5AapHGe/RRmD4d2rTxLfa6dAk6kUhiiugIysw6mdmLZvaNma0zswsau+F7772X6upqALZt28aQIUPo1asXc+bMqbM4tW3blnHjxlFYWKjiJEmrpCSDa6/109OnwzHHBBpHJKFFegT1B6AS6AL0A141s4+dcyti2ejXX3+9x7WliooKFi9eXOe1ptatW5OVlcX8+fPJz8+PZXMiCaGoCNaubUd1NVx/PVzQ6K95IqnNav/Yda8FzNoB24E+zrnVoXnPAhucczfW93ft27d3x9Tz9fDTTz9l48aNDQ590aJFC7Kzs+nVqxetWrXa9zOJQlFRER06dGiy9aUb7b+91dT43iDqu33zDWzZshyATp360aePeoqIlt53sUv0fffWW2994JzLqz0/kiOonkDV7uIU8jHwk9oLmtlYYCz4Xh2Kior2WtmuXbv48ssv9+oFoo51ccABB5Cbm1vnb58ao7q6us5sEplU3H81NUZ1dey3SLVqVcNBBxVRXBzHJ5OiUvF911ySdd9FUqCygB215hUD7Wsv6Jx7AngCIC8vzxUWFu61sjFjxvDpp59SWVlZ7wazs7NZvHgxRx55ZATxoldQUKDThY2QaPuvuhp27PCn0IqKoLj4u+m67teeV1zsj4Aao00byMnx4zeF33bP69gRFizIp7KyiOXLlzduY2kq0d53ySTR953VczohkgJVAtQeLi0b2BltiHXr1jFr1qx9Fif47igrXgVKEsuuXdEXlfB5O2p/fYpBu3Z1F5b67ofPy8nxBaohf/0rNPDWF5EwkRSo1UCGmR3mnPt3aF5fIOoGEhMnTtxrzKa6lJWVMXz4cFatWkXnzp2j3Yw0s/Ly2I9eioqgjkabUcvJ2XcR2Vehyc6GJrzEKSJNpMEC5Zz7xswWALeZ2WX4VnxnAT+OZkOrV6/mxRdfjKhAgf+d1JgxY1i4cGE0m5EoOecLRKRHK0VF8PnnR1NT8939iorGZWjRIvKjlbrut28PLVs2LoOIJJ5Im5lfBcwAtgBfA1dG28T8+uuvZ9euXXvN390zRE1NDeXl5Rx44IH07t2b/v37c8opp0SzibRUUwM7d0Z/Wiz8fujnaFHY84xvq1b+Gks0p8XCb+3aqUWbiOwtogLlnNsGnB3rRlasWMHLL79MVlYWAOXl5XTt2pU+ffpw7LHH0qdPH3r37s2hhx7apM3Jk0FV1Z4X+KM9VVZc7I+CGqNt2+hOi3322Yf89KdHf3u/TRsVGBFpes3S1VFWVha33347RxxxBL179+aQQw4hIyOhO1KPWGVldEcrte+XlDQ+Q/v2sV9/ycmJvh+4goIdHHFE43OLiOxLs1SJ7t27c9NNNzXHpqLi3J4X+GMpNHV0fhEVsz0LR6SnxXbPy872A92JiKSapP5oc84fgUR7Wmzjxv5UVPj7dVwWi0pGRvTNksNvWVm+kYCIiOwp0AJVU9O46y9FRbH+wDLz26nWrf0F/lh+/9KhA2Rm6vqLiEg8xK1Abd4Mt96670LTVD+wjPa02KpVSznttB9F/ANLERFpfnErUOvXw+TJDS+XnR379ZecnNh+YFlWVqYxeEREElzcClTnznDVVfsuNPqBpYiI1CduBer734ff/jZeaxcRkVSn9mMiIpKQVKBERCQhqUCJiEhCUoESEZGEpAIlIiIJSQVKREQSkgqUiIgkJBUoERFJSCpQIiKSkFSgREQkIZlr7Hjh9a3YbCuwLi4rb7xc4KugQyQx7b/Yad/FTvsudom+77o75/avPTNuBSqRmVmhcy4v6BzJSvsvdtp3sdO+i12y7jud4hMRkYSkAiUiIgkpXQvUE0EHSHLaf7HTvoud9l3sknLfpeU1KBERSXzpegQlIiIJTgVKREQSkgqUiIgkJBUowMwOM7NyM5sVdJZkYGb7mdl0M1tnZjvNbLmZ/SLoXInMzDqZ2Ytm9k1ov10QdKZkoPda00jWzzgVKO8PwLKgQySRDOAL4CdADnAzMNfMegQZKsH9AagEugAjgMfMrHewkZKC3mtNIyk/49K+QJnZcKAIeDPgKEnDOfeNc26Sc26tc67GOfcK8B/gmKCzJSIzawcMAW5xzpU45xYDLwMXBZss8em91njJ/BmX1gXKzLKB24D/CTpLMjOzLkBPYEXQWRJUT6DKObc6bN7HgI6goqT3WnSS/TMurQsUMBmY7pxbH3SQZGVmrYDZwNPOuX8FnSdBZQE7as0rBtoHkCVp6b0Wk6T+jEvZAmVmBWbm6rktNrN+wCnAgwFHTTgN7buw5VoAz+KvrVwdWODEVwJk15qXDewMIEtS0nsteqnwGZcRdIB4cc7l7+txM7sW6AF8bmbgv+W2NLNezrmj450vkTW07wDM77Tp+Iv+ZzjndsU7VxJbDWSY2WHOuX+H5vVFp6kiovdazPJJ8s+4tO3qyMwy2fNb7QT8i3mlc25rIKGSiJlNA/oBpzjnSgKOk/DM7HnAAZfh99trwI+dcypSDdB7LTap8BmXskdQDXHOlQKlu++bWQlQniwvXJDMrDtwOVABbAp9OwO43Dk3O7Bgie0qYAawBfga/yGh4tQAvddilwqfcWl7BCUiIoktZRtJiIhIclOBEhGRhKQCJSIiCUkFSkREEpIKlIiIJCQVKBERSUgqUCIikpBUoEREJCH9P15b8kEI3NI/AAAAAElFTkSuQmCC\n"
     },
     "metadata": {
      "needs_background": "light",
      "image/png": {
       "width": 424,
       "height": 280
      }
     },
     "output_type": "display_data"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00017-e300167e-4020-4ca1-b4b1-88bee1458661",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "31b03eb9",
    "execution_start": 1659354414941,
    "execution_millis": 20,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 401,
    "deepnote_output_heights": [
     309
    ]
   },
   "source": "[m for m in dir(keras.activations) if not m.startswith(\"_\")]",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 9,
     "data": {
      "text/plain": "['deserialize',\n 'elu',\n 'exponential',\n 'gelu',\n 'get',\n 'hard_sigmoid',\n 'linear',\n 'relu',\n 'selu',\n 'serialize',\n 'sigmoid',\n 'softmax',\n 'softplus',\n 'softsign',\n 'swish',\n 'tanh']"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00018-7ed29779-1608-44e9-a182-c29b4aacdffd",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "40548c01",
    "execution_start": 1659354414965,
    "execution_millis": 7,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 113.1875,
    "deepnote_output_heights": [
     21.1875
    ]
   },
   "source": "[m for m in dir(keras.layers) if \"relu\" in m.lower()]",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 10,
     "data": {
      "text/plain": "['LeakyReLU', 'PReLU', 'ReLU', 'ThresholdedReLU']"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "Let's train a neural network on Fashion MNIST using the Leaky ReLU:",
   "metadata": {
    "cell_id": "00019-760a01f6-747d-41d1-ab89-1af5bf3f3fe1",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00020-514384bb-e4a5-4987-b54a-5c19a62eef5f",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "e0f89a26",
    "execution_start": 1659354414969,
    "execution_millis": 860,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 320.5
   },
   "source": "(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()\nX_train_full = X_train_full / 255.0\nX_test = X_test / 255.0\nX_valid, X_train = X_train_full[:5000], X_train_full[5000:]\ny_valid, y_train = y_train_full[:5000], y_train_full[5000:]",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz\n29515/29515 [==============================] - 0s 0us/step\nDownloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz\n26421880/26421880 [==============================] - 0s 0us/step\nDownloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz\n5148/5148 [==============================] - 0s 0us/step\nDownloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz\n4422102/4422102 [==============================] - 0s 0us/step\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00021-477ee5a2-8efb-4e5c-aed2-cd1e321e624f",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "eab61a81",
    "execution_start": 1659354415833,
    "execution_millis": 100,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 256
   },
   "source": "tf.random.set_seed(42)\nnp.random.seed(42)\n\nmodel = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    keras.layers.Dense(300, kernel_initializer=\"he_normal\"),\n    keras.layers.LeakyReLU(),\n    keras.layers.Dense(100, kernel_initializer=\"he_normal\"),\n    keras.layers.LeakyReLU(),\n    keras.layers.Dense(10, activation=\"softmax\")\n])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00022-cac733f8-ae1a-4ceb-948c-b912dffe7142",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "b1938699",
    "execution_start": 1659354415943,
    "execution_millis": 0,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 112
   },
   "source": "model.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n              metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "scrolled": true,
    "cell_id": "00023-81facd41-30ae-434b-acab-789558ac166b",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "9d2223f7",
    "execution_start": 1659354415993,
    "execution_millis": 82704,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 508.75
   },
   "source": "history = model.fit(X_train, y_train, epochs=10,\n                    validation_data=(X_valid, y_valid))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/10\n1719/1719 [==============================] - 7s 4ms/step - loss: 1.2819 - accuracy: 0.6229 - val_loss: 0.8886 - val_accuracy: 0.7160\nEpoch 2/10\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.7955 - accuracy: 0.7362 - val_loss: 0.7130 - val_accuracy: 0.7656\nEpoch 3/10\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.6816 - accuracy: 0.7721 - val_loss: 0.6427 - val_accuracy: 0.7898\nEpoch 4/10\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.6217 - accuracy: 0.7944 - val_loss: 0.5900 - val_accuracy: 0.8064\nEpoch 5/10\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.5832 - accuracy: 0.8075 - val_loss: 0.5582 - val_accuracy: 0.8200\nEpoch 6/10\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.5553 - accuracy: 0.8156 - val_loss: 0.5350 - val_accuracy: 0.8238\nEpoch 7/10\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.5338 - accuracy: 0.8223 - val_loss: 0.5157 - val_accuracy: 0.8304\nEpoch 8/10\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.5173 - accuracy: 0.8272 - val_loss: 0.5079 - val_accuracy: 0.8284\nEpoch 9/10\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.5040 - accuracy: 0.8289 - val_loss: 0.4895 - val_accuracy: 0.8388\nEpoch 10/10\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.4924 - accuracy: 0.8321 - val_loss: 0.4816 - val_accuracy: 0.8396\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "Now let's try PReLU:",
   "metadata": {
    "cell_id": "00024-287e36ff-0a93-4736-8695-a48e0bf69e43",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00025-cdb48b47-3717-4784-8428-c4553946beaa",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "50c66201",
    "execution_start": 1659354498702,
    "execution_millis": 59,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 256
   },
   "source": "tf.random.set_seed(42)\nnp.random.seed(42)\n\nmodel = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    keras.layers.Dense(300, kernel_initializer=\"he_normal\"),\n    keras.layers.PReLU(),\n    keras.layers.Dense(100, kernel_initializer=\"he_normal\"),\n    keras.layers.PReLU(),\n    keras.layers.Dense(10, activation=\"softmax\")\n])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00026-61dc3ae1-872f-43fa-bfae-f48b450fa13a",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "b1938699",
    "execution_start": 1659354498767,
    "execution_millis": 2,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 112
   },
   "source": "model.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n              metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00027-35949b1b-3f41-4f92-9606-8225c417c52e",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "9d2223f7",
    "execution_start": 1659354498773,
    "execution_millis": 82538,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 508.75
   },
   "source": "history = model.fit(X_train, y_train, epochs=10,\n                    validation_data=(X_valid, y_valid))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/10\n1719/1719 [==============================] - 7s 4ms/step - loss: 1.3461 - accuracy: 0.6209 - val_loss: 0.9255 - val_accuracy: 0.7186\nEpoch 2/10\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.8197 - accuracy: 0.7355 - val_loss: 0.7305 - val_accuracy: 0.7630\nEpoch 3/10\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.6966 - accuracy: 0.7693 - val_loss: 0.6565 - val_accuracy: 0.7880\nEpoch 4/10\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.6331 - accuracy: 0.7909 - val_loss: 0.6004 - val_accuracy: 0.8046\nEpoch 5/10\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.5917 - accuracy: 0.8056 - val_loss: 0.5656 - val_accuracy: 0.8184\nEpoch 6/10\n1719/1719 [==============================] - 8s 4ms/step - loss: 0.5618 - accuracy: 0.8135 - val_loss: 0.5406 - val_accuracy: 0.8238\nEpoch 7/10\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.5390 - accuracy: 0.8205 - val_loss: 0.5196 - val_accuracy: 0.8314\nEpoch 8/10\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.5213 - accuracy: 0.8257 - val_loss: 0.5113 - val_accuracy: 0.8312\nEpoch 9/10\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.5070 - accuracy: 0.8288 - val_loss: 0.4916 - val_accuracy: 0.8378\nEpoch 10/10\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.4945 - accuracy: 0.8315 - val_loss: 0.4826 - val_accuracy: 0.8396\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "### ELU",
   "metadata": {
    "cell_id": "00028-2679c57e-c5f8-4740-bf88-9bcd4a6e4175",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 62
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00029-80b29783-e5f9-47da-9184-79c2cbec9094",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "6bde996b",
    "execution_start": 1659354581312,
    "execution_millis": 0,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "source": "def elu(z, alpha=1):\n    return np.where(z < 0, alpha * (np.exp(z) - 1), z)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00030-11f67a76-16d4-4560-a986-eb9207e1682f",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "7cc14d05",
    "execution_start": 1659354581313,
    "execution_millis": 1383,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 582.1875,
    "deepnote_output_heights": [
     null,
     282
    ]
   },
   "source": "plt.plot(z, elu(z), \"b-\", linewidth=2)\nplt.plot([-5, 5], [0, 0], 'k-')\nplt.plot([-5, 5], [-1, -1], 'k--')\nplt.plot([0, 0], [-2.2, 3.2], 'k-')\nplt.grid(True)\nplt.title(r\"ELU activation function ($\\alpha=1$)\", fontsize=14)\nplt.axis([-5, 5, -2.2, 3.2])\n\nsave_fig(\"elu_plot\")\nplt.show()",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Saving figure elu_plot\n",
     "output_type": "stream"
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAl/UlEQVR4nO3deZgU1dn38e8Ng+wgiI4LIkYFNURJmOSJGnViiALBYNTgHtEYCIRXiZqovOjja3g0GkwwKigGH8KigrgEkcUltooSFXEIEAFBZFX2BoZtmJnz/nF6cOhZm6mZqp7+fa6rrumpU13n7jM1fXedPnXKnHOIiIhETYOwAxARESmPEpSIiESSEpSIiESSEpSIiESSEpSIiESSEpSIiESSEpSIiESSEpSIiESSEpSkHTMbZ2bT61E9DczsSTPbYmbOzHJru85KYqmT15yoq42ZbTCzk+qivlSZ2fNmdlvYcWQy00wS9ZuZjQOuL6foA+fc9xPl7ZxzvSt4fgxY5JwbnLS+H/CYc65FoAFXr+7W+GM3nk71VFJ/b+BFIBf4HNjqnCuozToT9cZIet119ZoTdf0Jf+zdUNt1lVP3ecDtQDfgWOAG59y4pG2+BbwNnOic217XMQpkhR2A1Ik3gOuS1tX6G2Btqas3izp8UzoZ+NI5934d1VehunrNZtYMuAm4uC7qK0cLYBEwPrGU4ZxbaGafA9cCj9dhbJKgLr7MsM8591XSsrW2KzWzHmb2rpltM7OtZjbbzE4rVW5mdpuZfWZm+8xsrZk9kCgbB5wP/CbR7eXMrGNJmZlNN7P+iS6ihkn1PmNm06oTR3XqKbWfxmY2MlHnXjP7l5n9oFR5zMxGmdn9ZrbZzDaa2Qgzq/D/LFH/X4AOibq/KLWvx5K3LYmnOnUdSvum+poP9XUDvQAHvFdOm3QzszfNbI+ZLTez88ysr5mV2fZQOedmOOeGOuemAsWVbDoNuCqoeiU1SlBSm5oDI4Hv4buvtgOvmNlhifL7gbuBB4BvAj8H1iTKbgHmAv8LHJNYSspKPA+0Bn5cssLMWgB9gInVjKM69ZR4CLgCuBH4NrAQmGVmx5Ta5hqgEDgbGAwMSTynIrcA9wFrE3V/t5Jtk1VVV03bF6r3mqsTS7JzgY9d0ncMZvZd4F3gLeAM4F/A/wP+b+K1kLT9UDPLr2I5t5I4qvIh8D0za1qDfcghUhdfZuhhZvlJ6x53zt1Rm5U6514o/buZ3QDswP/D5wG/BYY4555ObLIc/6aJc267mRUAu51zX1Ww/21mNgP/5jgrsfoS/BvltFLbVRiHc25OVfUkntMcGAjc5Jx7NbHu18AFwG+AYYlN/+OcuyfxeJmZ/Qr4EfBsBa9hu5ntBIoqq78CFdaVSNQpt6+ZHcprTvl1AycA68tZ/zDwinNueKK+Z4BXgHecc/8sZ/sngCkV1FFiXRXllVkPNMJ/T7WiBvuRQ6AElRneAfonrYvXdqXmR2f9Afgv4Ej8GXsDoAP+O7DGwJs1rGYi8Hcza+ac241PVi845/ZWM47qOgn/RnWgm8k5V2Rmc4HTS23376TnrQeOSqGeVFRW1+nUvH2r+5qriqU8TYENpVeY2dH4M6sfllpdgP9blTl7SsSzFajN7uo9iZ86gwqBElRm2O2cW36Iz92B70ZLdji+q6wy0/FdVwPwn2ILgf8Ah1X2pBS9mthvHzN7E+gOXFTHcZTuptpfTtmhdKUXA5a0rlHS70HVdSiSh/+mGstmoE3SupLvJ+eVWtcZWOqcm1PeTsxsKDC08lDp6Zx7t4ptKtI28XPTIT5fakAJSqqyFOhlZpb0fcF3EmXlMrMjgFOBQc65txLrvsPXx9ynwD58N9BnFeymAGhYQRkAzrl9ZvY8/sypHfAVEEshjmrVg+/eKQDOSTzG/OCMs4BnqnjuodiE/16otDOBL6r5/CDatzZf8ydAv6R1h+MTW1Girpb4754q6/qs7S6+LsA659yGKreUwClBZYbGie6T0oqccyWfCluZWdek8rhz7gtgNP5L70fN7ClgL34E1lXATyupcxv+U/KvzGwNcBzwJ/zZC865nWb2CPCAme3Dd0MeAXRzzo1O7OML/PdVHYF8/PVB5Y24mojvyjoReDZpm0rjqG49zrldZjYaeNDMNgMr8d/xZAOjKmmHQ/VPYKSZ/RT/QWAAcDzVTFCH2r5J+6jN1zw7sd8jnHNbEuvy8GeNd5nZJPzf6UvgZDM7xTlXJtEeahdf4ju6kxO/NsCPouyK/9uvLrXpuYlYJQQaxZcZuuP/0Usvn5QqPzfxe+llBIBz7nPgPOAU4DX8qKYrgZ8752ZWVGHiDf4K/EisRfjrSO7Gf6ovcRfwYGL9p8ALQPtS5SPwn+D/gz+jqOg7o3fxn5JP5+DRe9WNo7r13AFMxo98y0vss4dz7ssKtq+Jp0st7wE7gZdS3EcQ7Vsrr9k5t5Cvj6WSdSvxZ0wDgQX419wd/3cL+hqxHL4+1pviRwp+gh9RCYCZNQF+BjwVcN1STZpJQkRCYWY9gEeA051zRWHHk8zMfgP0cc5dGHYsmUpnUCISCufcLPwZbfuqtg3JfuD/hB1EJtMZlIiIRJLOoEREJJKUoEREJJJCH2berl0717Fjx7DDKGPXrl00b9487DDSjtotNUuXLqWoqIjTT0+emEEqk07HmXOwfDns2AGHHQanngqNki+5rgNRbrOPP/54s3PuyOT1oSeojh07Mm/evKo3rGOxWIzc3Nyww0g7arfU5ObmEo/HI/k/EGXpcpwVF8PVV8P8+XDUUTBnDpxySjixRLnNzGxVeevVxSciUgucg1tugcmToWVLmDkzvOSUrpSgRERqwfDh8NhjvlvvH/+A73wn7IjSjxKUiEjAnngC7rkHGjSAZ5+FH/6w6udIWYEmKDObaGZfmtkOM1tmZjcFuX8RkaibOhUGDfKPR4+GSy8NN550FvQZ1ANAR+dcK/xEosPNrFvAdYiIRNKbb8I11/jvn4YPh/7Jd2GTlASaoJxzi51zJZNwusRyUpB1iIhE0ccfwyWXQEEB3HwzDK3qLlVSpcCHmZvZKPx9XpriZweeUc42/Unc4TU7O5tYLBZ0GDWWn58fybiiTu2Wmng8TlFRkdosRVE7ztasacrNN3+b/PzDuOCCDfTp8ylvvx12VAeLWptVR63MxVfqpma5wIPOueS7bR6Qk5PjongNSJSvGYgytVtqSq6DysvLCzuUtBKl42z9ejj7bFi1Ci66CKZN8yP3oiZKbZbMzD52zuUkr6+VUXzOuaLELZrb4+/tIiJS72zb5pPSqlXwX/8FL7wQzeSUrmp7mHkW+g5KROqh3bvh4oth0SI47TR49VWI6ExCaSuwBGVmR5nZlWbWwswamtlF+NuCvxlUHSIiUbB/P/TtC++9B+3bw+zZcMQRYUdV/wQ5SMLhu/OewCe+VcAQ59y0AOsQEQlVcTHcdJM/Y2rbFl57DY4/Puyo6qfAEpRzbhNwflD7ExGJojvugPHjoVkzmDHDd+9J7dBURyIi1fSnP8GIEZCVBS++6AdGSO1RghIRqYb//V/4/e/94/Hj/eg9qV1KUCIiVZg2DX71K//4kUfgqqvCjSdTKEGJiFTi3XfhiiugqAiGDfPTGEndUIISEanAv//tr3Xau9dP/HrffWFHlFmUoEREyrFypf+eaft2f8uMUaPALOyoMosSlIhIkg0b4MIL4auv/M0GJ02Chg3DjirzKEGJiJSyYwf07AnLl8O3vw0vvwxNmoQdVWZSghIRSdi719/T6ZNP4OSTYeZMaNUq7KgylxKUiAh+lN4118Bbb8HRR/spjLKzw44qsylBiUjGcw4GDfKzQ7Ru7Sd/PfHEsKMSJSgRyXj33ANjxvjvml55Bc44I+yIBJSgRCTD/fWvMHy4H6U3ZQqce27YEUkJJSgRyVjPPAO33OIf/+1v/qJciQ4lKBHJSLNmwfXX+8cPPQT9+oUajpRDCUpEMs4HH8Bll0FhIdx+O/zud2FHJOVRghKRjPLpp9CrF+ze7c+gHnww7IikIkpQIpIx1qzxUxht3Qq9e8NTT0EDvQtGlv40IpIRtmzxyWntWjjnHJg8GRo1CjsqqYwSlIjUe/n58JOfwJIl0KWLv9apWbOwo5KqKEGJSL1WUACXX+4HRpxwgp8lok2bsKOS6lCCEpF6q7jYDx+fPRuOPNLPr3fssWFHJdWlBCUi9ZJzMGQIPPsstGjhZybv1CnsqCQVSlAiUi/dfz88+igcdhj84x/QrVvYEUmqlKBEpN4ZMwaGDfO3aJ80CS64IOyI5FAoQYlIvfLCCzBwoH88apQfICHpSQlKROqNt96Cq6/2gyPuuw9+/euwI5KaUIISkXph/nzo08cPKx882HfxSXpTghKRtPfZZ9CjB+zcCVdeCY884r9/kvSmBCUiaW39ej+F0aZN/uff/6759eoL/RlFJG3F4/7M6Ysv4Hvf8wMkDjss7KgkKEpQIpKW9uzxd8BduBA6d4ZXX/UX5Er9EViCMrPGZjbWzFaZ2U4zyzOznkHtX0SkRFGRccUVMGcOHHecn8KoXbuwo5KgBXkGlQWsAc4HWgPDgClm1jHAOkQkwzkHI0Z04pVX/KSvr70GHTqEHZXUhqygduSc2wXcW2rVdDNbCXQDvgiqHhHJbHfeCbNmHUOzZr5b7/TTw45IakutfQdlZtlAJ2BxbdUhIpllxAh46CFo2LCYqVPhrLPCjkhqU2BnUKWZWSNgEvB359yScsr7A/0BsrOzicVitRFGjeTn50cyrqhTu6UmHo9TVFSkNquGWbOyefDB0wAYMmQBTZtuR81Wfen4v2nOuWB3aNYAeAZoBfRxzu2vbPucnBw3b968QGMIQiwWIzc3N+ww0o7aLTW5ubnE43Hy8vLCDiXSpk+HSy6BoiIYORLOPFPHWaqi/L9pZh8753KS1wfaxWdmBowFsoHLqkpOIiJVmTMHfv5zn5yGDoVbbgk7IqkrQXfxjQZOA7o75/YEvG8RyTALF/prnfbuhZtuguHDw45I6lKQ10GdAAwAugJfmVl+YrkmqDpEJHN88QVcdJGfLeJnP4PRozW/XqYJcpj5KkCHj4jU2MaNfl69L7+E88+HZ56BrFoZ0iVRpqmORCRSduyAnj39DOVdu/rbtTdpEnZUEgYlKBGJjH37fHfe/Plw0kkwaxa0bh12VBIWJSgRiYSiIrj2WvjnP+Hoo/0URtnZYUclYVKCEpHQOQe/+Q1MnQqtWvkzp298I+yoJGxKUCISunvvhSefhMaN4ZVX4Mwzw45IokAJSkRC9dhjcN99/i64kyfDeeeFHZFEhRKUiITmuefg5pv946eegj59wo1HokUJSkRC8dpr8Itf+O+f/vhHuPHGsCOSqFGCEpE69+GHcOmlsH8/3Hor/P73YUckUaQEJSJ1askS6NULdu2C666DP/1JUxhJ+ZSgRKTOrF3rpzDassUnqbFj/eAIkfLo0BCROrFli09Oa9bA2WfD889Do0ZhRyVRpgQlIrVu1y7o3Rs+/RS++U1/rVOzZmFHJVGnBCUitWr/frj8cvjXv6BDB5g9G9q2DTsqSQdKUCJSa4qL4YYb/NRF7dr5oeXHHRd2VJIulKBEpFY454eQT5oELVrAzJnQuXPYUUk6UYISkVrxxz/CI4/4gRAvvQQ5OWFHJOlGCUpEAve3v8HQof76pokToXv3sCOSdKQEJSKBeuklGDDAP378cejbN9x4JH0pQYlIYGIxuOoqPzji3nth4MCwI5J0pgQlIoH45BP46U/9bdsHDYJ77gk7Ikl3SlAiUmPLl0OPHrBzp+/S++tfNb+e1JwSlIjUyJdfwkUXwcaNfjDE+PHQsGHYUUl9oAQlIocsHoeePeHzz/0w8hdf9LdtFwmCEpSIHJI9e/wdcBcsgE6dYMYMaNky7KikPlGCEpGUFRb60XrvvAPHHuunMDryyLCjkvpGCUpEUuKcv87pH/+ANm18cjrhhLCjkvpICUpEUjJ0KDz9NDRtCtOn+9tniNQGJSgRqbY//9nPsdewIUyd6m88KFJblKBEpFomTIDbbvOPx43zt2wXqU1KUCJSpVdf9fd1AvjLX+Daa8ONRzKDEpSIVOr99+HnP4eiIrjrLhgyJOyIJFMoQYlIhRYtgp/8xF/z9Mtfwv/8T9gRSSYJNEGZ2WAzm2dm+8xsXJD7FpG6tWqVn8IoHodLLoEnntD8elK3sgLe33pgOHAR0DTgfYtIHdm0CS68ENavh/PPh2efhayg3y1EqhDoIeecexHAzHKA9kHuW0Tqxs6dfoTesmVw5pn+gtwmTcKOSjJRKJ+JzKw/0B8gOzubWCwWRhiVys/Pj2RcUad2S008HqeoqCgybVZQYNx11xnMn9+GY4/dwz33fMInnxSEHVYZOs5Sl45tFkqCcs6NAcYA5OTkuNzc3DDCqFQsFiOKcUWd2i01hx9+OPF4PBJtVlTk59ebPx+ys+Gdd5py0knRvBJXx1nq0rHNNIpPRHAObr4Znn8eWrWCWbPgpJPCjkoynRKUiHDffTBqlL+X07Rp0LVr2BGJBNzFZ2ZZiX02BBqaWROg0DlXGGQ9IhKcUaPg3nuhQQN47jk/ak8kCoI+gxoG7AHuBK5NPB4WcB0iEpApU2DwYP94zBh/vZNIVAQ9zPxe4N4g9ykiteONN/yces7BAw/4mSJEokTfQYlkoI8+8mdL+/fDb38Ld9wRdkQiZSlBiWSYpUv9hbi7dvkzqBEjNIWRRJMSlEgGWbfOT2G0eTP07OnvjNtA7wISUTo0RTLE1q1+8tfVq+Gss/w1T40ahR2VSMWUoEQywO7d0Ls3LF4Mp58O06dD8+ZhRyVSOSUokXpu/35/w8G5c6FDB5g9G9q2DTsqkaopQYnUY8XFcOONMGMGtGsHr70G7XWfAUkTSlAi9ZRzcPvtMHGi786bMQM6dw47KpHqU4ISqaceegj+8hc/EOKll+C73w07IpHUKEGJ1ENjx8Kdd/rrmyZOhB//OOyIRFKnBCVSz7z8MvTv7x8/9hj07RtqOCKHTAlKpB555x248ko/OOK//xsGDQo7IpFDpwQlUk8sWAAXXwz79sHAgT5BiaQzJSiReuDzz/0sETt2+GueHn1U8+tJ+lOCEklzX33l59fbsAF+9COYMAEaNgw7KpGaU4ISSWPbt/tJX1esgG7d/HDyxo3DjkokGEpQImlq717o0wfy8qBTJ5g5E1q2DDsqkeAoQYmkocJCuOoqePttOPZYP7/ekUeGHZVIsJSgRNKMc36U3ssvw+GH++TUsWPIQYnUAiUokTQzbBj87W/QtKm/bUaXLmFHJFI7lKBE0sjIkXD//X6U3vPPwznnhB2RSO1RghJJE5MmwW9/6x8//TT85CfhxiNS25SgRNLAzJnQr59//PDD8ItfhBqOSJ1QghKJuLlz4bLL/Mi9O+6AW28NOyKRuqEEJRJhixf7rrw9e/ydcR94IOyIROqOEpRIRK1e7efX27YNfvpTePJJza8nmUUJSiSCNm/28+utWwfnngvPPQdZWWFHJVK3lKBEIiY/H3r1gqVL4YwzYNo0f82TSKZRghKJkIICuPRS+OgjOPFEmDXLzxYhkomUoEQiorjYDx9//XU46ih47TU45piwoxIJjxKUSAQ4B7fcApMn+xnJZ82Ck08OOyqRcClBiUTA8OHw2GNw2GH+O6dvfzvsiETCF2iCMrO2ZvaSme0ys1VmdnWQ+xepj7Zsacw990CDBvDss5CbG3ZEItEQ9MDVx4ECIBvoCrxqZgucc4sDrkekXti0Cdau9UP0nnjCD5AQEc+cc8HsyKw5sA3o4pxbllg3AVjnnLuzoue1bNnSdevWLZAYghSPxzlcw6dSpnarvq1bYeHCPABOPLErHTqEG0860XGWuii32dtvv/2xcy4neX2QZ1CdgMKS5JSwADg/eUMz6w/0B2jUqBHxeDzAMIJRVFQUybiiTu1WPfn5WXz+eXMAsrKKadUqjpqt+nScpS4d2yzIBNUC2JG0bjvQMnlD59wYYAxATk6OmzdvXoBhBCMWi5GrLwNSpnar2rx5cMEFfuTeMcfkctRRcfLy8sIOK63oOEtdlNvMKpjDK8hBEvlAq6R1rYCdAdYhktby8qBHD9i5E668Ek45JeyIRKIryAS1DMgys9L/cmcCGiAhAnz4Ifzwh7BlC/TuDePHa/JXkcoElqCcc7uAF4H7zKy5mZ0D9AEmBFWHSLqaMwe6d4d4HC65BKZOhUaNwo5KJNqCvlB3ENAU2Ag8CwzUEHPJdP/8p79tRkm33pQp0Lhx2FGJRF+g10E557YClwS5T5F09vzzcN11sG8fXH89jB0LDRuGHZVIetBURyK1wDkYMQL69vXJadAgePppJSeRVChBiQSssBAGD4bf/c7//uCDfp69BvpvE0mJ7tEpEqDt2+Gaa+DVV/3Er+PHwxVXhB2VSHpSghIJyKJFfi69zz6Dtm3h5Zf97dpF5NCo00EkAFOmwPe/75PTmWf6O+IqOYnUjBKUSA3s2we33uq78Xbt8t17778P3/hG2JGJpD918Ykcok8/hauv9tMXZWXBn//sB0dodgiRYChBiaTIOXjySX/mtGePP1uaNMl38YlIcNTFJ5KC1av9PHoDB/rkdP318MknSk4itUEJSqQaiovh8cfhm9+EGTOgdWt/e/Zx46BV8hz+IhIIdfGJVGHxYvj1r/2Er+CHkj/2GBxzTLhxidR3OoMSqUA8DkOG+GHjc+bA0Uf7WchfeEHJSaQuKEGJJCkqgqee8jcTfOQRPyhi4ED4z3/gssvCjk4kc6iLTyTBOT/7w7BhPhkBnH++T1JnnhlqaCIZSWdQkvGcgzff9CPxLr3UJ6eOHeG55+Ctt5ScRMKiMyjJWM7BzJlw//3w3nt+XXY23H03/OpXfrJXEQmPEpRknMJCePFFeOABPwsE+Mldb7sNbrkFmjcPNTwRSVCCkoyxdasf/PD447BmjV939NFw++0wYAC0aBFufCJyMCUoqdecg3/9y99q/Zln/OwPAJ06+SHkN9wATZqEGqKIVEAJSuqlr76CCRP8bdaXLPl6fY8ecPPNcNFFusOtSNQpQUm9sWuXH/QwfryfjqioyK/PzoZf/AJ++Uvo3DncGEWk+pSgJK3t3Olvrz51qk9KJV14WVlwySVw443+rKlRo1DDFJFDoAQlaWfdOpg9G6ZNg1mz/E0DS3z/+9C3r79x4FFHhRejiNScEpRE3r59/jqlWbP8snDh12Vm/tbql1/uL7Jt3z68OEUkWEpQEjn79sFHH8E77/hlzhz//VKJ5s3hggugZ0/fjaeJW0XqJyUoCd2GDT4hffABvPuuHxZeutsOoEsX/11Sjx7wgx9A48bhxCoidUcJSurUxo2+i27ePJ+UPvrI36U2WZcucN55fjn3XDj22LqPVUTCpQQltWLXLn+jv4ULYdEi/3PhQp+gkrVoAd26wXe/68+OfvADOOKIuo9ZRKJFCUoO2b59sHIlfPaZX5Yvhw8/PIMtW2DVKj+LQ7KWLf3ZUdeu8L3v+aVzZ2jYsM7DF5GIU4KScjkH27f7OevWrPHdcKUfr1rlfxYXJz+zLeCvQzr1VPjWt/zSpYv/ecIJfuSdiEhVlKAyTGEhbNrkByZs3Oh/liwbN/opgtau9cknP7/yfTVoACee6O88e8opcPLJsGfPv7n00jM48UTdrkJEakYJKg0552dM2LEDtm3zs3Rv21b1482bYcuW8rveytOsGXToAMcf75fkx+UloVhsq6YTEpFABJKgzGww0A/4FvCsc65fEPtNV8XFPoHs3Xvwz8rW5ef7ZefOyn+WLGW71qrHDI480s+ykJ399VL69/btfRJq00bdcSISnqDOoNYDw4GLgKapPHHfPli2zE/sWXopLi67Lqj1hYWwfz8UFBz8s/TjdetO49FHy66v6HFBwdcJZ//+gFq1Ek2a+AEHbdv6RFKyVPZ7u3Z+ydJ5s4ikAXPV7e+pzs7MhgPtUzmDMmvpoFvS2r7AIGA30KucZ/VLLJuBy8spHwhcAawBriun/DbgYmApMKCc8mFAdyAPGFJO+f3A2cD7wNByykfStGlXGjZ8g4KC4TRowEFLly5PcsQRndm27RU+++xhGjTwo9hKlptumkCHDseTlzeZ118ffVBZw4YwdepUjj66HePGjWPcuHFlap8xYwbNmjVj1KhRTJkypUx5LBYDYMSIEUyfPv2gsqZNmzJz5kwA/vCHP/Dmm28eVH7EEUfwwgsvAHDXXXcxd+7cg8obNWrE66+/DsCQIUPIK7llbUKnTp0YM2YMAP3792fZsmUHlXft2pWRI0cCcO2117J27dqDys866yweeOABAC677DK2bNlyUPmPfvQj7r77bgB69uzJnpLZYxN69+7N7bffDkBubi7J+vbty6BBg9i9eze9epU99vr160e/fv3YvHkzl19e9tgbOHAgV1xxBWvWrOG668oee7fddhsXX3wxS5cuZcCAAeTl5VFYWEhOTg4Aw4YNo3v37uTl5TFkyJAyz7///vs5++yzef/99xk6tOyxN3LkSLp27cobb7zB8OHDy5Q/+eSTdO7cmVdeeYWHH364TPmECRM4/vjjmTx5MqNHjy5TPnXqVNq1C//Yu+aaa1i3bt1B5e3bt2fixImAjr3yjr0LL7yQoUOHHjj2koV57L399tsfO+dykp8TymdpM+sP9Pe/Neeww4oTXUkOM2jZci9t2uwEdrF2bWHiOV93N7Vrl0929haKirawbNn+A+tL9nH88XGOO+5L9u3bwIIFBZi5UuVw6qmb6NhxFbt2rWXu3L2YuQP7N3Occ84q2refT37+SmbP3nVgfck2P/vZEjp1asTKlf/hxRd3HFjfoIGjQQMYPHgep5wS5+OPFzBhQrzM6x8w4AM6dPiS999fyM6dZctPOmkuRx21gqVLFwPxA2d+JT744D1at27NkiVLiMfLPv+dd96hSZMmLFu2rNzykjeJFStWlCnfs2fPgfKVK1eWKS8uLj5Qvnr16jLlbdq0OVC+du3aMuXr168/UL5+/foy5WvXrj1QvmHDhjLlq1evPlC+adMmduzYcVD5ypUrD5Rv3bqVfUlTUqxYseJAeXlts2zZMmKxGHv37i23fMmSJcRiMbZv315u+eLFi4nFYmzcuLHc8oULF9KyZcsDbVdYWIhz7sC2CxYsICsri+XLl5f7/Pnz51NQUMCiRYvKLZ83bx7xeJwFCxaUW/7BBx/w5ZdfsnDhwnLL586dy4oVK1i8eHG55e+9F41jr6CgoEx5o0aNdOxVcuzt3buXWCxW7v8thH/slSf0M6icnBw3b968wGIISiwWK/dTjlRO7Zaa3Nxc4vF4mU/7UjkdZ6mLcpuZWblnUFXeU9TMYmbmKljm1E64IiKS6ars4nPO5dZBHCIiIgcJaph5VmJfDYGGZtYEKHTOFQaxfxERyTxVdvFV0zBgD3AncG3i8bCA9i0iIhkokDMo59y9wL1B7EtERASCO4MSEREJlBKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEkhKUiIhEUo0TlJk1NrOxZrbKzHaaWZ6Z9QwiOBERyVxBnEFlAWuA84HWwDBgipl1DGDfIiKSobJqugPn3C7g3lKrppvZSqAb8EVN9y8iIpmpxgkqmZllA52AxZVs0x/oD5CdnU0sFgs6jBrLz8+PZFxRp3ZLTTwep6ioSG2WIh1nqUvHNjPnXHA7M2sEzARWOOcGVOc5OTk5bt68eYHFEJRYLEZubm7YYaQdtVtqcnNzicfj5OXlhR1KWtFxlroot5mZfeycy0leX+V3UGYWMzNXwTKn1HYNgAlAATA40OhFRCTjVNnF55zLrWobMzNgLJAN9HLO7a95aCIiksmC+g5qNHAa0N05tyegfYqISAYL4jqoE4ABQFfgKzPLTyzX1HTfIiKSuYIYZr4KsABiEREROUBTHYmISCQpQYmISCQFeh3UIQVgtglYFWoQ5WsHbA47iDSkdkud2ix1arPURbnNTnDOHZm8MvQEFVVmNq+8C8ekcmq31KnNUqc2S106tpm6+EREJJKUoEREJJKUoCo2JuwA0pTaLXVqs9SpzVKXdm2m76BERCSSdAYlIiKRpAQlIiKRpAQlIiKRpARVTWZ2ipntNbOJYccSZWbW2MzGmtkqM9tpZnlm1jPsuKLIzNqa2UtmtivRXleHHVOU6diqmXR8D1OCqr7HgY/CDiINZAFrgPOB1sAwYIqZdQwzqIh6HH+Dz2zgGmC0mX0z3JAiTcdWzaTde5gSVDWY2ZVAHHgz5FAizzm3yzl3r3PuC+dcsXNuOrAS6BZ2bFFiZs2By4C7nXP5zrk5wDTgunAjiy4dW4cuXd/DlKCqYGatgPuAW8OOJR2ZWTbQCVgcdiwR0wkodM4tK7VuAaAzqGrSsVU96fwepgRVtT8AY51za8MOJN2YWSNgEvB359ySsOOJmBbAjqR124GWIcSSdnRspSRt38MyOkGZWczMXAXLHDPrCnQH/hJyqJFRVZuV2q4BMAH/Hcvg0AKOrnygVdK6VsDOEGJJKzq2qi/d38NqfEfddOacy62s3MyGAB2B1WYG/lNvQzM73Tn3ndqOL4qqajMA8401Fv/lfy/n3P7ajisNLQOyzOwU59xniXVnou6qSunYSlkuafwepqmOKmFmzTj4U+7t+D/2QOfcplCCSgNm9gTQFejunMsPOZzIMrPnAAfchG+vGcDZzjklqQro2EpNur+HZfQZVFWcc7uB3SW/m1k+sDcd/rBhMbMTgAHAPuCrxKc2gAHOuUmhBRZNg4CngY3AFvybhpJTBXRspS7d38N0BiUiIpGU0YMkREQkupSgREQkkpSgREQkkpSgREQkkpSgREQkkpSgREQkkpSgREQkkpSgREQkkv4/WSNimQyxs/AAAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light",
      "image/png": {
       "width": 424,
       "height": 280
      }
     },
     "output_type": "display_data"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "Implementing ELU in TensorFlow is trivial, just specify the activation function when building each layer:",
   "metadata": {
    "cell_id": "00031-21104b27-91a9-4552-8101-960c581391d3",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00032-76cfe53a-0408-4b97-a676-0865a9323e7b",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "2141551a",
    "execution_start": 1659354582202,
    "execution_millis": 503,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 113.1875,
    "deepnote_output_heights": [
     21.1875
    ]
   },
   "source": "keras.layers.Dense(10, activation=\"elu\")",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 20,
     "data": {
      "text/plain": "<keras.layers.core.dense.Dense at 0x7f95594d2a60>"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "### SELU",
   "metadata": {
    "cell_id": "00033-9fab7351-3d3c-4bdf-a51b-555afa980808",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 62
   }
  },
  {
   "cell_type": "markdown",
   "source": "This activation function was proposed in this [great paper](https://arxiv.org/pdf/1706.02515.pdf) by Günter Klambauer, Thomas Unterthiner and Andreas Mayr, published in June 2017. During training, a neural network composed exclusively of a stack of dense layers using the SELU activation function and LeCun initialization will self-normalize: the output of each layer will tend to preserve the same mean and variance during training, which solves the vanishing/exploding gradients problem. As a result, this activation function outperforms the other activation functions very significantly for such neural nets, so you should really try it out. Unfortunately, the self-normalizing property of the SELU activation function is easily broken: you cannot use ℓ<sub>1</sub> or ℓ<sub>2</sub> regularization, regular dropout, max-norm, skip connections or other non-sequential topologies (so recurrent neural networks won't self-normalize). However, in practice it works quite well with sequential CNNs. If you break self-normalization, SELU will not necessarily outperform other activation functions.",
   "metadata": {
    "cell_id": "00034-cdb44f99-1e3d-42ab-9295-7593d9a7546e",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 186.734375
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00035-f32deea6-55d8-43e2-8534-c6cb7f2e890b",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "2ab65f55",
    "execution_start": 1659354582219,
    "execution_millis": 10,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 171
   },
   "source": "from scipy.special import erfc\n\n# alpha and scale to self normalize with mean 0 and standard deviation 1\n# (see equation 14 in the paper):\nalpha_0_1 = -np.sqrt(2 / np.pi) / (erfc(1/np.sqrt(2)) * np.exp(1/2) - 1)\nscale_0_1 = (1 - erfc(1 / np.sqrt(2)) * np.sqrt(np.e)) * np.sqrt(2 * np.pi) * (2 * erfc(np.sqrt(2))*np.e**2 + np.pi*erfc(1/np.sqrt(2))**2*np.e - 2*(2+np.pi)*erfc(1/np.sqrt(2))*np.sqrt(np.e)+np.pi+2)**(-1/2)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00036-bcf818fb-c0f3-4093-9695-e0721a9cbb5f",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d77417cb",
    "execution_start": 1659354582234,
    "execution_millis": 10,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "source": "def selu(z, scale=scale_0_1, alpha=alpha_0_1):\n    return scale * elu(z, alpha)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00037-11a2029a-2cf2-4a9e-9515-a46f6e5a8ab1",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "ded9e3c5",
    "execution_start": 1659354582248,
    "execution_millis": 651,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 582.1875,
    "deepnote_output_heights": [
     null,
     282
    ]
   },
   "source": "plt.plot(z, selu(z), \"b-\", linewidth=2)\nplt.plot([-5, 5], [0, 0], 'k-')\nplt.plot([-5, 5], [-1.758, -1.758], 'k--')\nplt.plot([0, 0], [-2.2, 3.2], 'k-')\nplt.grid(True)\nplt.title(\"SELU activation function\", fontsize=14)\nplt.axis([-5, 5, -2.2, 3.2])\n\nsave_fig(\"selu_plot\")\nplt.show()",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Saving figure selu_plot\n",
     "output_type": "stream"
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmIUlEQVR4nO3deXwV1f3/8deHsMgmUcBUReVrFZW6IOZni7Y1FuqCICrWDVRqFdRqxYobglJBUUSLVkGwWBRQQXED1H7VNn61WitUqkUFNxBcQQkQ9iTn98e5kXATQm4yyZl77/v5eMyDmzuTmU8mw31nZs6cY845RERE4qZR6AJERESqooASEZFYUkCJiEgsKaBERCSWFFAiIhJLCigREYklBZTIDpjZFDOb0wDbKTAzZ2btGmBbA83sMzMrM7MR9b29HdQywMyKQ9Yg8aSAkpSYWXszG29mS8xsk5l9bWYvm9kvKyxTmPigTZ4eq7CMM7PTq1h/x8S8/CrmFZrZvfX4s20vIK4A+ke8rSVmNiTp7deB3YFvo9xWFdveBbgPuAPYExhbn9tL2nZVv/cZwL4NVYOkj8ahC5C0MwtoAfwG+AjYDTgGaJu03F+AoUnvbaj36uqBc251A21nM/BVA2xqH/z//TnOuS8bYHvVcs5tIE2PDalfOoOSGjOzXOBnwHXOuZedc0udc28558Y65x5LWny9c+6rpKleP+jN7Idm9oyZfWVm68zs32bWK2mZpmZ2q5ktTZwBfmJmvzOzjsDfE4utSPylPyXxPd9f4ktcGvvazHKS1vuImT1bkzrMrBAfEneUn10m3q90Bmdmp5nZu4lal5nZDWZmFeYvMbNhZjbRzNaY2XIzu7qafTQAeDvx5SeJ7XU0sxFm9t/kZSteeitfxszOMrOPzWytmT2dfMZpZudXqPlrM3uovNbEIo8ntrukqu0k3htkZh+Z2ebEvxclzXeJ38XjiX38iZlFepYr4SmgJBXFielkM9spdDFVaAU8D/wSOAx/tvekmR1YYZmHgPOA3wMH4c8Ei4BlQN/EMj/CX2q7ooptPA60SWwDADNrBfQBptWwjtOA5cDNie3sXtUPY2ZHJLb3JHAIcB1wPXBZ0qJXAu8CXYHbgTFm1q2qdeIvp52QeH1kYtvLtrNsVToCZwKnAscBhwO3VKh5EDARfwZ9KNATKA++/5f496LEdsu/3oaZnQrcC4wDDgbuBsabWe+kRW8EnsHv4xnAg2a2dwo/i8Sdc06TphpP+A/x74CNwBv4+xc/TlqmENjM1kArny6tsIwDTq9i/R0T8/KrmFcI3Jtivf8EhiVe759Y9wnbWbYgMb9d0vtT8JfDyr9+Epha4ev+wGpgp5rUkfh6CTCkuu0D04G/JS0zAlietJ5Hk5b5sOK2qqglP7Gdjknr/W/ScgOA4qRlNgJtKrx3A/BRha+XA7dVs+1Kv/cqtvMP4MEqfgevJa1ndIWvGwPrgf6h/49oim7SGZSkxDk3C9gD6I0/SzgK+KeZJd9vmgF0SZqm12dtZtbSzMaY2Xtmtipx2SgfKP+r+nCgjK2X8mprGnCKmbVIfN0PmOWc21jDOmrqIPyHdUWvAXua2c4V3nsnaZkv8PcG68NSt+2l2u+3ZWa74RtdvFzHbWzv5+6c9N73P7dzrgRYQf393BKAGklIyhIfxC8mppvN7M/ACDMb6/yNfoDVzrmParH6NYl/21QxLxd/prI9Y/GXr4bgzyLWAw8DTWtRR3XmAiVAHzN7GegBHN/AdVQchmBLFfNS/eOzDLCk95pUsVwU26qt5KEXQtYiDUC/TInCe/g/dup8X8o59x2wEjii4vuJM4b9gEXVfPtPgYedc7Occ+/gLzf9sML8Bfhj/tjtfH95uOZsZ355jZvw94b64e/HfIW//FjTOsq3Ve12gPeBo5Pe+yn+Et/aHXxvqlYAeRUbYODPemvMOfcN8DnQvZrFtlD7n/u9VOqR9KczKKkxM2uL/2B+EH95ZS3+0tU1wMvOuTUVFm9hZj9IWsXmRACV62hmXZKW+QS4C7jOzL7A3+dqCwzHf4g+Xk2Ji4FTzewZ/AfhTVQITefcYjObCfzZzK4A/g10wN+LmQosxf8VfpKZzQY2OOe29wDpNPylrP/B3wMqq2kdCUuAn5nZNGCTc25lFdu4E3jL/IO0j+AbFVxF5eb7USgEdgWGmn9erQCo9JxaDdwC/NHMvsafabYAujvn7kzMXwJ0N7NX8D/3qirWcQe+pd984H/xZ6P98I1LJJuEvgmmKX0moBlwK/AWsAp/6epDfKDsWmG5QvwHffKUfJO7qqkX/i/sy/EhWIw/A3mMCjf1t1PfPsBLwLrE9wwB5gBTkn6GMfi/9DcBHwOXVZg/HPgSf8lrSuK9KVRoJJF4z/Aftg44tBZ1/AT4D77RgUu8V0BSIw38h/K7+DOuZfhGCVZh/hIqN7YopJrGJFTRSCLx/iB8SK9L7O8rqNxIotqGFIn3foM/2yl/ruvBCvN6J46ZLcCSatZxMf45uy2Jfy9Kml9VY4tK+0JTek+W+MWKiIjEiu5BiYhILCmgREQklhRQIiISSwooERGJpeDNzNu1a+c6duwYuoxK1q1bR8uWLUOXkXa031KzaNEiSktL6dw5uZMEqU5cj7PiYli8GJyDDh0gLy90RVvFdZ8BzJ8/f6Vzrn3y+8EDqmPHjsybNy90GZUUFhZSUFAQuoy0o/2WmoKCAoqKimL5fyDO4nicLV4M3br5cLrsMrjnHrDkvjkCiuM+K2dmS6t6X5f4RETqaMUK6NkTvvsOevWCcePiFU7pSgElIlIHGzZAnz7w8cfQtSs8+ijk7KgzJ6kRBZSISC2VlcH558Mbb8Bee8GcOdCqVeiqMocCSkSkloYOhccfh9atYe5c2L3KoSeltiINKDObZmZfJoaeXmxmF0a5fhGRuHjgAbj9dn8574kn4JBDQleUeaI+gxqN74ByZ+BkYFRi2GoRkYzx17/CJZf41/ffD8cdF7aeTBVpQDnnFjo/Vg5s7Z06eRwcEZG09c478KtfQWkpXH89XKjrRPUm8uegzGw8vvv85sDbwHNVLDMQGAiQl5dHYWFh1GXUWXFxcSzrijvtt9QUFRVRWlqqfZaiUMfZypVNufTSrqxduxPHHvsNPXq8R7r86tLx/2a9DLdhZjlAN/z4Nrc755KHZv5efn6+i+NDinF+qC3OtN9SU/6g7oIFC0KXklZCHGfFxfDzn8Pbb8PRR8NLL8FOdR5DuuHE+f+mmc13zuUnv18vrficc6XOudfwo5VeUh/bEBFpKCUlcNZZPpz22w+efjq9wild1Xcz88boHpSIpDHn4IorfDPytm3hueegXbvQVWWHyALKzHYzs7PMrJWZ5ZjZ8cDZwMtRbUNEpKH98Y8wfjw0berPnPbfP3RF2SPKRhIOfznvfnzwLQUGO+eejXAbIiIN5qmnYMgQ//qhh+CnPw1bT7aJLKCccyuAY6Jan4hISG++Cf36+Ut8t97q70FJw1JXRyIiST79FHr39h3B/uY3cN11oSvKTgooEZEKVq3yQ2esWAG//CVMmKChM0JRQImIJGzeDH37wgcfwMEH+45gmzQJXVX2UkCJiODvNV10Efz97/CDH/hm5W3ahK4quymgRESAm2+Ghx+GFi38uE577x26IlFAiUjWmzoVRoyARo3gscfgCI3BEAsKKBHJaoWFvqUewLhxvvWexIMCSkSy1vvvw6mnwpYtMHgwXH556IqkIgWUiGSlb76Bk06CoiLo0wfGjg1dkSRTQIlI1tmwAU4+2T+Qm58P06f7odslXhRQIpJVysrg3HN9V0b77AOzZ0PLlqGrkqoooEQkq1x7Lcya5Z9xmjvXP/Mk8aSAEpGsMWGCv9fUuLEPqR/9KHRFUh0FlIhkheeeg8su868feAC6dw9bj+yYAkpEMt6CBXDmmf7+07BhMGBA6IqkJhRQIpLRli/3zcmLi+Gcc3yXRpIeFFAikrHWrPHh9MUX8POfw4MPauiMdKKAEpGMVFLiL+u98w506uSHb2/WLHRVkgoFlIhkHOd8g4gXXoB27XwDiV13DV2VpEoBJSIZZ+xYmDjRnzE9+yz88IehK5LaUECJSEZ5/HG45hr/eupU6NYtbD1SewooEckYb7zhuzECuP12+NWvwtYjdaOAEpGM8PHHvgPYTZtg4EC4+urQFUldKaBEJO199x307AkrV8IJJ8B996k5eSZQQIlIWtu0CU45BRYvhkMPhRkzfF97kv4UUCKStpyDCy6AV1+FPfbwvZPvvHPoqiQqCigRSVs33QSPPOLHc5ozBzp0CF2RREkBJSJpacoUGDkSGjWCmTPh8MNDVyRRU0CJSNqZPz+Xiy7yr++91zeQkMyjgBKRtPLee3DTTQdTUgJXXQWXXBK6IqkvCigRSRtffeXPltata0zfvjBmTOiKpD4poEQkLaxbB717w9KlcNBBa5g61d9/kswV2a/XzJqZ2WQzW2pma81sgZmdGNX6RSR7lZZCv34wbx78z//ALbe8S/PmoauS+hbl3x+NgWXAMUAbYBgw08w6RrgNEclCQ4bAM89Abq5/1mmXXbaELkkaQGQB5Zxb55wb4Zxb4pwrc87NAT4FjohqGyKSfe69F8aNgyZN/KCDBx0UuiJpKPXWIYiZ5QGdgIVVzBsIDATIy8ujsLCwvsqoteLi4ljWFXfab6kpKiqitLRU+2w7Xn+9LcOHHwwYV1/9PvA1hYU6zmojHfeZOeeiX6lZE+B54GPn3KDqls3Pz3fz5s2LvIa6KiwspKCgIHQZaUf7LTUFBQUUFRWxYMGC0KXEzvz58POfw/r1MGKE7zWinI6z1MV5n5nZfOdcfvL7kbeBMbNGwFRgM3BZ1OsXkcz32WfQq5cPp/POgxtvDF2RhBDpJT4zM2AykAf0dM7pTqaIpGT1ajjpJP/M07HHwgMPaOiMbBX1PagJwEFAD+fchojXLSIZbssWPwruf/8LBx4Is2ZB06ahq5JQonwOah9gENAF+MrMihNTv6i2ISKZyznfbdGLL8Juu8Fzz8Euu4SuSkKK7AzKObcU0Im4iNTKbbfB5Mmw007w7LP+gVzJbuooRESCe+wxGDrU32uaPh1+/OPQFUkcKKBEJKjXXoMBA/zrsWPhtNOCliMxooASkWA+/BD69IFNm+DSS+HKK0NXJHGigBKRIFau9ENnfPed//fuu9WcXLalgBKRBrdxI5xyCnz0kR+qfcYMaFxvHa9JulJAiUiDKiuDX/8a/vEP6NAB5syBVq1CVyVxpIASkQY1bJhvtde6tR86Y489QlckcaWAEpEG8+c/w+jRkJMDjz8Ohx4auiKJMwWUiDSIF1+Eiy/2r8ePh+OPD1uPxJ8CSkTq3bvvwumn+6Hbr70WBg4MXZGkAwWUiNSrL77wvZOvWQNnnAG33hq6IkkXCigRqTfFxdC7NyxbBt26wZQp0EifOlJDOlREpF6UlsI558C//w0//CE88ww0bx66KkknCigRiZxzMHgwzJ4Nu+7qh85o3z50VZJuFFAiErm774Z77/WDDT79NHTqFLoiSUcKKBGJ1NNPw+9/71//5S/ws58FLUfSmAJKRCLz1lv+vpNzMGqUfy1SWwooEYnEkiW+xd6GDXDBBX4AQpG6UECJSJ0VFfkhM77+Grp3h/vv19AZUncKKBGpk82boW9feP996NwZnngCmjQJXZVkAgWUiNSaczBoEPztb/CDH/jm5Lm5oauSTKGAEpFau+UW3ztEixb+mad99gldkWQSBZSI1Mr06TB8uL/X9MgjkJ8fuiLJNAooEUnZ//2fb6kH8Mc/Qp8+YeuRzKSAEpGULFoEp5ziG0f87ndwxRWhK5JMpYASkRpbscI3J1+1Ck4+Ge66K3RFkskUUCJSIxs2+FD65BM44gh/3yknJ3RVkskUUCKyQ2VlcN558M9/wt57+xZ7LVuGrkoynQJKRHbo+uv9A7g77wxz58Luu4euSLKBAkpEqjVxIowZA40bw6xZcPDBoSuSbKGAEpHteuEF+O1v/euJE6FHj7D1SHZRQIlIlf7zH/jVr/zQ7TfcsPW5J5GGooASkUo+/xxOOgmKi+Hss2HkyNAVSTaKNKDM7DIzm2dmm8xsSpTrFpGGsXYt9OrlQ+qnP/Wj4mroDAmhccTr+wIYBRwPNI943SJSz0pK4MwzYcEC2H9/P3x7s2ahq5JsFWlAOeeeBDCzfKBDlOsWkfrlnO+66PnnoW1bP3RG27ahq5JsFvUZVI2Y2UBgIEBeXh6FhYUhyqhWcXFxLOuKO+231BQVFVFaWhqLfTZzZgcmTNiPJk3KGDFiAcuXr2H58tBVVU3HWerScZ8FCSjn3CRgEkB+fr4rKCgIUUa1CgsLiWNdcaf9lprc3FyKioqC77NZs/ww7QDTpjXijDO6Bq1nR3ScpS4d95la8YlkuX/+E/r395f4Ro+GM84IXZGIp4ASyWKffOI7gN24ES66CK69NnRFIltFeonPzBon1pkD5JjZTkCJc64kyu2ISN2tWuWfdVqxAo47Du67T83JJV6iPoMaBmwArgP6J14Pi3gbIlJHmzbBaafBBx/AIYfA449DkyahqxLZVtTNzEcAI6Jcp4hEyzl/Oa+w0PdKPneu76VcJG50D0oky/zhDzB1qh/Pac4c2Guv0BWJVE0BJZJFHn7YB1SjRjBjBnSNd2tyyXIKKJEs8fe/w4UX+tf33OMbSIjEmQJKJAu8/z6ceips2QJXXrl1jCeROFNAiWS4r7+Gnj1h9WofUnfcEboikZpRQIlksPXr/YO4S5bAkUfCtGmQkxO6KpGaUUCJZKjSUt+F0b/+BR07wrPPQosWoasSqTkFlEiGuuYaeOopaNPGP+uUlxe6IpHUKKBEMtD48XDXXb53iCefhM6dQ1ckkjoFlEiGmTsXLr/cv37gAfjFL8LWI1JbCiiRDPL2237I9rIyuPFGOP/80BWJ1J4CSiRDLFvmH75dt843jhgxInRFInWjgBLJAGvW+HD68ks45hj48581dIakPwWUSJrbssWPgvvuu3DAAb7lXrNmoasSqTsFlEgac853W/TXv0L79vDcc7DLLqGrEomGAkokjY0Z41vq7bSTfxB3331DVyQSHQWUSJqaOROuu87fa5o2DX7yk9AViURLASWShl5/Hc47z78eMwb69g1bj0h9UECJpJmPPoI+fWDTJrj4YrjqqtAVidQPBZRIGvn2Wz90xsqVcOKJ8Kc/qTm5ZC4FlEia2LTJj+f04Ydw2GF+yPbGjUNXJVJ/FFAiacA5uOACePVV2HNP399e69ahqxKpXwookTRw443wyCPQqpUPpz33DF2RSP1TQInE3IMPwqhRfiTcmTP95T2RbKCAEomxl16CQYP86/vu8w0jRLKFAkokphYu9M83lZTA1VdvDSqRbKGAEomhr77yzcnXrIHTT4fbbgtdkUjDU0CJxMy6ddCrF3z2me++6OGHoZH+p0oW0mEvEiOlpXDOOTB/vu/49ZlnoHnz0FWJhKGAEomRq67yvZLvsosfOmO33UJXJBKOAkokJu65B+6+G5o08YMOHnBA6IpEwlJAicTAs8/C4MH+9YMP+mHbRbJdpAFlZrua2VNmts7MlprZOVGuXyQTrV+fw9ln++6Mbr4Z+vcPXZFIPETd1eR9wGYgD+gCzDWz/zjnFka8HZGMsGkTfPppS0pKYMAAGDYsdEUi8WHOuWhWZNYSWAUc7JxbnHhvKvC5c+667X1f69at3RFHHBFJDVEqKioiNzc3dBlpR/stNf/4xwJKSiA3twuHHqqhM2pKx1nq4rzPXnnllfnOufzk96M8g+oElJSHU8J/gEpX081sIDAQoEmTJhQVFUVYRjRKS0tjWVfcab/V3KpVTSkp8a/32GMNq1eXhS0ojeg4S1067rMoA6oVsCbpvdVApUEBnHOTgEkA+fn5bt68eRGWEY3CwkIKCgpCl5F2tN9qZuVKOPBAgAI6dFjPwoX/Cl1SWtFxlro47zPbzqWDKBtJFAM7J723M7A2wm2IZISRI/3ouLm50Lbt5tDliMRSlAG1GGhsZvtXeO8wQA0kRCr45BOYMMHfb9pvv9DViMRXZAHlnFsHPAncbGYtzexooA8wNaptiGSCG26ALVvg3HOhZcvQ1YjEV9QP6l4KNAe+AR4FLlETc5Gt/vUveOwxaNbMX+YTke2L9Dko59x3wClRrlMkU5SVwe9+518PHgx77x20HJHYU1dHIg1k6lR4803YfXd/mU9EqqeAEmkAa9bAtdf612PGQOtKD1+ISDIFlEgDGDkSvv4aunWDfv1CVyOSHhRQIvXsgw9g3DjfrPxPf1J3RiI1pYASqUfOwZVXQkkJXHghxLDbSZHYUkCJ1KOZM+GFF6BNG7jlltDViKQXBZRIPVm5Ei6/3L++4w5o3z5sPSLpRgElUk+uvBJWrIBjj/WX90QkNQookXrw/PMwbRrstBNMmqSGESK1oYASidjatTBokH89cqQ6hBWpLQWUSMSuvx6WLfMt9gYPDl2NSPpSQIlE6IUX4L77oHFjmDzZ/ysitaOAEonIN9/AgAH+9R/+AIcdFrQckbSngBKJgHPwm9/47oyOOWZrv3siUnsKKJEIjB8Pc+b4IdynToWcnNAViaQ/BZRIHS1cCEOG+NcPPAB77RW2HpFMoYASqYPiYjjzTNi4ES64AE4/PXRFIplDASVSS+X3nRYuhAMPhLvvDl2RSGZRQInU0tixvjPY1q3h6aehVavQFYlkFgWUSC289BJcd51//fDDcMABYesRyUQKKJEULV0KZ50FZWVwww1wyimhKxLJTAookRSsWQMnnwzffgsnnOAfyBWR+qGAEqmhLVt8K7133oFOnWD6dD3vJFKfFFAiNeCc76H8xRf9wIPPPw+77hq6KpHMpoASqYGRI+Evf4HmzX2PEfvuG7oikcyngBLZgcmT4aaboFEjeOwxOPLI0BWJZAcFlEg1HnkELrrIv77nHt9AQkQahgJKZDueeALOO8/ff7rlFvjtb0NXJJJdFFAiVZg9G84+G0pLYfhwGDo0dEUi2UcBJZJk7lzfnLykxPdSrmedRMJQQIlU8OijvmeIzZvhsstgzBgwC12VSHZSQIkk3H8/9Ovnz5yuucY3ilA4iYSjgBIBbrsNLrnEN4gYPRpuv13hJBJaJAFlZpeZ2Twz22RmU6JYp0hDKCmByy+H66/3gTR+/NZeykUkrMYRrecLYBRwPNA8onWK1KvVq/1ouH/9KzRtCg895HspF5F4iCSgnHNPAphZPtAhinWK1KdPP4VeveC993zfek89BUcfHboqEakoqjOolJjZQGAgQF5eHoWFhSHKqFZxcXEs64q7dNhv8+fnMmpUZ4qKmrLPPusYPfpdtmzZSIiyi4qKKC0tjf0+i5t0OM7iJh33WZCAcs5NAiYB5Ofnu4KCghBlVKuwsJA41hV3cd5vZWVw661w442+McTxx8OMGS1p0+YnwWrKzc2lqKgotvssruJ8nMVVOu6zHTaSMLNCM3PbmV5riCJF6mrlSjjpJN8rBPiQmjsX2rQJW5eIbN8Oz6CccwUNUIdIvfnb3+D882H5cj+G0/TpfjRcEYm3qJqZNzaznYAcIMfMdjKzIJcPRcpt2ABXXgndu/tw+vGP4e23FU4i6SKqB3WHARuA64D+idfDIlq3SMrmz4cjjoBx4/yw7CNGwKuvwt57h65MRGoqqmbmI4ARUaxLpC6Ki30YjRvneyI/8ECYOhXy80NXJiKpUldHkjFmz4bOneHOO30rvcGD4d//VjiJpCvdJ5K099FHfliMZ57xX3ftCpMm+Ut8IpK+dAYlaWvVKvj97/1Z0zPPQKtW/tLem28qnEQygc6gJO1s2AATJ8KoUfDtt76T11//2n+9xx6hqxORqCigJG1s2gSTJ8Mtt8AXX/j3jjkG7rrLX9YTkcyigJLYW78epkzxYzR99pl/r0sXuPlm3+Grxm0SyUwKKImtb7/14zPdc4/vqgjgRz+CP/wBTj0VGukOqkhGU0BJ7Lz9th9+fdo0f/YEvqn4tdf6YMrJCVufiDQMBZTEwvr1MHOmD6Y339z6/gknwDXXQEGBLuWJZBsFlATjnH+QdupUP5ptUZF/PzfXd+46aBAcdFDICkUkJAWUNLjFi+HRR+GRR/zrckceCRdf7Idhb9EiXH0iEg8KKGkQH37oH6adMQPmzdv6/m67+UA6/3w9XCsi21JASb0oKYHXX/f9482eDYsWbZ3XujWcdhqccw784hfQWEehiFRBHw0SmU8/hblzd2fiRPjf/4Xvvts6LzcXevb0rfBOOgmaNw9WpoikCQWU1NqyZX6MpZdf9qPWLlkCcMD38/ffH3r39tPRR0OTJqEqFZF0pICSGtm0yT+f9MYbfnr9dfj8822X2WUXOPjgFZx5Znt69IADDqh6XSIiNaGAkkrWr4d33vGBVD698w5s3rztcm3aQLdu/j5S9+5w2GHw6qsLKSgoCFK3iGQWBVQW27DBN154/30/vfeenxYtgrKyyst37uwDqXw68EB1NyQi9UcBleE2bvT3hj79FD75xE8ffOADackS/7BsspwcOOQQOPxwP3Xt6s+O2rRp6OpFJJspoNJYWZnvRPXzz7edPvtsaxgl3yeqKCfHN2Q46KBtp86d1cpORMJTQMWMc77LnxUrtp1WroRvvtk2iL74ArZsqX59OTmw996w775bp/JQ2m8/aNq0QX4sEZGUKaDqQVkZrF3rg6aoCFav3vq6qmnVKj+0RHkQlZTUfFu77AJ77rnttNdeW8Nor730IKyIpKes+ehyzrdC27jRN5neuHHrVNXXb7+dx0cf+a/XrYPi4pr/u2FD3Wpt3Rrat6962mOPrUG0xx7qs05EMlfwgPrySxg+3J81bNmy/X+rm7e9ZcoDqTx0UlO3brRbtvRnN7m5O57atIF27fzUvj00a1anTYuIZARzVTXjasgCrLWD5F5CzwAuBdYDPav4rgGJaSVwehXzLwHOBJYB51bYlm8W3bLlVbRp05tGjRaxYsUgGjVim6lz52E0a3YIrVp9yVtvDSYnx7+fk+Ons866lcMPP4qlS1/n4YeHVpp/993j6Nq1Cy+99BKjRo2qVN3EiRM54IADmD17NnfeeWel+VOnTmWvvfZixowZTJgwodL8J554gnbt2jFlyhSmTJlSaf5zzz1HixYtGD9+PDNnzqw0v7CwEICxY8cyZ86cbeY1b96c559/HoCRI0fy8ssvbzO/bdu2zJo1C4Drr7+eN954Y5v5TZo04cUXXwRg8ODBLFiwYJv5nTp1YtKkSQAMHDiQxRW7Mwe6dOnCuHHjAOjfvz/Lly/fZn63bt0YPXo0AH379uXbb7/dZn737t0ZPnw4ACeeeCIbkk5ne/XqxZAhQwCqfF7rjDPO4NJLL2X9+vX07Fn52BswYAADBgxg5cqVnH565WPvkksu4cwzz2TZsmWce+65leZfddVV9O7dm0WLFjFo0CAWLFhASUkJ+fn5AAwbNowePXqwYMECBg8eXOn7b731Vo466ihef/11hg4dWmn+uHHj6NIl84+9fv368XlSC6AOHTowbdo0QMdeVcfecccdx9ChQ78/9pKFPPZeeeWV+c65/OTvCX4G1bSpv1RltnXq2tU/+FlW5of7rjjPDI47zvfnVlwMI0ZUnt+/P5xyir+nc8UVW4On3FVX+e53Fi3yYw4lGzYMGjd+n9zcXKr4PXHCCXDUUb43haefrjxfzwaJiNRd8DOo/Px8N6/i+AsxUVhYqB4RakH7LTUFBQUUFRVV+mtfqqfjLHVx3mdmVuUZlP7WFxGRWFJAiYhILCmgREQklhRQIiISSwooERGJpToHlJk1M7PJZrbUzNaa2QIzOzGK4kREJHtFcQbVGP9E7DFAG2AYMNPMOkawbhERyVJ1flDXObcOGFHhrTlm9im+e4gldV2/iIhkp8h7kjCzPKATsLCaZQYCAwHy8vK+7/4kToqLi2NZV9xpv6WmqKiI0tJS7bMU6ThLXTrus0h7kjCzJsDzwMfOuSo6EapMPUlkFu231KgnidrRcZa6OO+zWvckYWaFZua2M71WYblGwFRgM3BZpNWLiEjW2eElPudcwY6WMTMDJgN5QE/n3A7GeRUREaleVPegJuAHUOrhnKvjcH0iIiLRPAe1DzAI6AJ8ZWbFialfXdctIiLZK4pm5ksBi6AWERGR76mrIxERiSUFlIiIxFLwEXXNbAWwNGgRVWsHrAxdRBrSfkud9lnqtM9SF+d9to9zrn3ym8EDKq7MbF5VD45J9bTfUqd9ljrts9Sl4z7TJT4REYklBZSIiMSSAmr7JoUuIE1pv6VO+yx12mepS7t9pntQIiISSzqDEhGRWFJAiYhILCmgREQklhRQNWRm+5vZRjObFrqWODOzZmY22cyWmtlaM1tgZieGriuOzGxXM3vKzNYl9tc5oWuKMx1bdZOOn2EKqJq7D3grdBFpoDGwDDgGaAMMA2aaWceQRcXUffgBPvOAfsAEM/tR2JJiTcdW3aTdZ5gCqgbM7CygCHg5cCmx55xb55wb4Zxb4pwrc87NAT4FjghdW5yYWUugLzDcOVfsnHsNeBY4N2xl8aVjq/bS9TNMAbUDZrYzcDPw+9C1pCMzywM6AQtD1xIznYAS59ziCu/9B9AZVA3p2KqZdP4MU0Dt2EhgsnNueehC0o2ZNQGmAw855z4IXU/MtALWJL23GmgdoJa0o2MrJWn7GZbVAWVmhWbmtjO9ZmZdgB7AHwOXGhs72mcVlmsETMXfY7ksWMHxVQzsnPTezsDaALWkFR1bNZfun2F1HlE3nTnnCqqbb2aDgY7AZ2YG/q/eHDPr7JzrWt/1xdGO9hmA+Z01GX/zv6dzbkt915WGFgONzWx/59yHifcOQ5erqqVjK2UFpPFnmLo6qoaZtWDbv3KH4H/ZlzjnVgQpKg2Y2f1AF6CHc644cDmxZWaPAQ64EL+/ngOOcs4ppLZDx1Zq0v0zLKvPoHbEObceWF/+tZkVAxvT4RcbipntAwwCNgFfJf5qAxjknJserLB4uhR4EPgG+Bb/oaFw2g4dW6lL988wnUGJiEgsZXUjCRERiS8FlIiIxJICSkREYkkBJSIisaSAEhGRWFJAiYhILCmgREQklhRQIiISS/8f9QwjBRtWHJsAAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light",
      "image/png": {
       "width": 424,
       "height": 280
      }
     },
     "output_type": "display_data"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "By default, the SELU hyperparameters (`scale` and `alpha`) are tuned in such a way that the mean output of each neuron remains close to 0, and the standard deviation remains close to 1 (assuming the inputs are standardized with mean 0 and standard deviation 1 too). Using this activation function, even a 1,000 layer deep neural network preserves roughly mean 0 and standard deviation 1 across all layers, avoiding the exploding/vanishing gradients problem:",
   "metadata": {
    "cell_id": "00038-9ef25560-7b47-43a6-888b-38d1bc162eaa",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 97.171875
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00039-49373f84-efba-462e-8bea-243b981edab1",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "8724ea11",
    "execution_start": 1659354582912,
    "execution_millis": 3451,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 432.875
   },
   "source": "np.random.seed(42)\nZ = np.random.normal(size=(500, 100)) # standardized inputs\nfor layer in range(1000):\n    W = np.random.normal(size=(100, 100), scale=np.sqrt(1 / 100)) # LeCun initialization\n    Z = selu(np.dot(Z, W))\n    means = np.mean(Z, axis=0).mean()\n    stds = np.std(Z, axis=0).mean()\n    if layer % 100 == 0:\n        print(\"Layer {}: mean {:.2f}, std deviation {:.2f}\".format(layer, means, stds))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Layer 0: mean -0.00, std deviation 1.00\nLayer 100: mean 0.02, std deviation 0.96\nLayer 200: mean 0.01, std deviation 0.90\nLayer 300: mean -0.02, std deviation 0.92\nLayer 400: mean 0.05, std deviation 0.89\nLayer 500: mean 0.01, std deviation 0.93\nLayer 600: mean 0.02, std deviation 0.92\nLayer 700: mean -0.02, std deviation 0.90\nLayer 800: mean 0.05, std deviation 0.83\nLayer 900: mean 0.02, std deviation 1.00\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "Using SELU is easy:",
   "metadata": {
    "cell_id": "00040-91d01907-6e81-436a-bf0c-016a812bb72d",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00041-9dcca0a1-91e8-47c2-8da7-ccf8403732a2",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "c5f96f08",
    "execution_start": 1659354586373,
    "execution_millis": 17,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 131.1875,
    "deepnote_output_heights": [
     21.1875
    ]
   },
   "source": "keras.layers.Dense(10, activation=\"selu\",\n                   kernel_initializer=\"lecun_normal\")",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 25,
     "data": {
      "text/plain": "<keras.layers.core.dense.Dense at 0x7f9559696c10>"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "Let's create a neural net for Fashion MNIST with 100 hidden layers, using the SELU activation function:",
   "metadata": {
    "cell_id": "00042-6f754991-6143-4a18-b478-152ce0242f45",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00043-b69d8580-e583-42b3-bf9e-e408a7434304",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "8905d08a",
    "execution_start": 1659354586394,
    "execution_millis": 29,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "source": "np.random.seed(42)\ntf.random.set_seed(42)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00044-34ec33c3-aafe-4c0f-bc82-e535234d2a72",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "51bf21d5",
    "execution_start": 1659354586436,
    "execution_millis": 1060,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 202
   },
   "source": "model = keras.models.Sequential()\nmodel.add(keras.layers.Flatten(input_shape=[28, 28]))\nmodel.add(keras.layers.Dense(300, activation=\"selu\",\n                             kernel_initializer=\"lecun_normal\"))\nfor layer in range(99):\n    model.add(keras.layers.Dense(100, activation=\"selu\",\n                                 kernel_initializer=\"lecun_normal\"))\nmodel.add(keras.layers.Dense(10, activation=\"softmax\"))",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00045-9654741a-156f-4dd5-9dcc-e2e502e78ce6",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "b1938699",
    "execution_start": 1659354587500,
    "execution_millis": 5,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 112
   },
   "source": "model.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n              metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "Now let's train it. Do not forget to scale the inputs to mean 0 and standard deviation 1:",
   "metadata": {
    "cell_id": "00046-690b38a9-ff19-4259-bdcc-c04a7578bc57",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00047-8832747a-8c7b-4219-8aae-504a551e661d",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "e22bdb74",
    "execution_start": 1659354587512,
    "execution_millis": 660,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 148
   },
   "source": "pixel_means = X_train.mean(axis=0, keepdims=True)\npixel_stds = X_train.std(axis=0, keepdims=True)\nX_train_scaled = (X_train - pixel_means) / pixel_stds\nX_valid_scaled = (X_valid - pixel_means) / pixel_stds\nX_test_scaled = (X_test - pixel_means) / pixel_stds",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00048-21f08972-34e4-4b9f-ad0d-d9fda7ef8b2f",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "16c124d1",
    "execution_start": 1659354588193,
    "execution_millis": 156990,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 306.875
   },
   "source": "history = model.fit(X_train_scaled, y_train, epochs=5,\n                    validation_data=(X_valid_scaled, y_valid))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/5\n1719/1719 [==============================] - 35s 18ms/step - loss: 1.3126 - accuracy: 0.4951 - val_loss: 1.0706 - val_accuracy: 0.5906\nEpoch 2/5\n1719/1719 [==============================] - 30s 17ms/step - loss: 0.9123 - accuracy: 0.6536 - val_loss: 0.8169 - val_accuracy: 0.6684\nEpoch 3/5\n1719/1719 [==============================] - 31s 18ms/step - loss: 0.8725 - accuracy: 0.6685 - val_loss: 0.7552 - val_accuracy: 0.7070\nEpoch 4/5\n1719/1719 [==============================] - 31s 18ms/step - loss: 0.8291 - accuracy: 0.6922 - val_loss: 0.6890 - val_accuracy: 0.7562\nEpoch 5/5\n1719/1719 [==============================] - 29s 17ms/step - loss: 0.7144 - accuracy: 0.7379 - val_loss: 0.6470 - val_accuracy: 0.7704\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "Now look at what happens if we try to use the ReLU activation function instead:",
   "metadata": {
    "cell_id": "00049-403fe517-2a00-4cc8-88e0-d46fdfeeb505",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00050-9f571986-a4cc-4a40-8a16-eca59b16baf8",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "8905d08a",
    "execution_start": 1659354745163,
    "execution_millis": 6,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "source": "np.random.seed(42)\ntf.random.set_seed(42)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00051-04f3eb43-5ee6-406c-8efc-abeb6630e865",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "9e618bc5",
    "execution_start": 1659354745180,
    "execution_millis": 1196,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 166
   },
   "source": "model = keras.models.Sequential()\nmodel.add(keras.layers.Flatten(input_shape=[28, 28]))\nmodel.add(keras.layers.Dense(300, activation=\"relu\", kernel_initializer=\"he_normal\"))\nfor layer in range(99):\n    model.add(keras.layers.Dense(100, activation=\"relu\", kernel_initializer=\"he_normal\"))\nmodel.add(keras.layers.Dense(10, activation=\"softmax\"))",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00052-3aa92fb9-d470-4449-a9b8-2e3bd46a93b0",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "b1938699",
    "execution_start": 1659354746389,
    "execution_millis": 7,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 112
   },
   "source": "model.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n              metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00053-2f510055-5edc-415b-9138-7a2b281d8e31",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "16c124d1",
    "execution_start": 1659354746414,
    "execution_millis": 138380,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 306.875
   },
   "source": "history = model.fit(X_train_scaled, y_train, epochs=5,\n                    validation_data=(X_valid_scaled, y_valid))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/5\n1719/1719 [==============================] - 30s 16ms/step - loss: 1.8088 - accuracy: 0.2686 - val_loss: 1.7779 - val_accuracy: 0.2616\nEpoch 2/5\n1719/1719 [==============================] - 26s 15ms/step - loss: 1.1340 - accuracy: 0.5239 - val_loss: 1.0905 - val_accuracy: 0.5316\nEpoch 3/5\n1719/1719 [==============================] - 28s 16ms/step - loss: 1.1595 - accuracy: 0.5107 - val_loss: 0.9522 - val_accuracy: 0.5894\nEpoch 4/5\n1719/1719 [==============================] - 26s 15ms/step - loss: 0.9180 - accuracy: 0.6209 - val_loss: 0.8417 - val_accuracy: 0.6406\nEpoch 5/5\n1719/1719 [==============================] - 27s 16ms/step - loss: 0.8130 - accuracy: 0.6737 - val_loss: 0.7211 - val_accuracy: 0.7272\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "Not great at all, we suffered from the vanishing/exploding gradients problem.",
   "metadata": {
    "cell_id": "00054-0456dcb8-847a-4397-9b83-033f28b846d5",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "markdown",
   "source": "# Batch Normalization",
   "metadata": {
    "cell_id": "00055-e21a5ade-05d0-4468-b52a-3bf1be075df7",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 82
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00056-869a3830-0f91-43b5-9508-fd7e4bf5fec4",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "1b705dd1",
    "execution_start": 1659354884366,
    "execution_millis": 111,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 220
   },
   "source": "model = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dense(300, activation=\"relu\"),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dense(100, activation=\"relu\"),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dense(10, activation=\"softmax\")\n])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00057-9d129414-c578-4851-9e83-9fcd6b257a89",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "4e6a3b95",
    "execution_start": 1659354884486,
    "execution_millis": 312,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 611.875
   },
   "source": "model.summary()",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Model: \"sequential_4\"\n_________________________________________________________________\n Layer (type)                Output Shape              Param #   \n=================================================================\n flatten_4 (Flatten)         (None, 784)               0         \n                                                                 \n batch_normalization (BatchN  (None, 784)              3136      \n ormalization)                                                   \n                                                                 \n dense_212 (Dense)           (None, 300)               235500    \n                                                                 \n batch_normalization_1 (Batc  (None, 300)              1200      \n hNormalization)                                                 \n                                                                 \n dense_213 (Dense)           (None, 100)               30100     \n                                                                 \n batch_normalization_2 (Batc  (None, 100)              400       \n hNormalization)                                                 \n                                                                 \n dense_214 (Dense)           (None, 10)                1010      \n                                                                 \n=================================================================\nTotal params: 271,346\nTrainable params: 268,978\nNon-trainable params: 2,368\n_________________________________________________________________\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00058-5064c024-9839-4d89-b6a3-b2c9a11da98a",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "2fed604",
    "execution_start": 1659354884601,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 188.75,
    "deepnote_output_heights": [
     78.75
    ]
   },
   "source": "bn1 = model.layers[1]\n[(var.name, var.trainable) for var in bn1.variables]",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 37,
     "data": {
      "text/plain": "[('batch_normalization/gamma:0', True),\n ('batch_normalization/beta:0', True),\n ('batch_normalization/moving_mean:0', False),\n ('batch_normalization/moving_variance:0', False)]"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00059-d4f67237-03ab-4f2d-ac77-d10a420476f6",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d50434ac",
    "execution_start": 1659354884664,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "#bn1.updates #deprecated",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00060-20b33420-5e54-40d2-8563-305afd243e13",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "b1938699",
    "execution_start": 1659354884664,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 112
   },
   "source": "model.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n              metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00061-1d42f336-4a12-475b-b3bb-e9a2927a115d",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "9d2223f7",
    "execution_start": 1659354884665,
    "execution_millis": 143007,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 508.75
   },
   "source": "history = model.fit(X_train, y_train, epochs=10,\n                    validation_data=(X_valid, y_valid))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/10\n1719/1719 [==============================] - 10s 5ms/step - loss: 0.8750 - accuracy: 0.7123 - val_loss: 0.5524 - val_accuracy: 0.8230\nEpoch 2/10\n1719/1719 [==============================] - 9s 5ms/step - loss: 0.5753 - accuracy: 0.8031 - val_loss: 0.4725 - val_accuracy: 0.8468\nEpoch 3/10\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.5189 - accuracy: 0.8206 - val_loss: 0.4375 - val_accuracy: 0.8552\nEpoch 4/10\n1719/1719 [==============================] - 9s 5ms/step - loss: 0.4826 - accuracy: 0.8325 - val_loss: 0.4151 - val_accuracy: 0.8606\nEpoch 5/10\n1719/1719 [==============================] - 9s 5ms/step - loss: 0.4565 - accuracy: 0.8408 - val_loss: 0.3995 - val_accuracy: 0.8644\nEpoch 6/10\n1719/1719 [==============================] - 9s 5ms/step - loss: 0.4398 - accuracy: 0.8472 - val_loss: 0.3866 - val_accuracy: 0.8688\nEpoch 7/10\n1719/1719 [==============================] - 9s 6ms/step - loss: 0.4242 - accuracy: 0.8511 - val_loss: 0.3762 - val_accuracy: 0.8704\nEpoch 8/10\n1719/1719 [==============================] - 9s 5ms/step - loss: 0.4143 - accuracy: 0.8539 - val_loss: 0.3711 - val_accuracy: 0.8736\nEpoch 9/10\n1719/1719 [==============================] - 9s 6ms/step - loss: 0.4023 - accuracy: 0.8580 - val_loss: 0.3630 - val_accuracy: 0.8752\nEpoch 10/10\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.3915 - accuracy: 0.8626 - val_loss: 0.3572 - val_accuracy: 0.8756\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "Sometimes applying BN before the activation function works better (there's a debate on this topic). Moreover, the layer before a `BatchNormalization` layer does not need to have bias terms, since the `BatchNormalization` layer has some as well, it would be a waste of parameters, so you can set `use_bias=False` when creating those layers:",
   "metadata": {
    "cell_id": "00062-3211cdc0-7551-4228-b56f-41c30842deee",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 97.171875
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00063-9c5df926-83d5-463e-a457-d19586899c32",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "83239de",
    "execution_start": 1659355027672,
    "execution_millis": 23,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 256
   },
   "source": "model = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dense(300, use_bias=False),\n    keras.layers.BatchNormalization(),\n    keras.layers.Activation(\"relu\"),\n    keras.layers.Dense(100, use_bias=False),\n    keras.layers.BatchNormalization(),\n    keras.layers.Activation(\"relu\"),\n    keras.layers.Dense(10, activation=\"softmax\")\n])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00064-7e65673b-0f5a-44a9-826f-639c41ab7a1f",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "b1938699",
    "execution_start": 1659355027707,
    "execution_millis": 0,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 112
   },
   "source": "model.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n              metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00065-8a2d470f-6be0-4838-a407-f6ea342bbecd",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "9d2223f7",
    "execution_start": 1659355027716,
    "execution_millis": 94005,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 508.75
   },
   "source": "history = model.fit(X_train, y_train, epochs=10,\n                    validation_data=(X_valid, y_valid))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/10\n1719/1719 [==============================] - 10s 5ms/step - loss: 1.0317 - accuracy: 0.6757 - val_loss: 0.6767 - val_accuracy: 0.7812\nEpoch 2/10\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.6790 - accuracy: 0.7793 - val_loss: 0.5566 - val_accuracy: 0.8180\nEpoch 3/10\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.5960 - accuracy: 0.8037 - val_loss: 0.5007 - val_accuracy: 0.8362\nEpoch 4/10\n1719/1719 [==============================] - 9s 5ms/step - loss: 0.5447 - accuracy: 0.8192 - val_loss: 0.4666 - val_accuracy: 0.8448\nEpoch 5/10\n1719/1719 [==============================] - 9s 5ms/step - loss: 0.5109 - accuracy: 0.8279 - val_loss: 0.4434 - val_accuracy: 0.8536\nEpoch 6/10\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.4898 - accuracy: 0.8338 - val_loss: 0.4262 - val_accuracy: 0.8546\nEpoch 7/10\n1719/1719 [==============================] - 9s 5ms/step - loss: 0.4712 - accuracy: 0.8396 - val_loss: 0.4130 - val_accuracy: 0.8568\nEpoch 8/10\n1719/1719 [==============================] - 9s 5ms/step - loss: 0.4560 - accuracy: 0.8439 - val_loss: 0.4035 - val_accuracy: 0.8610\nEpoch 9/10\n1719/1719 [==============================] - 9s 5ms/step - loss: 0.4441 - accuracy: 0.8475 - val_loss: 0.3943 - val_accuracy: 0.8638\nEpoch 10/10\n1719/1719 [==============================] - 9s 5ms/step - loss: 0.4333 - accuracy: 0.8505 - val_loss: 0.3875 - val_accuracy: 0.8664\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "## Gradient Clipping",
   "metadata": {
    "cell_id": "00066-28764b1a-17e5-4a9c-931c-1fbf1faa8fb7",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "markdown",
   "source": "All Keras optimizers accept `clipnorm` or `clipvalue` arguments:",
   "metadata": {
    "cell_id": "00067-e4ef0f44-e5a5-4ac8-bb00-5150a0c8734c",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00068-9c742259-6cc9-42e8-93f6-fbcb0cbbc761",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "9ff1f795",
    "execution_start": 1659355121721,
    "execution_millis": 5,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "optimizer = keras.optimizers.SGD(clipvalue=1.0)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00069-e1c36548-bb2a-4883-8167-2a65a5486d11",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "25280eb0",
    "execution_start": 1659355121727,
    "execution_millis": 11,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "optimizer = keras.optimizers.SGD(clipnorm=1.0)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "## Reusing Pretrained Layers",
   "metadata": {
    "cell_id": "00070-8ba7c429-de3e-43b1-8400-4d2d79f0c485",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "markdown",
   "source": "### Reusing a Keras model",
   "metadata": {
    "cell_id": "00071-9c90f5cc-1db3-418d-8e82-6f95cdbe7d7b",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 62
   }
  },
  {
   "cell_type": "markdown",
   "source": "Let's split the fashion MNIST training set in two:\n* `X_train_A`: all images of all items except for sandals and shirts (classes 5 and 6).\n* `X_train_B`: a much smaller training set of just the first 200 images of sandals or shirts.\n\nThe validation set and the test set are also split this way, but without restricting the number of images.\n\nWe will train a model on set A (classification task with 8 classes), and try to reuse it to tackle set B (binary classification). We hope to transfer a little bit of knowledge from task A to task B, since classes in set A (sneakers, ankle boots, coats, t-shirts, etc.) are somewhat similar to classes in set B (sandals and shirts). However, since we are using `Dense` layers, only patterns that occur at the same location can be reused (in contrast, convolutional layers will transfer much better, since learned patterns can be detected anywhere on the image, as we will see in the CNN chapter).",
   "metadata": {
    "cell_id": "00072-ff2d5d86-c400-423b-98df-dbeac7c95f6e",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 254.125
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00073-390ba1bd-5e12-4d20-bb48-996f4a528e20",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d0756ea",
    "execution_start": 1659355121742,
    "execution_millis": 300,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 292
   },
   "source": "def split_dataset(X, y):\n    y_5_or_6 = (y == 5) | (y == 6) # sandals or shirts\n    y_A = y[~y_5_or_6]\n    y_A[y_A > 6] -= 2 # class indices 7, 8, 9 should be moved to 5, 6, 7\n    y_B = (y[y_5_or_6] == 6).astype(np.float32) # binary classification task: is it a shirt (class 6)?\n    return ((X[~y_5_or_6], y_A),\n            (X[y_5_or_6], y_B))\n\n(X_train_A, y_train_A), (X_train_B, y_train_B) = split_dataset(X_train, y_train)\n(X_valid_A, y_valid_A), (X_valid_B, y_valid_B) = split_dataset(X_valid, y_valid)\n(X_test_A, y_test_A), (X_test_B, y_test_B) = split_dataset(X_test, y_test)\nX_train_B = X_train_B[:200]\ny_train_B = y_train_B[:200]",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00074-12437ca1-061d-4563-89c4-96e8e8385d05",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "22b2e7c1",
    "execution_start": 1659355122058,
    "execution_millis": 16,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 113.1875,
    "deepnote_output_heights": [
     21.1875
    ]
   },
   "source": "X_train_A.shape",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 47,
     "data": {
      "text/plain": "(43986, 28, 28)"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00075-e97dca75-988c-4398-9122-2e4c8cb76222",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "fbb0d290",
    "execution_start": 1659355122084,
    "execution_millis": 13,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 113.1875,
    "deepnote_output_heights": [
     21.1875
    ]
   },
   "source": "X_train_B.shape",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 48,
     "data": {
      "text/plain": "(200, 28, 28)"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00076-95c77667-7e58-4333-81f8-e5f250db4228",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "7fad8294",
    "execution_start": 1659355122115,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 132.375,
    "deepnote_output_heights": [
     40.375
    ]
   },
   "source": "y_train_A[:30]",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 49,
     "data": {
      "text/plain": "array([4, 0, 5, 7, 7, 7, 4, 4, 3, 4, 0, 1, 6, 3, 4, 3, 2, 6, 5, 3, 4, 5,\n       1, 3, 4, 2, 0, 6, 7, 1], dtype=uint8)"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00077-693977fc-4822-4473-a41b-b4c61d2006f1",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "5810ddf5",
    "execution_start": 1659355122169,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 132.375,
    "deepnote_output_heights": [
     40.375
    ]
   },
   "source": "y_train_B[:30]",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 50,
     "data": {
      "text/plain": "array([1., 1., 0., 0., 0., 0., 1., 1., 1., 0., 0., 1., 1., 0., 0., 0., 0.,\n       0., 0., 1., 1., 0., 0., 1., 1., 0., 1., 1., 1., 1.], dtype=float32)"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00078-8ddeb8c4-518c-4107-bb37-fde32a4778de",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d690901b",
    "execution_start": 1659355122171,
    "execution_millis": 1,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "source": "tf.random.set_seed(42)\nnp.random.seed(42)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00079-eea32128-e45a-4c32-8ba7-a787728dd06c",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "16ab0d30",
    "execution_start": 1659355122214,
    "execution_millis": 5,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 148
   },
   "source": "model_A = keras.models.Sequential()\nmodel_A.add(keras.layers.Flatten(input_shape=[28, 28]))\nfor n_hidden in (300, 100, 50, 50, 50):\n    model_A.add(keras.layers.Dense(n_hidden, activation=\"selu\"))\nmodel_A.add(keras.layers.Dense(8, activation=\"softmax\"))",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00080-cd0a977a-77b2-4831-90eb-f5016da870af",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "66101eb4",
    "execution_start": 1659355122229,
    "execution_millis": 11,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 112
   },
   "source": "model_A.compile(loss=\"sparse_categorical_crossentropy\",\n                optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n                metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00081-07912f64-8985-4713-bb23-a56658b8c9c4",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "30bf0d4b",
    "execution_start": 1659355122248,
    "execution_millis": 152202,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 714
   },
   "source": "history = model_A.fit(X_train_A, y_train_A, epochs=20,\n                    validation_data=(X_valid_A, y_valid_A))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/20\n1375/1375 [==============================] - 6s 4ms/step - loss: 0.5926 - accuracy: 0.8103 - val_loss: 0.3890 - val_accuracy: 0.8677\nEpoch 2/20\n1375/1375 [==============================] - 6s 4ms/step - loss: 0.3523 - accuracy: 0.8786 - val_loss: 0.3291 - val_accuracy: 0.8822\nEpoch 3/20\n1375/1375 [==============================] - 6s 4ms/step - loss: 0.3170 - accuracy: 0.8894 - val_loss: 0.3013 - val_accuracy: 0.8999\nEpoch 4/20\n1375/1375 [==============================] - 5s 3ms/step - loss: 0.2972 - accuracy: 0.8975 - val_loss: 0.2894 - val_accuracy: 0.9018\nEpoch 5/20\n1375/1375 [==============================] - 5s 4ms/step - loss: 0.2834 - accuracy: 0.9023 - val_loss: 0.2773 - val_accuracy: 0.9066\nEpoch 6/20\n1375/1375 [==============================] - 6s 4ms/step - loss: 0.2729 - accuracy: 0.9061 - val_loss: 0.2733 - val_accuracy: 0.9076\nEpoch 7/20\n1375/1375 [==============================] - 5s 4ms/step - loss: 0.2641 - accuracy: 0.9091 - val_loss: 0.2720 - val_accuracy: 0.9088\nEpoch 8/20\n1375/1375 [==============================] - 5s 4ms/step - loss: 0.2573 - accuracy: 0.9125 - val_loss: 0.2590 - val_accuracy: 0.9138\nEpoch 9/20\n1375/1375 [==============================] - 7s 5ms/step - loss: 0.2518 - accuracy: 0.9138 - val_loss: 0.2561 - val_accuracy: 0.9150\nEpoch 10/20\n1375/1375 [==============================] - 11s 8ms/step - loss: 0.2468 - accuracy: 0.9154 - val_loss: 0.2540 - val_accuracy: 0.9155\nEpoch 11/20\n1375/1375 [==============================] - 12s 9ms/step - loss: 0.2422 - accuracy: 0.9175 - val_loss: 0.2497 - val_accuracy: 0.9153\nEpoch 12/20\n1375/1375 [==============================] - 9s 6ms/step - loss: 0.2382 - accuracy: 0.9188 - val_loss: 0.2513 - val_accuracy: 0.9126\nEpoch 13/20\n1375/1375 [==============================] - 8s 6ms/step - loss: 0.2350 - accuracy: 0.9197 - val_loss: 0.2447 - val_accuracy: 0.9160\nEpoch 14/20\n1375/1375 [==============================] - 11s 8ms/step - loss: 0.2315 - accuracy: 0.9213 - val_loss: 0.2415 - val_accuracy: 0.9175\nEpoch 15/20\n1375/1375 [==============================] - 8s 5ms/step - loss: 0.2287 - accuracy: 0.9212 - val_loss: 0.2448 - val_accuracy: 0.9195\nEpoch 16/20\n1375/1375 [==============================] - 13s 9ms/step - loss: 0.2254 - accuracy: 0.9226 - val_loss: 0.2388 - val_accuracy: 0.9195\nEpoch 17/20\n1375/1375 [==============================] - 11s 8ms/step - loss: 0.2230 - accuracy: 0.9232 - val_loss: 0.2412 - val_accuracy: 0.9170\nEpoch 18/20\n1375/1375 [==============================] - 6s 4ms/step - loss: 0.2200 - accuracy: 0.9245 - val_loss: 0.2427 - val_accuracy: 0.9150\nEpoch 19/20\n1375/1375 [==============================] - 7s 5ms/step - loss: 0.2177 - accuracy: 0.9251 - val_loss: 0.2331 - val_accuracy: 0.9205\nEpoch 20/20\n1375/1375 [==============================] - 6s 4ms/step - loss: 0.2155 - accuracy: 0.9262 - val_loss: 0.2330 - val_accuracy: 0.9200\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00082-deb098e3-2378-4757-8cef-b715bc73a320",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "ea8abdd9",
    "execution_start": 1659355274451,
    "execution_millis": 51,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "model_A.save(\"my_model_A.h5\")",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00083-5acf6e69-cefb-493c-8fd1-711387f43035",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d11678ed",
    "execution_start": 1659355274507,
    "execution_millis": 88,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 148
   },
   "source": "model_B = keras.models.Sequential()\nmodel_B.add(keras.layers.Flatten(input_shape=[28, 28]))\nfor n_hidden in (300, 100, 50, 50, 50):\n    model_B.add(keras.layers.Dense(n_hidden, activation=\"selu\"))\nmodel_B.add(keras.layers.Dense(1, activation=\"sigmoid\"))",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00084-1d0c1db2-4497-4a46-b39b-66151824f106",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "cc682955",
    "execution_start": 1659355274600,
    "execution_millis": 13,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 112
   },
   "source": "model_B.compile(loss=\"binary_crossentropy\",\n                optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n                metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00085-e1d5e180-a5b6-4e0f-946f-7f0eb4e7c350",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d1e4da5d",
    "execution_start": 1659355274624,
    "execution_millis": 5982,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 714
   },
   "source": "history = model_B.fit(X_train_B, y_train_B, epochs=20,\n                      validation_data=(X_valid_B, y_valid_B))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/20\n7/7 [==============================] - 1s 74ms/step - loss: 0.9573 - accuracy: 0.4650 - val_loss: 0.6314 - val_accuracy: 0.6004\nEpoch 2/20\n7/7 [==============================] - 0s 34ms/step - loss: 0.5692 - accuracy: 0.7450 - val_loss: 0.4784 - val_accuracy: 0.8529\nEpoch 3/20\n7/7 [==============================] - 0s 47ms/step - loss: 0.4503 - accuracy: 0.8650 - val_loss: 0.4102 - val_accuracy: 0.8945\nEpoch 4/20\n7/7 [==============================] - 0s 39ms/step - loss: 0.3879 - accuracy: 0.8950 - val_loss: 0.3647 - val_accuracy: 0.9178\nEpoch 5/20\n7/7 [==============================] - 0s 34ms/step - loss: 0.3435 - accuracy: 0.9250 - val_loss: 0.3300 - val_accuracy: 0.9320\nEpoch 6/20\n7/7 [==============================] - 0s 22ms/step - loss: 0.3081 - accuracy: 0.9300 - val_loss: 0.3019 - val_accuracy: 0.9402\nEpoch 7/20\n7/7 [==============================] - 0s 28ms/step - loss: 0.2800 - accuracy: 0.9350 - val_loss: 0.2804 - val_accuracy: 0.9422\nEpoch 8/20\n7/7 [==============================] - 0s 31ms/step - loss: 0.2564 - accuracy: 0.9450 - val_loss: 0.2606 - val_accuracy: 0.9473\nEpoch 9/20\n7/7 [==============================] - 0s 22ms/step - loss: 0.2362 - accuracy: 0.9550 - val_loss: 0.2428 - val_accuracy: 0.9523\nEpoch 10/20\n7/7 [==============================] - 0s 27ms/step - loss: 0.2188 - accuracy: 0.9600 - val_loss: 0.2281 - val_accuracy: 0.9544\nEpoch 11/20\n7/7 [==============================] - 0s 67ms/step - loss: 0.2036 - accuracy: 0.9700 - val_loss: 0.2150 - val_accuracy: 0.9584\nEpoch 12/20\n7/7 [==============================] - 0s 47ms/step - loss: 0.1898 - accuracy: 0.9700 - val_loss: 0.2036 - val_accuracy: 0.9584\nEpoch 13/20\n7/7 [==============================] - 0s 38ms/step - loss: 0.1773 - accuracy: 0.9750 - val_loss: 0.1931 - val_accuracy: 0.9615\nEpoch 14/20\n7/7 [==============================] - 0s 36ms/step - loss: 0.1668 - accuracy: 0.9800 - val_loss: 0.1838 - val_accuracy: 0.9635\nEpoch 15/20\n7/7 [==============================] - 0s 35ms/step - loss: 0.1570 - accuracy: 0.9900 - val_loss: 0.1746 - val_accuracy: 0.9686\nEpoch 16/20\n7/7 [==============================] - 0s 37ms/step - loss: 0.1481 - accuracy: 0.9900 - val_loss: 0.1674 - val_accuracy: 0.9686\nEpoch 17/20\n7/7 [==============================] - 0s 34ms/step - loss: 0.1406 - accuracy: 0.9900 - val_loss: 0.1604 - val_accuracy: 0.9706\nEpoch 18/20\n7/7 [==============================] - 0s 35ms/step - loss: 0.1334 - accuracy: 0.9900 - val_loss: 0.1539 - val_accuracy: 0.9706\nEpoch 19/20\n7/7 [==============================] - 0s 37ms/step - loss: 0.1268 - accuracy: 0.9900 - val_loss: 0.1482 - val_accuracy: 0.9716\nEpoch 20/20\n7/7 [==============================] - 0s 33ms/step - loss: 0.1208 - accuracy: 0.9900 - val_loss: 0.1431 - val_accuracy: 0.9716\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00086-a9166540-a6d9-4f0f-9c2c-41bf4b152205",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "a88bc509",
    "execution_start": 1659355280611,
    "execution_millis": 27,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 551.3125
   },
   "source": "model_B.summary()",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Model: \"sequential_7\"\n_________________________________________________________________\n Layer (type)                Output Shape              Param #   \n=================================================================\n flatten_7 (Flatten)         (None, 784)               0         \n                                                                 \n dense_224 (Dense)           (None, 300)               235500    \n                                                                 \n dense_225 (Dense)           (None, 100)               30100     \n                                                                 \n dense_226 (Dense)           (None, 50)                5050      \n                                                                 \n dense_227 (Dense)           (None, 50)                2550      \n                                                                 \n dense_228 (Dense)           (None, 50)                2550      \n                                                                 \n dense_229 (Dense)           (None, 1)                 51        \n                                                                 \n=================================================================\nTotal params: 275,801\nTrainable params: 275,801\nNon-trainable params: 0\n_________________________________________________________________\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00087-79c5be6f-d28a-4739-9742-0a6cdcc8b733",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "924be521",
    "execution_start": 1659355280770,
    "execution_millis": 85,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 112
   },
   "source": "model_A = keras.models.load_model(\"my_model_A.h5\")\nmodel_B_on_A = keras.models.Sequential(model_A.layers[:-1])\nmodel_B_on_A.add(keras.layers.Dense(1, activation=\"sigmoid\"))",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "Note that `model_B_on_A` and `model_A` actually share layers now, so when we train one, it will update both models. If we want to avoid that, we need to build `model_B_on_A` on top of a *clone* of `model_A`:",
   "metadata": {
    "cell_id": "00088-5513e478-0eaf-4eeb-a0a6-8b8b1188ed24",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 74.78125
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00089-9afb6fa8-adfd-4e98-9628-63f2c5544a93",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d7ed2fc8",
    "execution_start": 1659355280864,
    "execution_millis": 140,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 130
   },
   "source": "model_A_clone = keras.models.clone_model(model_A)\nmodel_A_clone.set_weights(model_A.get_weights())\nmodel_B_on_A = keras.models.Sequential(model_A_clone.layers[:-1])\nmodel_B_on_A.add(keras.layers.Dense(1, activation=\"sigmoid\"))",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00090-edcc54cf-a890-4978-bb47-863cf3cc690e",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d5ccaaa9",
    "execution_start": 1659355281006,
    "execution_millis": 15,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 166
   },
   "source": "for layer in model_B_on_A.layers[:-1]:\n    layer.trainable = False\n\nmodel_B_on_A.compile(loss=\"binary_crossentropy\",\n                     optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n                     metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00091-664ce718-1275-4856-9a4b-89540525a743",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "a5f6e1d7",
    "execution_start": 1659355281023,
    "execution_millis": 9243,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 876
   },
   "source": "history = model_B_on_A.fit(X_train_B, y_train_B, epochs=4,\n                           validation_data=(X_valid_B, y_valid_B))\n\nfor layer in model_B_on_A.layers[:-1]:\n    layer.trainable = True\n\nmodel_B_on_A.compile(loss=\"binary_crossentropy\",\n                     optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n                     metrics=[\"accuracy\"])\nhistory = model_B_on_A.fit(X_train_B, y_train_B, epochs=16,\n                           validation_data=(X_valid_B, y_valid_B))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/4\n7/7 [==============================] - 1s 131ms/step - loss: 0.2654 - accuracy: 0.9400 - val_loss: 0.2794 - val_accuracy: 0.9280\nEpoch 2/4\n7/7 [==============================] - 0s 46ms/step - loss: 0.2556 - accuracy: 0.9400 - val_loss: 0.2698 - val_accuracy: 0.9331\nEpoch 3/4\n7/7 [==============================] - 0s 45ms/step - loss: 0.2462 - accuracy: 0.9400 - val_loss: 0.2610 - val_accuracy: 0.9351\nEpoch 4/4\n7/7 [==============================] - 0s 51ms/step - loss: 0.2376 - accuracy: 0.9400 - val_loss: 0.2528 - val_accuracy: 0.9361\nEpoch 1/16\n7/7 [==============================] - 1s 103ms/step - loss: 0.2128 - accuracy: 0.9450 - val_loss: 0.2049 - val_accuracy: 0.9645\nEpoch 2/16\n7/7 [==============================] - 0s 77ms/step - loss: 0.1703 - accuracy: 0.9550 - val_loss: 0.1723 - val_accuracy: 0.9716\nEpoch 3/16\n7/7 [==============================] - 0s 41ms/step - loss: 0.1411 - accuracy: 0.9650 - val_loss: 0.1495 - val_accuracy: 0.9817\nEpoch 4/16\n7/7 [==============================] - 0s 40ms/step - loss: 0.1200 - accuracy: 0.9800 - val_loss: 0.1329 - val_accuracy: 0.9817\nEpoch 5/16\n7/7 [==============================] - 0s 68ms/step - loss: 0.1049 - accuracy: 0.9900 - val_loss: 0.1204 - val_accuracy: 0.9838\nEpoch 6/16\n7/7 [==============================] - 0s 44ms/step - loss: 0.0932 - accuracy: 0.9950 - val_loss: 0.1105 - val_accuracy: 0.9858\nEpoch 7/16\n7/7 [==============================] - 0s 41ms/step - loss: 0.0840 - accuracy: 0.9950 - val_loss: 0.1023 - val_accuracy: 0.9858\nEpoch 8/16\n7/7 [==============================] - 0s 42ms/step - loss: 0.0764 - accuracy: 0.9950 - val_loss: 0.0955 - val_accuracy: 0.9868\nEpoch 9/16\n7/7 [==============================] - 0s 31ms/step - loss: 0.0700 - accuracy: 0.9950 - val_loss: 0.0894 - val_accuracy: 0.9868\nEpoch 10/16\n7/7 [==============================] - 0s 31ms/step - loss: 0.0642 - accuracy: 0.9950 - val_loss: 0.0846 - val_accuracy: 0.9878\nEpoch 11/16\n7/7 [==============================] - 0s 32ms/step - loss: 0.0597 - accuracy: 1.0000 - val_loss: 0.0802 - val_accuracy: 0.9868\nEpoch 12/16\n7/7 [==============================] - 0s 17ms/step - loss: 0.0555 - accuracy: 1.0000 - val_loss: 0.0763 - val_accuracy: 0.9878\nEpoch 13/16\n7/7 [==============================] - 0s 16ms/step - loss: 0.0518 - accuracy: 1.0000 - val_loss: 0.0730 - val_accuracy: 0.9878\nEpoch 14/16\n7/7 [==============================] - 0s 16ms/step - loss: 0.0486 - accuracy: 1.0000 - val_loss: 0.0702 - val_accuracy: 0.9878\nEpoch 15/16\n7/7 [==============================] - 0s 16ms/step - loss: 0.0459 - accuracy: 1.0000 - val_loss: 0.0677 - val_accuracy: 0.9878\nEpoch 16/16\n7/7 [==============================] - 0s 16ms/step - loss: 0.0435 - accuracy: 1.0000 - val_loss: 0.0653 - val_accuracy: 0.9878\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "So, what's the final verdict?",
   "metadata": {
    "cell_id": "00092-87425082-4f0c-4572-aabb-5cf44400922c",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00093-e8d52fa2-7733-4ebd-8493-7ae9095e303b",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "a4df2056",
    "execution_start": 1659355290273,
    "execution_millis": 597,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 159.375,
    "deepnote_output_heights": [
     null,
     21.1875
    ]
   },
   "source": "model_B.evaluate(X_test_B, y_test_B)",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "63/63 [==============================] - 0s 2ms/step - loss: 0.1408 - accuracy: 0.9705\n",
     "output_type": "stream"
    },
    {
     "output_type": "execute_result",
     "execution_count": 64,
     "data": {
      "text/plain": "[0.1408407986164093, 0.9704999923706055]"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00094-46e6ae8c-f3f7-4994-ad1a-db8fc2f582ca",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "cd723a9f",
    "execution_start": 1659355290641,
    "execution_millis": 255,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 159.375,
    "deepnote_output_heights": [
     null,
     21.1875
    ]
   },
   "source": "model_B_on_A.evaluate(X_test_B, y_test_B)",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "63/63 [==============================] - 0s 2ms/step - loss: 0.0562 - accuracy: 0.9940\n",
     "output_type": "stream"
    },
    {
     "output_type": "execute_result",
     "execution_count": 65,
     "data": {
      "text/plain": "[0.056181780993938446, 0.9940000176429749]"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "Great! We got quite a bit of transfer: the error rate dropped by a factor of 4.9!",
   "metadata": {
    "cell_id": "00095-61e26b76-b973-4825-b0f5-9d905ace9028",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00096-2bae5a4d-04ca-438c-9e26-423b2ef86994",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "32551fb6",
    "execution_start": 1659355290854,
    "execution_millis": 43,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 113.1875,
    "deepnote_output_heights": [
     21.1875
    ]
   },
   "source": "(100 - 97.05) / (100 - 99.40)",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 66,
     "data": {
      "text/plain": "4.916666666666718"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "# Faster Optimizers",
   "metadata": {
    "cell_id": "00097-616e0013-1a1a-439f-898e-fb46f2e60520",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 82
   }
  },
  {
   "cell_type": "markdown",
   "source": "## Momentum optimization",
   "metadata": {
    "cell_id": "00098-35b78e8b-fb3c-4232-9ce6-5edd25a0a13a",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00099-6ce61cb7-18eb-465c-86a5-14b1a6168d63",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "365c883a",
    "execution_start": 1659355290855,
    "execution_millis": 0,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "optimizer = keras.optimizers.SGD(learning_rate=0.001, momentum=0.9)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "## Nesterov Accelerated Gradient",
   "metadata": {
    "cell_id": "00100-37fe5f6b-271c-43ed-aec6-99a85abaa1a5",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00101-1f34d88e-1068-4424-a46f-2c8021268a8a",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "77ebd4c2",
    "execution_start": 1659355290856,
    "execution_millis": 1,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "optimizer = keras.optimizers.SGD(learning_rate=0.001, momentum=0.9, nesterov=True)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "## AdaGrad",
   "metadata": {
    "cell_id": "00102-cf004f78-b730-46c4-ac96-93a773b0c4a2",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00103-c282d956-4d62-4bee-86b5-fa5643c64712",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "62181912",
    "execution_start": 1659355290860,
    "execution_millis": 2,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "optimizer = keras.optimizers.Adagrad(learning_rate=0.001)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "## RMSProp",
   "metadata": {
    "cell_id": "00104-df3e469e-84ca-4006-a230-dc18d4fd06d5",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00105-0823aa08-5989-46c4-9566-4ee0f101235a",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "bec957f3",
    "execution_start": 1659355290889,
    "execution_millis": 0,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "optimizer = keras.optimizers.RMSprop(learning_rate=0.001, rho=0.9)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "## Adam Optimization",
   "metadata": {
    "cell_id": "00106-1ffe9d5c-762a-4ffc-9ff9-e00bd11f9721",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00107-4b29499d-670a-4c8f-89d4-e1b7e96bb97f",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "ee763982",
    "execution_start": 1659355290890,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "optimizer = keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "## Adamax Optimization",
   "metadata": {
    "cell_id": "00108-8bb8b68f-2c4f-4515-b70f-fc4a801d418a",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00109-9bcbfedc-c5c6-4137-ac62-bf95a27e3d66",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "c0578b04",
    "execution_start": 1659355290891,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "optimizer = keras.optimizers.Adamax(learning_rate=0.001, beta_1=0.9, beta_2=0.999)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "## Nadam Optimization",
   "metadata": {
    "cell_id": "00110-b73058cc-d87e-4327-94b9-788458e41d56",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00111-c4114d01-6305-4ced-907e-ee9dc8447b1a",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d92e086c",
    "execution_start": 1659355290893,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "optimizer = keras.optimizers.Nadam(learning_rate=0.001, beta_1=0.9, beta_2=0.999)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "## Learning Rate Scheduling",
   "metadata": {
    "cell_id": "00112-7670e5c1-acb5-4581-b4b5-511b5f1381b7",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "markdown",
   "source": "### Power Scheduling",
   "metadata": {
    "cell_id": "00113-12b290f0-3d70-4cf3-a7b3-94ac65437dd1",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 62
   }
  },
  {
   "cell_type": "markdown",
   "source": "$$\\eta(t) = \\dfrac{\\eta_0}{\\left(1 + \\dfrac{t}{s}\\right)^c}$$\n\n* Keras uses $c=1$ and `s = 1 / decay`",
   "metadata": {
    "cell_id": "00114-6a27dd4a-0c94-486e-8458-2e3f15cb24ff",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 138.859375
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00115-1f776f8f-6f34-43aa-9864-2070d71538df",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "5fbabd3",
    "execution_start": 1659355290896,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "optimizer = keras.optimizers.SGD(learning_rate=0.01, decay=1e-4)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00116-2a1d7118-c9c9-4518-92ea-8b85348b1da5",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "b2ffe852",
    "execution_start": 1659355290949,
    "execution_millis": 45,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 184
   },
   "source": "model = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.Dense(10, activation=\"softmax\")\n])\nmodel.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00117-07d97011-cece-48ca-909a-1cbf19aaa51a",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "8938db83",
    "execution_start": 1659355290994,
    "execution_millis": 262527,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 732
   },
   "source": "n_epochs = 25\nhistory = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n                    validation_data=(X_valid_scaled, y_valid))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/25\n1719/1719 [==============================] - 10s 5ms/step - loss: 0.4898 - accuracy: 0.8265 - val_loss: 0.4063 - val_accuracy: 0.8610\nEpoch 2/25\n1719/1719 [==============================] - 13s 8ms/step - loss: 0.3793 - accuracy: 0.8652 - val_loss: 0.3731 - val_accuracy: 0.8718\nEpoch 3/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.3467 - accuracy: 0.8777 - val_loss: 0.3745 - val_accuracy: 0.8722\nEpoch 4/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.3258 - accuracy: 0.8847 - val_loss: 0.3511 - val_accuracy: 0.8786\nEpoch 5/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.3107 - accuracy: 0.8896 - val_loss: 0.3448 - val_accuracy: 0.8786\nEpoch 6/25\n1719/1719 [==============================] - 8s 5ms/step - loss: 0.2972 - accuracy: 0.8939 - val_loss: 0.3417 - val_accuracy: 0.8844\nEpoch 7/25\n1719/1719 [==============================] - 13s 7ms/step - loss: 0.2871 - accuracy: 0.8980 - val_loss: 0.3379 - val_accuracy: 0.8826\nEpoch 8/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.2779 - accuracy: 0.9011 - val_loss: 0.3421 - val_accuracy: 0.8798\nEpoch 9/25\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.2698 - accuracy: 0.9030 - val_loss: 0.3290 - val_accuracy: 0.8848\nEpoch 10/25\n1719/1719 [==============================] - 16s 9ms/step - loss: 0.2624 - accuracy: 0.9059 - val_loss: 0.3283 - val_accuracy: 0.8862\nEpoch 11/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.2564 - accuracy: 0.9091 - val_loss: 0.3265 - val_accuracy: 0.8878\nEpoch 12/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2504 - accuracy: 0.9111 - val_loss: 0.3338 - val_accuracy: 0.8802\nEpoch 13/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2448 - accuracy: 0.9133 - val_loss: 0.3247 - val_accuracy: 0.8898\nEpoch 14/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2407 - accuracy: 0.9143 - val_loss: 0.3289 - val_accuracy: 0.8864\nEpoch 15/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2354 - accuracy: 0.9165 - val_loss: 0.3233 - val_accuracy: 0.8880\nEpoch 16/25\n1719/1719 [==============================] - 8s 5ms/step - loss: 0.2314 - accuracy: 0.9184 - val_loss: 0.3209 - val_accuracy: 0.8912\nEpoch 17/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2276 - accuracy: 0.9187 - val_loss: 0.3248 - val_accuracy: 0.8892\nEpoch 18/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2235 - accuracy: 0.9215 - val_loss: 0.3195 - val_accuracy: 0.8912\nEpoch 19/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2202 - accuracy: 0.9227 - val_loss: 0.3232 - val_accuracy: 0.8904\nEpoch 20/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2169 - accuracy: 0.9238 - val_loss: 0.3207 - val_accuracy: 0.8898\nEpoch 21/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.2133 - accuracy: 0.9255 - val_loss: 0.3204 - val_accuracy: 0.8888\nEpoch 22/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.2107 - accuracy: 0.9267 - val_loss: 0.3187 - val_accuracy: 0.8882\nEpoch 23/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2078 - accuracy: 0.9272 - val_loss: 0.3207 - val_accuracy: 0.8912\nEpoch 24/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2050 - accuracy: 0.9291 - val_loss: 0.3208 - val_accuracy: 0.8898\nEpoch 25/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.2027 - accuracy: 0.9288 - val_loss: 0.3201 - val_accuracy: 0.8896\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00118-4affc36f-e71f-44ca-9217-8890d0e57886",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "87c5cfc9",
    "execution_start": 1659355553522,
    "execution_millis": 155,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 650,
    "deepnote_output_heights": [
     288
    ]
   },
   "source": "import math\n\nlearning_rate = 0.01\ndecay = 1e-4\nbatch_size = 32\nn_steps_per_epoch = math.ceil(len(X_train) / batch_size)\nepochs = np.arange(n_epochs)\nlrs = learning_rate / (1 + decay * epochs * n_steps_per_epoch)\n\nplt.plot(epochs, lrs,  \"o-\")\nplt.axis([0, n_epochs - 1, 0, 0.01])\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Learning Rate\")\nplt.title(\"Power Scheduling\", fontsize=14)\nplt.grid(True)\nplt.show()",
   "execution_count": null,
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEeCAYAAAC30gOQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAA1QUlEQVR4nO3deXxU5dn/8c+VBRIIIbJDZFMEWWSpa0UrdW8rBbXLU5dqtWq1tv7cUau11lbR0sfWWpfHKnWt2qK407pQRVxQkV1wYwv7FghJSAjX749zgsMwk0wwM5Nkvu/Xa15k7nOfM9ecprm8z72ZuyMiItLYstIdgIiItExKMCIikhRKMCIikhRKMCIikhRKMCIikhRKMCIikhRKMCLNhJmdbWZlSbr2XDO7sYHnLDazK+K9F1GCkWbFzCaamYevajP73Mz+YGZt0x1bfcysr5k9YmbLzWybma0wsxfMbES6Y2skBwN/TXcQ0nTkpDsAkT3wCnAmkAscCdwPtAUuTGdQtcws192ro8uA/wCfAT8ASoAewPFAh5QHmQTuvjbdMUjTohaMNEfb3H2Vuy9z98eAR4GxAGbW2szuMLPVZlZpZu+Y2RG1J4bvx0W8fyRsDXUL37cJWxdHhO/NzK4ys8/MrMLM5pjZGRHn9wnP/5GZvWZmFcAFMWIeDOwL/Nzdp7v7End/291/4+6vRlyvvZndbWYrw/gXmNkPIy9kZseEj7S2mtnrZtY36vhoM/sgPP8LM/udmbWKON7FzCaH32eJmZ0THWz4nb4XVVbnI7AYj8zczM43s6fCWD+PvHdhnUPN7MMw1plm9u3wvFHxPkeaDyUYaQkqCFozALcBPwTOAUYAc4CXzax7eHwqMCri3KOAdRFlhwPbgffC9zcD5wI/BwYBtwD3mtl3omK4heDx0CDgmRgxrgV2AKeaWcwnB2ZmwIthTD8Jr3UZUBVRrTVwTfj9vg4UAfdEXOMEgoT7F4Kkdg7wPeD3EdeYCPQDjiVIzD8G+sSKqRHcAEwGhgFPAA+YWa8w1gLgeeBj4EDgKuD2JMUh6eDueunVbF4Efxyfj3h/CEGCeILgMVkV8OOI49kEj6VuDt+fCJQRPB7uB2wmSCL3hsdvBl4Jf25LkLyOjIrhDuDF8Oc+gAOXJxD7z4Gt4ef/F/gtMDji+HEESWhgnPPPDj9rQETZ6cA2wML3bwDXR503NvxMA/qH1xgZcbw3UAPcGFHmwPeirrMYuKIB7x24JeJ9DlAOnBG+vwDYAORH1DktPG9Uun/X9PrqL7VgpDk60czKzKwSeJvgj+ovCB5B5QJv1VZ095qwzqCwaBpBK+BgglbLNII+nVHh8VEErRzCc/IIWkBltS+Cvp59o2J6v76g3f0uoBvBH9FpwBjgIzM7M6wyAljp7gvquMw2d18Y8X4F0ArYK3x/IHBdVLyPESTLbsBAgiRW20LD3ZeE10mG2RGfs52gJdclLNofmOvuFRH1301SHJIG6uSX5ugN4HygGljhYYd6bT9KHMF/UruXmdkHwDcJEsjrwDtALzPrR5B4avtoav8DbDSwNOp61VHvtyYSuLtvAZ4FnjWzXwFTCFoyDydyPsHju10uGRVrFvAb4KkY50Z2wte3jLoTtHgi5caqWI/o++To0XzGUIKR5qjc3T+NUf4ZwSOykeHPmFk2QV/FYxH1phIkmP2BP7l7pZm9C1zHrv0v8wkeP/V299ca+0u4u5vZx8DXwqKZQHczG1hPK6YuHwL7x7k/hJ+XRfBocXpY1otgRFuktUD3iPO6Rr5vJB8DZ5lZfkQr5pBG/gxJIyUYaTHcfauZ3Q2MN7N1wBfApUBXdp2fMRW4nKA/4MOIsuuA/7p7VXi9LWb2B+APYQf8G0ABcBiww93vSzQ2MxtO0LJ4mCBxVRF05p8DPB5We5XgEdG/zOxSYBFBP1Fbd38mwY+6CXjezJYATxIkzCHAIe5+lbsvNLOXCQYqnE/Qx/TH8N9IrwE/N7PpBP0zvwcqE/2+CXqMoM/r/8zs9wRJ7trwmDaqagHUVJWW5mqCDv8HgY+AocCJ7r4yos608N83wz4aCBJMDl/2v9S6HrgRuAKYRzCX5VSC5NUQy4HPCUZVvRPGdjnwB4L+I9x9B/Atgj6kR4AFwJ8I+lgS4u5TgO8QtNDeC1/j2PUR39lh/K8BzxH8oV8cdanLw3inAv8kmGu0JtE4Eox1C8Hjx8EErbfbCe41NH4ykzSoHXkiIpJ2ZjYGeBro4u7r0h2PfDV6RCYiaWNmZxG0lJYRPMq7A3hOyaVlSOkjMjPrYGZPh7N6l5jZaXHqmZmNN7P14Wt8+Ay89vh9ZrbQzHaY2dkxzr/UzFaZ2WYze8DMWifxa4nInutK0C+1ELgLeAk4o84zpNlIdR/MXQSdm10JJojdbWaDY9Q7n2By2DCCZ+ij2XX5jVnARXzZQbtTOJN5HHAMwQSyfQg6V0WkiXH329y9j7u3dvfe7n5R2DcjLUDK+mAsWO12IzDE3ReFZQ8DJe4+LqrudGBi7SgdMzsXOM/dD4uqNw24390nRpQ9Bix292vD98cAj7p7XXMkRESkkaWyD6Y/sL02uYRmEQzVjDY4PBZZL1ZLJ5bBBGsfRZ7b1cw6uvv6yIrhMM3zAbLyCw/Mad9l57E+hRpgB7Bjxw6ysnQvoum+7E73JLaWfl8WLVq0zt07xzqWygRTQLDuU6RSoF2cuqVR9QrMzLz+Jlescwk/Z5cEE7aQ7gNo3X0/737WHQAUF+Xz1rij6/mYzDB16lRGjRqV7jCaHN2X3emexNbS70s45yqmVKbVMqAwqqwQiPW8NbpuIVCWQHKJdy5xPmc3eblZXHnCgESqiohIHVKZYBYBOWa2X0TZMILJa9HmhcfqqxdLrHNXRz8ei2fM8B6MHVGc4EeJiEg8KUsw7r4VmATcZGZtzWwkwWqysRb5ewi4zMyKzawHwaziibUHzayVmeURLMaXa2Z5ZpYVce65ZjbIzIqAX0WeG0+fwiwGdS/ko6WlaPKpiMhXl+qep4uAfIIlJx4HLnT3eWZ2ZLiseK17CZawmAPMBV4Iy2r9m2DtpMMJ+lAqgG8AuPvLBJtOvU6wPMYS4NeJBHfOEX1ZuHoL0z9LqLEjIiJ1SOlMfnffQLi1bVT5mwSd87XvnWB3u6viXGdUPZ/zR4IF/Bpk9LDu3PrSAh6Y9gUj+3Vq6OkiIhKh5Y6d2wOtc7I547DevPrxGr5Yl9D2HiIiEocSTJTTD+1Nq+wsJr7V0MVyRUQkkhJMlM7tWjN6WA+e+mA5pRXRm/GJiEiilGBi+MnIPpRX1fDkjGXpDkVEpNlSgolhSHF7Du3bgYnTF7O9Zke6wxERaZaUYOI454i+lGyq4D/zV6c7FBGRZkkJJo5jB3alZ4d8HlBnv4jIHlGCiSM7yzj78L7MWLyR2cs3pTscEZFmRwmmDj84aG8KWufw4FuL0x2KiEizowRTh3Z5uXz/oL15fvYKVm+uTHc4IiLNihJMPc4+vA/bdziPvBN3ywMREYlBCaYevTu25diBXXn03aVUVtekOxwRkWZDCSYB54zsy4atVUz+qCTdoYiINBtKMAk4bJ8ODOxeyAPTFmuvGBGRBCnBJMDMOGdkH+0VIyLSAEowCRo9rAedClrxwDRNvBQRSYQSTILycrM5/VDtFSMikiglmAY4/bBe2itGRCRBSjAN0KVdnvaKERFJkBJMA2mvGBGRxCjBNJD2ihERSYwSzB7QXjEiIvVTgtkDxw7sSoe2uVzyj4/oO+4FRt76Gs/M1Cx/EZFIOekOoDl6btYKtlRup7ommNVfsqmCaybNAWDsiOJ0hiYi0mSoBbMHbp+ycGdyqVVRXcPtUxamKSIRkaZHCWYPrNhU0aByEZFMpASzB3oU5TeoXEQkEynB7IErTxhAfm72LmW52caVJwxIU0QiIk2POvn3QG1H/u1TFrJiUwU52UabVtkcP7hrmiMTEWk6lGD20NgRxTsTzfuLN/C9e97mr69/xhVqxYiIAHpE1igO6tOBscN7cN8bn7NkvVZaFhEBJZhGM+5bA8nJNn77/IJ0hyIi0iQowTSSbu3z+MXR+/HKgtVMXbgm3eGIiKSdEkwjOueIPvTt1Jabnp9P1XYthCkimS2lCcbMOpjZ02a21cyWmNlpceqZmY03s/Xha7yZWcTx4Wb2gZmVh/8OjzjW2szuMbPVZrbBzJ4zs5Ss39I6J5sbThrE52u3MnG6NiUTkcyW6hbMXUAV0BU4HbjbzAbHqHc+MBYYBgwFRgMXAJhZK2Ay8AiwF/B3YHJYDnAJ8PXwvB7ARuDO5Hyd3X1z/y4cvX8X/vTKJ6zZXJmqjxURaXJSlmDMrC1wKnC9u5e5+zTgWeDMGNXPAia4+3J3LwEmAGeHx0YRDK++w923ufufAQOODo/3Baa4+2p3rwSeAGIlsaS54aRBVNc4t778cSo/VkSkSUnlPJj+wHZ3XxRRNgs4KkbdweGxyHqDI47NdvfI1SZnh+UvA38D/mRmPYBNBC2ll2IFZGbnE7SW6Ny5M1OnTm3YN6rDcb2ymfRhCYNy19Nvr+z6T2iiysrKGvW+tBS6L7vTPYktk+9LKhNMAbA5qqwUaBenbmlUvYKwHyb6WPR1PgGWASVADTAHuDhWQO5+H3AfwIABA3zUqFEJfpX6Hfz17bw/YSqTl7fmmTEjyc6y+k9qgqZOnUpj3peWQvdld7onsWXyfUllH0wZUBhVVghsSaBuIVAWtlrqu85dQGugI9AWmEScFkwytW2dw7XfHsicklKeen9Zqj9eRCTtUplgFgE5ZrZfRNkwYF6MuvPCY7HqzQOGRo4qI+jQrz0+HJjo7hvcfRtBB/8hZtbpq3+FhvnusB4c3GcvbpuykNLy6lR/vIhIWqUswbj7VoLWxE1m1tbMRgJjgIdjVH8IuMzMisO+lMuBieGxqQSPvn4ZDkmuffz1WvjvDODHZtbezHKBi4AV7r4uGd+rLmbGjd8dzKbyKv73lUX1nyAi0oKkepjyRUA+sAZ4HLjQ3eeZ2ZFmVhZR717gOYL+k7nAC2EZ7l5FMIT5xwSd+OcAY8NygCuASoK+mLXAt4GTk/qt6jC4R3t+dEgvHn5nCQtXxXoaKCLSMqV0NWV330CQHKLL3yTovK9978BV4SvWdWYCB8Y5tp5g5FiTccXxA3h+9kpufHYej513KLs+3RMRaZm0VEwK7NW2FVcc35+3P1/PS3NXpTscEZGUUIJJkdMO7c3A7oX87oUFVFTVpDscEZGkU4JJkews48bRgyjZVMEhv3+FvuNeYOStr/HMzJJ0hyYikhTa0TKFVpZWkm3GlsrtAJRsquCaSXOAL7dhFhFpKdSCSaHbpyykZpcVbqCiuobbpyxMU0QiIsmjBJNCKzZVNKhcRKQ5U4JJoR5F+Q0qFxFpzpRgUujKEwaQn7vrysoGXDRq3/QEJCKSREowKTR2RDG3nHIAxUX5GNC5XWuyDF5ZsJodO7ze80VEmhONIkuxsSOKdxkx9tDbi7lh8jzueeMzLhrVL42RiYg0LrVg0uzMw3pz0tDu/GHKQt75fH26wxERaTRKMGlmZtx66lD6dGzLLx6fydot29IdkohIo0g4wZjZt8zseTObb2Y9w7KfmtkxyQsvMxS0zuGvZ3yNLZXVXPKPmdSoP0ZEWoCEEoyZnQ48SbAEfl8gNzyUTZwVj6Vh9u9WyG/HDGH6Z+v5k/aOEZEWINEWzFXAee5+KbA9ovwdgh0kpRF8/6CefP/Avbnz9U/576K16Q5HROQrSTTB7Ae8HaO8DChsvHDkpjFDGNC1Hf/vHzM1w19EmrVEE8wKoH+M8m8AnzVeOJLfKpu7Tv8aVdt3cPFjH1JdsyPdIYmI7JFEE8x9wJ/NbGT4vqeZnQXcBtydlMgy2L6dC7j11KF8uHQT41/6ON3hiIjskYQmWrr7bWbWHvgPkAe8DmwD/uDudyUxvow1elgPZizewP3TvuCgPh04cUi3dIckItIgCQ9TdvfrgE7AIcBhQGd3vz5ZgQlc952BDNu7PVf+cxZL15enOxwRkQZJqAVjZg8Al7j7FuD9iPK2wJ3ufk6S4storXOy+ctpX+M7f36T0/7vbXZ4sGlZj6J8rjxhgDYpE5EmLdEWzFlArDXl84EfN144Eq1nhzb84KCeLN9UyYrSSpwvd8LUdssi0pTVmWDMrIOZdSRYVX6v8H3tqzNwErA6FYFmspfmrtqtTDthikhTV98jsnWAh6/5MY478OvGDkp2pZ0wRaQ5qi/BfJOg9fIacCqwIeJYFbDE3VckKTYJ9SjKpyRGMtFOmCLSlNWZYNz9vwBm1hdY5u6a9ZcGV54wgGsmzaGiumaX8h8evHeaIhIRqV+i82CWAJhZD6AX0Crq+BuNH5rUqh0tdvuUhazYVEHXwjyqamp44K3FHD+4G/t302o9ItL0JDpMuQfwGMHSME7w2CxyTfnsWOdJ44neCXPZhnK+f8/bnHH/ezx5wWHs07kgjdGJiOwu0WHKdwA1wCCgHDgS+D6wADgxKZFJnXp2aMMjPz0Ud+f0+99l2QZNxBSRpiXRBHMUcLW7f0zQclnr7pOAq4HfJis4qVu/LgU88tNDKa+q4fT732VVaWW6QxIR2SnRBJNPMGQZgpFkXcKf5wNDGzsoSdzA7oX8/ZxDWF+2jdPvf4f1ZdpyWUSahkQTzMfA/uHPHwE/M7PewM8BTSdPs+E9i3jg7IMp2VTBmX97j9Ly6nSHJCKScIL5E1C7nO9NwPHA58BFwLVJiEsa6NB9OnLvmQfx6ZoyznrwPcq2ba//JBGRJEoowbj7o+4+Mfz5Q6APcDDQy92fSvTDwiVmnjazrWa2xMxOi1PPzGy8ma0PX+PNzCKODzezD8ysPPx3eNT5XzOzN8yszMxWm9klicbYnB3VvzN/OW0Ec0pKOXfiDCqqauo/SUQkSRJerj+Su5eHiWarmY1rwKl3EawA0BU4HbjbzAbHqHc+MBYYRtDHMxq4AMDMWgGTgUeAvYC/A5PDcsysE/AycC/QEegH/LuBX7HZOn5wN/74g2G8t3gDP3vkA7ZtV5IRkfSodx5M+Af7UKAaeNXda8wsl6D/5RqCOTC3JnCdtgTLzQxx9zJgmpk9C5wJRCeps4AJ7r48PHcCcB5wDzAqjPsOd3eCnTavAI4mSCyXAVPc/dHwWtsIhlNnjDHDi6msruHqf83h+3dPZ93WKlZu0jL/IpJadSYYMzsceAFoTzA8eYaZnQ08DeQSDFF+IMHP6g9sd/dFEWWzCIZARxscHousNzji2OwwudSaHZa/TLAZ2hwzm07QenkX+Lm7L43x/c4naC3RuXNnpk6dmuBXafq6Aod3z2Z6yeadZSWbKrjqqY+Yv2A+h/fITeg6ZWVlLeq+NBbdl93pnsSWyfelvhbMb4EpwM3AT4BLgecJOvofjvojX58CYHNUWSnQLk7d0qh6BWE/TPSx6OvsDXwNOA6YA9wGPA6MjP4Qd78PuA9gwIABPmrUqMS/TTNw3TuvAbsuklm1A15Yms21p41K6BpTp06lpd2XxqD7sjvdk9gy+b7U1wczDPitu88FridoxVzj7g81MLkAlAHRi2YVAlsSqFsIlIWfWd91KoCn3X2Gu1cCvwEON7P2DYy32dMy/yKSTvUlmA7AWgg69gmWiZm5h5+1CMgxs/0iyoYB82LUnRcei1VvHjA0clQZwUCA2uOz2XWdtIYmwhYj3nL+XQpbpzgSEclEiYwiq93JsiPBH+vCqJ0tOyTyQe6+FZgE3GRmbc1sJDAGeDhG9YeAy8ysOFxo83JgYnhsKsG6aL80s9ZmdnFY/lr474PAyeFQ5lyCltc0d49+rNbiXXnCAPJzd1+HtKxyOx8s2ZiGiEQkkySSYOYTtGLWEPR/zAjfryVYPmZtAz7vIoJlZ9YQ9Itc6O7zzOxIMyuLqHcv8BxBH8pcgoEG9wK4exXBEOYfA5uAc4CxYTnu/hrB5M8Xws/pB8Scb9PSjR1RzC2nHEBxUT4GFBflc+2396dzu9b86P/e4blZ2itORJInkR0tG427byBIDtHlbxIkr9r3DlwVvmJdZyZwYB2fczdw91cMt0WIXuYf4HsH9uRnD3/ALx6fyeJ1W7n46H7s+sRRROSrS2hHS2lZOrRtxcM/PYRx/5rDhP8sYvH6cm455QBa5ezRvFsRkZgS2nBMWp7WOdn88QfD6NOxLf/7yiKWbyzn3jMPpKhNq/pPFhFJgP6TNYOZGZccux9/+p/hzFy6iZP/Op0v1m1Nd1gi0kIowQhjhhfz2HmHUlpRzcl/fYv3vtiQ7pBEpAVQghEADurTgacvOpwObVtx+v3vcN3Tsxl562uc/fJWRt76Gs/M1LY/ItIwSjCyU++ObXn6wpH07tCGR99dRkk4479kUwXXTJqjJCMiDZJQJ7+ZxVvQ0oFK4FPgCXfXxIpmrn2bXMpj7CNTUV3D7VMWaiVmEUlYoqPIOgNHAjsIJj4CDAEM+AA4hWCG/pHu/lFjBymptbK0Mma51jATkYZI9BHZW8BLwN7u/g13/wbBqsUvEmzm1Ztg5vyEpEQpKRVvDbO2rXOo2r4jxdGISHOVaIK5BLgpXPAS2Ln45e+AS8NlWsYDwxs9Qkm5WGuYZWcZZdu2c8rdb/HpmrI4Z4qIfCnRBFMAdI9R3o0vl3jZjCZutgiRa5hBsIbZhO8P474zD2TFpkpOuvNNHnlnCQ3fsUFEMkmiCeFp4G9mdhXBYpcABxNs5jUpfH8IwZL80gLUrmEWvVnS8J5FXPHP2fzqmblMXbiGW08dSqcCLf8vIrtLtAXzM4KdLR8BPgtfjxBsUXxRWGcBcF5jByhNS5fCPCaefTC/Hj2INz5Zx4l3vMnrC9ekOywRaYISSjDuXu7uPyPYgGxE+Org7heG+7zg7h9pBFlmyMoyfjKyL89ePJKObVvxkwdncOOz86is3n14s4hkrgb1mYTJZHaSYpFmZv9uhUy+eCTjX/6YB99azPTP1gXLzry7lBWbKuhRlM+VJwzQ3BmRDJXoRMs8gpFkxwBdiGr5uPvQxg9NmoO83Gx+PXowowZ04eJHP+D2KQt3HqtdAQBQkhHJQIm2YP4KnAw8BUwng/e5l9iO6t+Ztnm5bNm262MyrQAgkrkSTTBjge+7+ytJjEWaudVaAUBEIiQ6iqwcWJbMQKT5i7cCgBk8PXO55s2IZJhEE8xtwGWmjdulDrFWAGidk0VxUT6XPjGLH977DgtWbk5TdCKSaok+IjuOYLHLE81sPlAdedDdv9vYgUnzU9vPcvuUhbuMIvvusB48+f4yxr/8MSfdOY0ff703lx7Xn8K83DRHLCLJlGiCWUcwm1+kTrUrAET7n0N6ceKQbtw+ZSETpy/muVkrueZb+3PK14pRw1ikZUoowbj7T5IdiLR8RW1a8buTD+B/Du7F9ZPncvlTs3j8vaXcNGYIi1Zv2a3lo5FnIs2bFqeUlDtg7/ZMuvBwnvpgGeNfXsi3//wm2VlGzY5gEIDmz4i0DHE7+c1stpntFf48J3wf85W6cKWlyMoyfnhwL167/CjatsremVxq1c6fEZHmq64WzL+AbeHP/0xBLJKBitq0irlFM2j+jEhzFzfBuPtvYv0s0th6FOVTEiOZZGcZkz8q4aShPcjO0kAAkeYm0XkwIkkTa/5Mq2yjc0ErLvnHR5x4xxu8OGclO3ZooqZIc5JQgjGzDmZ2t5ktMrNNZrY58pXsIKVli9xB0wh20Lzte8N4a9wx/OW0Eexw56JHP+Q7d07jP/NXa0UAkWYi0VFkfyPYA+Y+YAVa7FIaWbz5MycN7cG3hnTn2Vkl/OmVTzjvofcZtnd7Lj2uP0f178zkj1ZoeLNIE5VogjkGOM7d301mMCKxZGcZJ4/Ym9FDezDpwxL+9OonnP3gDPp2bENJaSVV23cAGt4s0tQk2gezBihLZiAi9cnJzuIHB/fk9StGcfPYISzZUL4zudTS8GaRpiPRBHMdcJOZFSQzGJFEtMrJ4ozDehOvK0bDm0WahkQTzK+A44E1ZrZAEy2lKahre4A7X/2EDVurUhyRiERKNMH8E/gDMB74B8EkzMhXQsLRaE+b2VYzW2Jmp8WpZ2Y23szWh6/xkVsFmNlwM/vAzMrDf4fHuEarMBkuTzQ+aV5iD2/OYkDXdkz4zyK+fsurXDNpNp+s3pKmCEUyW72d/GaWC7QF7nL3JV/x8+4CqoCuwHDgBTOb5e7zouqdT7CL5jCCEWv/Ab4A7jGzVsBk4A6CrZwvACab2X7uHvmfrFcCa4F2XzFmaaLibQ8wdkQxn67Zwt+mLWbSh8t5/L1lHNW/M+ce0Zcj9+uEmfHMzBKNPhNJsnoTjLtXm9mFBH/M95iZtQVOBYa4exkwzcyeBc4ExkVVPwuY4O7Lw3MnAOcB9wCjwrjv8GBCxJ/N7ArgaODlsH5f4AzgMuD/vkrc0rTFG97cr0s7bjnlAK48YQCPvrOEh95Zwo8feI/+XQsY0auIyR+toLJao89EkinRYcr/JvgD/sBX+Kz+wHZ3XxRRNgs4KkbdweGxyHqDI47N9l1n280Oy18O398JXAvU2dtrZucTtJbo3LkzU6dOTeiLZJKysrJmf18OyIbffz2bd1e2Ysricp6YsfuAyIrqGn47eRZFpZ8kdM2WcF8am+5JbJl8XxJNMK8CvzezocAHwNbIg+4+KYFrFADRs/5Lif0IqyA8FlmvIOyHiT62y3XM7GQg292fNrNRdQXk7vcRTB5lwIABPmpUndUz0tSpU2kp9+U44Dp3+l7zYszjGyo94e/aku5LY9E9iS2T70uiCeYv4b+/jHHMgewY5dHKgMKoskIgVg9sdN1CoMzd3cziXid8DHcb8O0E4pEMZGYUx1lc0wxueXEBPzi4J/t21oh8ka8qoVFk7p5VxyuR5AKwCMgxs/0iyoYB0R38hGXD4tSbBwyNHFUGDA3L9wP6AG+a2SpgEtDdzFaZWZ8E45QWLt7os0HdC7l/2hccM+G/fO/u6Tz5/jLKq7anKUqR5i9lO1q6+1Yzm0QwYfOnBKPIxgCHx6j+EHCZmb1I0EK6nKBfBWAqUAP80szuIej8B3gN2AH0jLjO4QStr68RjCgTqXP02ZotlUz6sIQnZyzjqn/O5qbn5jN6WHd+cFBPhvcs2rn2WcmmCorfeU2jz0TqkHCCCXe3/BbQC2gVeczdb0rwMhcRDBRYA6wHLnT3eWZ2JPCSu9c+l7gX2AeYE76/PyzD3avMbGxYdiuwABgbMUR5VUTMG4Ad7r6zTATijz7r0i6Pnx21Lxd8Yx/eX7KRJ2Ys45mZK3j8vWV0K2zN+q1VVNdoa2eRRCSUYMzsMOAFgh0uOwMlQPfw/WIgoQTj7hsI5rdEl79J0Hlf+96Bq8JXrOvMBA5M4POmAnsnEptIJDPj4D4dOLhPB349ehDPz17JDZPn7kwutWrXPlOCEdldojP5bwceBYqBSoIhy72A9wlm94u0WO3ycvnRIb3YXhN78bOSTRW8OGclFXG2fhbJVIk+IhsKnBuO4qoBWrv752Z2NfAYQfIRadHibe2cZXDRox/SplU2xw7syuhhPfhG/060zkl0/ItIy5RogolcgmU10Jug76MM6NHYQYk0RVeeMIBrJs2hovrLlkp+bja/GzuEbu3zeG72Sl6eu5JnZ62gXV4Oxw/qxuhh3RnZrxMvzF6ppWkk4ySaYD4EDiYYajwVuNnMuhIsx6LVlCUjRI4+K9lUQXFUoji8XyduGjOY6Z+t57lZK5gybxX/+nA5bXKz2Fbj1OzQ4ADJLIkmmOv4csb9rwiGEd9JkHB+koS4RJqk2tFn8WZn52ZncVT/zhzVvzO/O3kIby5axy8en0nNjt03Rhv/8sdKMNKiJZRg3P39iJ/XEgxXFpE6tM7J5thBXamsjt35v7K0kjPuf5djBnbh2IFd6dmhTYojFEmuBk20NLODgH2B58OJk22Bbe6u6c4iccQbHFDQOofVmyv5zXPz+c1z8xnQtV2QbAZ1ZfjeRWRlaVsBad4SnQfTlWAPlkMIZtbvB3wO/JFg2PIlyQpQpLmLNzjg5rFDGDuimCXrt/LKgjW8Mn81977xOX+d+hmdClqxb+e2zFy6iSpN7JRmKtEWzP8SjB7rCCyNKH+KL5dwEZEY6lqaBqB3x7ace0Rfzj2iL6Xl1UxdtIZXFqzh+VkriJ55U1Fdw21T1HcjzUOiCeYY4Bh337jrGpN8RjDhUkTqEG9pmmjt2+QyZngxY4YX8/ysFTHrrNhUyS8fn8kR+3XiyP060b19fmOHK9IoEk0w+ew6F6ZWZ4JHZCLSyOL13eTnZjP9s/U8Gyagfl0KOKJfkGwO3acjBa1z1HcjTUKiCeYN4GyCXSIB3MyygasJNiMTkUYWr+/mllMOYMzwHny8agvTPlnHm5+u4x8zljJx+mJysoxeHdqwdEM52zXvRtIs0QRzFfBfMzsYaA1MINiiuD0wMkmxiWS0+vpuBnYvZGD3Qs77xj5UVtfwwZKNvPnJOv427fOdyaVWRXUNN78wn28d0E1L2EjKJDoPZr6ZHQBcSLCCch5BB/9d7r4yifGJZLRE+27ycrMZ2a8TI/t14t7/fhazzrqyKg648d+M6FnEoX07cOg+HRnRq4g2rb78M6BHa9KYEp4HE+6p8uvIMjPrbWZPuvsPGj0yEdkj8fpuOrRtxckjinnviw385fVP+fNrn5KTZRywd3sO6duBHTuch99ZQmV1sOqAHq3JV/VVd7QsAk5thDhEpJHE67u54aRBOxPFlspq3l+ykfe+2MB7X2zggWlf7LbXDdTud6Nh0bJnUrZlsoikRn19NxDscfPNAV345oAuAFRU1TDwhpdjXq9kUyXnTpzBiF5FjOi1F8N6FlHQevfHatpGWqIpwYi0QIn23dTKb5VNcZxHa21aZbNkQzmvfrwGADMY0LUdI3oV4cDTH5awbbseq8nulGBEBIj/aO33Jx/A2BHFlJZX89HyTcxcupGZSzfxwuyVbK7cfRnCiuoabnlpAWOG9yBqYrZkmDoTjJk9W8/5hY0Yi4ikUX2P1tq3yd25FQHAjh3Ovte+uNtyNgCrN2/jwJtfYXCPQg4obh+89m5PcVH+zqSjEWstX30tmPUJHP+ikWIRkTRryKO1rCyLO2KtfX4uxw3sypySUu5748t5OXu1yWVIcXta52TxxqK1Wsizhaszwbi7NhMTkbjiPVb7zXcH70wUldU1LFy1hdklpcxdXsqcklLmr9y827Uqqmu48bl59OtSQL8uBeTlxp4QqpZP86E+GBHZY/VtIw3BJNBhPYsY1rNoZ1nfcS/EfLS2qbyak+6cRnaWsU+ntgzsXsj+3dsxsHshg7oXMv3TdVz79NydCU0tn6ZNCUZEvpL6tpGOJd6jtS7tWvPr0YP5eNVmFqzczAdLNu5c1BMgyyBqFZxwrs5CJZgmSAlGRFIu3qO1a789kO8M7c53hnbfWV5aUc3HKzfz8aot/PrZeTGvV7Kpgp88+B79u7ajX5eCnf+2jTFfR4/WUkcJRkRSLpHJoLXa5+dy6D4dOXSfjtz3xudxtzBYWVrJW5+up6pmx87y4qJ8+nctwIA3P123c7UCPVpLDSUYEUmLhk4Ghbq3MBg7opjtNTtYuqGcRavL+HTNFhatLmPR6i18vGrLbteqqK7huqfnsKm8in06F7BP57b0aJ9PVtauc3fU8tlzSjAi0mzU1/LJyc4Kk0UB0G3nefEGFWytquHG5+bvfJ+Xm0XfTkGy2bdzARu3buPJ95drpYI9pAQjIs3KnrR84g0qKC7K4+mfj+TztVv5fO1WPltbxudry5hbUspLc1buNqAAgpbPDZPn0ioniz4d29K7Y5td+npqaY02JRgRyQDxHq1decL+dGmXR5d2eRy2T8ddztm2vYb9f/VyzJbP5srtXPTohzvfd27Xmj4d29CnY1v6dGrLms2V/GPGsoxv+SjBiEiL15BBBbVa52THbfl0b5/H/WcdxOJ15Sxev5Ul67eyeH05b3yylqc+WB7zehXVNfzqmblUVtfQs0MbenVoQ/f2eeRkZ+1SryX1+SjBiEhGaMxBBVefuD+De7RncI/2u51TXrWdwTdMidnyKdu2nXFhSwYgO8voUZRHz72ChLOlspp/z1+9R6PdmmJiUoIREYljT1o+bVrlxG359CjK48kLvs7SDeUs31DB0g3lLN1QzrKN5byyYDXryqp2O6eiuoar/zWbd7/YwN575VNclE9x+G/Xwjyys4xnZpbskgibyiO5lCYYM+sA/A04HlgHXOPuj8WoZ8CtwE/DovuBce7u4fHh4XUGAguAc939o/DYlcBZQO/wM/7q7rcn71uJSEvWmC2fq07Yn733asPee7WBfXc/L95ot23bd/DveatYv3XXBJSTZXRrn8eazdt2mf8DQWIa/3L9u5Ems+WT6hbMXUAV0BUYDrxgZrPcPXp67vnAWGAY4MB/CFZtvsfMWgGTgTuAvwIXAJPNbD93rwIM+DEwm+B/wn+b2TJ3/0dyv5qISCCRNdpiiT/aLZ+3xh1NedV2VmyqYPnGCko2VVCyMfg5cjmdSCtLKxl64xR6FOXTvX0e3Yvy6dE+j+7t8+lelMe8ks1M+M9CKqsbPhihNjG16tbvwHh1UpZgzKwtcCowxN3LgGnhfjNnAuOiqp8FTHD35eG5E4DzgHuAUWHcd4Qtmj+b2RXA0cDL7n5bxHUWmtlkYCSgBCMiKbMna7TFH+02AAgev/Xr0o5+Xdrtct4HSzbGTEyFeTmMGV7MytIKVmyq5KNlm9hYXl1nDBXVNVw/eS5V23fQtX0e3QqDV2F+zi57+UTHGYuFT52SzsxGAG+5e5uIsiuAo9x9dFTdUuB4d383fH8Q8Lq7tzOzS8Nj34qo/3x4fELUdQz4ELjX3e+JEdP5BK0lOnfufOCTTz7ZSN+25SgrK6OgoCDdYTQ5ui+70z2JraH3ZfqKav61qJr1lU7HPOPU/rkc3iO33nMmzq2iKuIpWassOHtIq93O3VbjbKx0NlQ6t82oTDiuVtmwV2ujqLXxRemOnZ+18u//j20rP4m5dWkqH5EVANGbQJQC7eLULY2qVxAmjOhjdV3nRiALeDBWQO5+H3AfwIABAzzR/8rIJA35r69MovuyO92T2Bp6X0YB1zbwM0YBg/agL+XRT16LPRihfR5PXPB1Vm2uZFVpJas3V7KytJJVmytZXVpJ1caNCcWVygRTxu5bLBcCuy8StHvdQqDM3d3MErqOmV1M0BdzpLtv+yqBi4g0dY06GOHE/enZoQ09O7SJed7IW2MnpmhZ9dZoPIuAHDPbL6JsGBBr/e154bFY9eYBQ632YWBgaOR1zOwcgn6dY2r7cUREZFdjRxRzyykHUFyUjxEMJqhdOLQuV54wgPw4O45GSlkLxt23mtkk4CYz+ynBKLIxwOExqj8EXGZmLxKMIrscuDM8NhWoAX5pZvcQdP4DvAZgZqcDvwe+6e6fJ+fbiIi0DHvS8okcJbeyjnqpbMEAXATkA2uAx4EL3X2emR0ZPvqqdS/wHDAHmAu8EJYRDkUeS/D4axNwDjA2LAe4GegIzDCzsvC1Wwe/iIjsubEjinlr3NFUrfr0g3h1UjoPxt03ECSH6PI3CTrva987cFX4inWdmUDMsdfu3rcxYhURka8m1S0YERHJEEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFClNMGbWwcyeNrOtZrbEzE6LU8/MbLyZrQ9f483MIo4PN7MPzKw8/Hd4oueKiEhqpLoFcxdQBXQFTgfuNrPBMeqdD4wFhgFDgdHABQBm1gqYDDwC7AX8HZgcltd5roiIpE7KEoyZtQVOBa539zJ3nwY8C5wZo/pZwAR3X+7uJcAE4Ozw2CggB7jD3be5+58BA45O4FwREUmRnBR+Vn9gu7sviiibBRwVo+7g8FhkvcERx2a7u0ccnx2Wv1zPubsws/MJWjwA28xsbmJfJaN0AtalO4gmSPdld7onsbX0+9I73oFUJpgCYHNUWSnQLk7d0qh6BWFfSvSx6OvEPTcqKeHu9wH3AZjZ++5+UOJfJzPovsSm+7I73ZPYMvm+pLIPpgwojCorBLYkULcQKAsTRH3XqetcERFJkVQmmEVAjpntF1E2DJgXo+688FisevOAoVEjw4ZGHY93roiIpEjKEoy7bwUmATeZWVszGwmMAR6OUf0h4DIzKzazHsDlwMTw2FSgBvilmbU2s4vD8tcSOLcu9zX8W2UE3ZfYdF92p3sSW8beF0vlkyMz6wA8ABwHrAfGuftjZnYk8JK7F4T1DBgP/DQ89X7g6trHXGY2IiwbBCwAznX3mYmcKyIiqZHSBCMiIplDS8WIiEhSKMGIiEhSZHyCSXR9tExjZlPNrNLMysLXwnTHlGpmdrGZvW9m28xsYtSxY8zs43A9vNfNLO5ks5Ym3n0xsz5m5hG/M2Vmdn0aQ02pcNDR38K/I1vM7CMz+1bE8Yz7ncn4BEPi66NloovdvSB8DUh3MGmwAriZYGDKTmbWiWBE5PVAB+B94ImUR5c+Me9LhKKI35vfpjCudMsBlhGsTtIe+BXwZJh4M/J3JpUz+ZuciPXRhrh7GTDNzGrXRxuX1uAk7dx9EoCZHQTsHXHoFGCeuz8VHr8RWGdm+7v7xykPNMXquC8ZLZyKcWNE0fNm9gVwINCRDPydyfQWTLz10dSCCdxiZuvM7C0zG5XuYJqQXda7C/+wfIZ+b2otMbPlZvZg+F/uGcnMuhL8jZlHhv7OZHqCacj6aJnmamAfoJhgothzZrZvekNqMupbDy9TrQMOJlj88ECC+/FoWiNKEzPLJfjufw9bKBn5O5PpCaYh66NlFHd/1923hFsi/B14C/h2uuNqIvR7E0O4Dcf77r7d3VcDFwPHm1mL/iMazcyyCFYoqSK4B5ChvzOZnmAasj5apnOCfXckar27sC9vX/R7E612FnfG/J0JVxL5G8GgoVPdvTo8lJG/MxnzP3wsDVwfLWOYWZGZnWBmeWaWY2anA98g2G8nY4TfPQ/IBrJr7wfwNDDEzE4Nj99AsEdRi+2sjRTvvpjZoWY2wMyyzKwj8GdgqrtHPxpqye4GBgKj3b0iojwzf2fcPaNfBEMGnwG2AkuB09IdU7pfQGdgBkHzfRPwDnBcuuNKw324keC/wiNfN4bHjgU+BioIFmDtk+54031fgB8BX4T/X1pJsPBst3THm8L70ju8F5UEj8RqX6dn6u+M1iITEZGkyOhHZCIikjxKMCIikhRKMCIikhRKMCIikhRKMCIikhRKMCIikhRKMCItVLg3y/fSHYdkLiUYkSQws4nhH/jo1zvpjk0kVTJ6PxiRJHuFYG+hSFXpCEQkHdSCEUmebe6+Kuq1AXY+vrrYzF4It9BdYmZnRJ5sZgeY2StmVmFmG8JWUfuoOmeZ2Zxw++LVZvb3qBg6mNlT4Zbgn0d/hkgyKcGIpM9vgGeB4QR77jwU7hJZu9ruFIK1rA4BTgYOJ2KbYjO7ALgXeBAYSrCdwtyoz7gBmEywku8TwANm1itp30gkgtYiE0kCM5sInEGw8GGku9z9ajNz4H53Py/inFeAVe5+hpmdB/wB2Nvdt4THRwGvA/u5+6dmthx4xN1jbu8dfsat7n5N+D6HYIO98939kcb7tiKxqQ9GJHneAM6PKtsU8fPbUcfeBr4T/jyQYDn3yA2ppgM7gEFmtplgt9FX64lhdu0P7r7dzNYCXRKKXuQrUoIRSZ5yd/80CddtyGOH6qj3jh6NS4roF00kfQ6L8X5B+PMC4ICo7YYPJ/j/7AJ3XwOUAMckPUqRPaQWjEjytDazblFlNe6+Nvz5FDObQbD51PcIksWh4bFHCQYBPGRmNwB7EXToT4poFf0O+F8zWw28ALQBjnH3Ccn6QiINoQQjkjzHEuzsGKkE2Dv8+UbgVIKthdcCP3H3GQDuXm5mJwB3AO8RDBaYDFxSeyF3v9vMqoDLgfHABuDFJH0XkQbTKDKRNAhHeH3f3f+Z7lhEkkV9MCIikhRKMCIikhR6RCYiIkmhFoyIiCSFEoyIiCSFEoyIiCSFEoyIiCSFEoyIiCTF/weMhp0qu/HzfAAAAABJRU5ErkJggg==\n"
     },
     "metadata": {
      "needs_background": "light",
      "image/png": {
       "width": 408,
       "height": 286
      }
     },
     "output_type": "display_data"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "### Exponential Scheduling",
   "metadata": {
    "cell_id": "00119-7cc265b7-056d-4814-8f16-e9942c1cf120",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 62
   }
  },
  {
   "cell_type": "markdown",
   "source": "$$\\eta(t) = \\eta_0\\cdot 0.1^{t/s}$$",
   "metadata": {
    "cell_id": "00120-803ed7bd-d6a4-4d7c-b15f-5b0f2ad7f921",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00121-80fd0403-cc1e-461e-a88f-07c479d5150b",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "81a4b745",
    "execution_start": 1659355553676,
    "execution_millis": 5,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "source": "def exponential_decay_fn(epoch):\n    return 0.01 * 0.1**(epoch / 20)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00122-75764e0a-bc49-446b-804a-d3bcb8fd8ac4",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "ef6689e6",
    "execution_start": 1659355553694,
    "execution_millis": 2,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 166
   },
   "source": "def exponential_decay(lr0, s):\n    def exponential_decay_fn(epoch):\n        return lr0 * 0.1**(epoch / s)\n    return exponential_decay_fn\n\nexponential_decay_fn = exponential_decay(lr0=0.01, s=20)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00123-d4ba7365-92a8-4d5c-bcf4-8cdaad46580f",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "bd3b96e6",
    "execution_start": 1659355553705,
    "execution_millis": 58,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 202
   },
   "source": "model = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.Dense(10, activation=\"softmax\")\n])\nmodel.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\nn_epochs = 25",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00124-aa4de55a-e06d-4040-9b10-8741337c2033",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "693d6a82",
    "execution_start": 1659355553763,
    "execution_millis": 322753,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 750
   },
   "source": "lr_scheduler = keras.callbacks.LearningRateScheduler(exponential_decay_fn)\nhistory = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n                    validation_data=(X_valid_scaled, y_valid),\n                    callbacks=[lr_scheduler])",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.8520 - accuracy: 0.7539 - val_loss: 1.0364 - val_accuracy: 0.7288 - lr: 0.0100\nEpoch 2/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.6711 - accuracy: 0.7997 - val_loss: 0.8246 - val_accuracy: 0.8082 - lr: 0.0089\nEpoch 3/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.5753 - accuracy: 0.8206 - val_loss: 0.6832 - val_accuracy: 0.8062 - lr: 0.0079\nEpoch 4/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.5655 - accuracy: 0.8287 - val_loss: 0.5217 - val_accuracy: 0.8330 - lr: 0.0071\nEpoch 5/25\n1719/1719 [==============================] - 11s 7ms/step - loss: 0.4831 - accuracy: 0.8490 - val_loss: 0.5566 - val_accuracy: 0.8558 - lr: 0.0063\nEpoch 6/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.4433 - accuracy: 0.8590 - val_loss: 0.5107 - val_accuracy: 0.8522 - lr: 0.0056\nEpoch 7/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.4101 - accuracy: 0.8672 - val_loss: 0.5292 - val_accuracy: 0.8572 - lr: 0.0050\nEpoch 8/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.3819 - accuracy: 0.8757 - val_loss: 0.5385 - val_accuracy: 0.8370 - lr: 0.0045\nEpoch 9/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.3510 - accuracy: 0.8841 - val_loss: 0.5049 - val_accuracy: 0.8662 - lr: 0.0040\nEpoch 10/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.3212 - accuracy: 0.8912 - val_loss: 0.4730 - val_accuracy: 0.8798 - lr: 0.0035\nEpoch 11/25\n1719/1719 [==============================] - 12s 7ms/step - loss: 0.2946 - accuracy: 0.8993 - val_loss: 0.4756 - val_accuracy: 0.8760 - lr: 0.0032\nEpoch 12/25\n1719/1719 [==============================] - 11s 7ms/step - loss: 0.2777 - accuracy: 0.9052 - val_loss: 0.4832 - val_accuracy: 0.8732 - lr: 0.0028\nEpoch 13/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.2606 - accuracy: 0.9109 - val_loss: 0.5483 - val_accuracy: 0.8732 - lr: 0.0025\nEpoch 14/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.2423 - accuracy: 0.9168 - val_loss: 0.4700 - val_accuracy: 0.8766 - lr: 0.0022\nEpoch 15/25\n1719/1719 [==============================] - 13s 7ms/step - loss: 0.2214 - accuracy: 0.9228 - val_loss: 0.4602 - val_accuracy: 0.8804 - lr: 0.0020\nEpoch 16/25\n1719/1719 [==============================] - 12s 7ms/step - loss: 0.2116 - accuracy: 0.9272 - val_loss: 0.4991 - val_accuracy: 0.8844 - lr: 0.0018\nEpoch 17/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.1886 - accuracy: 0.9338 - val_loss: 0.5062 - val_accuracy: 0.8808 - lr: 0.0016\nEpoch 18/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.1731 - accuracy: 0.9394 - val_loss: 0.4961 - val_accuracy: 0.8828 - lr: 0.0014\nEpoch 19/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.1647 - accuracy: 0.9439 - val_loss: 0.5205 - val_accuracy: 0.8894 - lr: 0.0013\nEpoch 20/25\n1719/1719 [==============================] - 12s 7ms/step - loss: 0.1497 - accuracy: 0.9496 - val_loss: 0.5282 - val_accuracy: 0.8898 - lr: 0.0011\nEpoch 21/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.1388 - accuracy: 0.9530 - val_loss: 0.5830 - val_accuracy: 0.8848 - lr: 0.0010\nEpoch 22/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.1287 - accuracy: 0.9571 - val_loss: 0.5445 - val_accuracy: 0.8904 - lr: 8.9125e-04\nEpoch 23/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.1192 - accuracy: 0.9610 - val_loss: 0.5927 - val_accuracy: 0.8904 - lr: 7.9433e-04\nEpoch 24/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.1110 - accuracy: 0.9633 - val_loss: 0.6210 - val_accuracy: 0.8920 - lr: 7.0795e-04\nEpoch 25/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.1037 - accuracy: 0.9658 - val_loss: 0.6488 - val_accuracy: 0.8898 - lr: 6.3096e-04\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00125-c29ff8ed-1c7f-4bb6-abbb-3930d09b2bc8",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "e355338a",
    "execution_start": 1659355876528,
    "execution_millis": 526,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 488,
    "deepnote_output_heights": [
     288
    ]
   },
   "source": "plt.plot(history.epoch, history.history[\"lr\"], \"o-\")\nplt.axis([0, n_epochs - 1, 0, 0.011])\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Learning Rate\")\nplt.title(\"Exponential Scheduling\", fontsize=14)\nplt.grid(True)\nplt.show()",
   "execution_count": null,
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEeCAYAAAC30gOQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAA240lEQVR4nO3deXxU1f3/8dcnGwkECAEECbIpYkE2EdeiqFVs1a+4dNO6tHVt/WnrbtXWaq2itbVWq1K3urVqi6KiohajuKCyKKuAgghh3wKBQFg+vz/uDQ7DTDLBzEySeT8fj3kkc86ZO5+5hHxy7jn3HHN3RERE6ltWugMQEZGmSQlGRESSQglGRESSQglGRESSQglGRESSQglGRESSQglGJAXM7Fwzq6jja0rN7N5kxRS+x5dmdmUSjnu6mdXpHojoc7Q750waFiUYSSoze8zMPMZjQrpjS5bw850eVfwM0CMJ73WemU0xswozKzezqWb2h/p+nzRJyjmT1MlJdwCSEd4Ezooqq0pHIOni7pVAZX0e08x+BtwD/Br4H5AL7A8cWp/vky7JOGeSWurBSCpsdvelUY/VAGZ2pJltMbOh1Y3N7EIzW2dmPcLnpWb2gJn91czWhI87zSwr4jVtzOyfYV2lmb1pZn0i6s8N/8o/xsymm9kGM3vLzLpHBmpmJ5nZJDPbZGbzzexWM8uLqP/SzG4wswfDGBeZ2VWR9eG3z4U9mS8j3z+i3d5mNtrMloaxTDazE+t4Xv8PGOXuD7r75+4+y92fc/fLoz7T98zsw/C8rDKzl8wsP6JJfrzPE76+tZmNNLPlZrbezN42swOj2pxtZgvMbKOZvQx0iKq/ycymR5XVeAksxjm7Kfy3+5GZfRHG8oKZtYtok2Nmf4n4OfmLmd1vZqW1n06pb0owklbu/jZwJ/BEmCT2A/4M/D93nxfR9EyCn9dDgQuBC4BfRdQ/BhwMnAwcBGwEXjOzgog2zYDrgJ+FxykCHqiuNLNhwFPAvUCfsN3pwB+jwv41MA04ABgB3GFm1b2GweHX84E9I55HKwReBY4F+gP/BUaFnz9RS4GDqhNxLGZ2PPAi8AYwCDgKeJud/+/H/TxmZsAYoAQ4ERgIvAOMM7M9wzYHE5z/kcAA4CXg5jp8jrroBvwQOAU4Lozn1oj6K4FzgfOAQwg+5xlJikVq4+566JG0B8Evnq1ARdRjRESbXOBjYBQwGXgm6hilwBzAIspuABaF3/cEHDgior41UA6cFz4/N2zTK6LNmcDm6uMS/OK8Meq9h4fxVrf5EvhXVJu5wA0Rzx04ParNuUBFLedqQtRxSoF7a2i/J/BB+H5zgSeBs4HciDbvAf+u4Rg1fh7g6PDzF0S1+QS4Ovz+aeCNqPqHgl8vO57fBEyv6Zwk8PwmYBPQOqLseuDziOdLgGsjnhswGyhN9/+FTHyoByOp8A7BX7aRjzurK919C8FfmScCexD0UKJN8PA3RugDoMTMWgHfAraHZdXHLCf4q7x3xGs2u/vsiOeLgTygTfh8EHB9eCmtIrw88zTQAugY8bqpUbEtDuNOmJm1MLM7zGxmeCmnAjgQ6JLoMdx9ibsfCvQF7ib4Zfog8JGZNQ+bDSQYn6lJTZ9nENAcWBF1XvYH9g7bfIuIcx+Kfl5fFoT/trvEamatCf6dPqquDH9mPkLSQoP8kgob3f3zWtpUX84oAtoDa+vpvSOT0tY4dVkRX38PPBfjOCsivt8S4zh1/WPtT8DxBJd05hJc0nucIOHVibtPB6YD95nZt4HxwA8Ieo+JqOnzZAHLgCExXreuDmFuJ0iAkXLr8Ppq9XHuJUX0DyNpFw603wv8kmCs4Ekzi/7j5+BwPKDaIcBid18HzOLr8ZnqY7Yi+Mt+Zh1CmQzs58GAefQjOjnVZAuQXUubbwOPu/t/3X0qsIivewTfRPXnLQy/TgGO+QbHm0wwYL89xjlZHraZRfDvESn6+QqgQ9S/4YBvENcuwp7NUiLGvcL3izcOJkmmHoykQjMz6xhVts3dV5hZNvAE8La7P2hm/yG4tPU74MaI9p2Au83s7wSJ4yrgDwDuPtfMRgMPmtkFBL2fWwn+wn66DnHeDLxsZguAZwl6PPsDB7n71XU4zpfAMWb2NsFluTUx2swBTgnj3kLwefNjtIvLzO4nuEQ0jiBB7UkwNrUReD1sdivwkpl9TnAujGBw/EF335jA27xJMI4z2syuBj4juAx1PPCmu48nmCr9vpldB/wHGEowCB+pFCgGfmNm/w7bRN8rVB/+ClxtZnMIku2FBOdlSRLeS2qhHoykwncI/oNHPqaEdb8B9gF+DuDuq4BzgGvDyz3VniLoFXwI/AN4GPhLRP1PCa61vxh+bQ4c78G9FAlx97HACQQzrT4KH9cCXyX+UQG4IjzGQr7+nNEuB5YTXM56lWCAf3wd3+cNgplzzxIkrOfD8mPdfQ6Au79C8Mv+u2Esb4exbU/kDcIxjO8RJLF/EAyYPwv0IkhuuPsEgn+/iwnGc04lGJCPPM6ssP6CsM2x7Do7rz78ieAPlkcJzikE52VTEt5LalE9M0akwQrvYZju7pekOxZpfMxsCvCuu/+/dMeSaXSJTESaDDPrCgwj6KnlEtyP1C/8KimmBCMiTcl2gnuB7iQYApgJfNfdJ6Y1qgylS2QiIpIUGuQXEZGk0CWyUFFRke+zzz7pDqPB2bBhAy1atEh3GA2OzsuudE5ia+rnZdKkSSvdvX2sOiWYUIcOHZg4UZdpo5WWljJ06NB0h9Hg6LzsSucktqZ+XsL7xmLSJTIREUkKJRgREUkKJRgREUkKJRgREUkKJRgREUkKJRgREUkKJRgREUkKJRgREUkKJRgREUkKJRgREUkKJRgREUkKJRgREUkKJRgREUkKJRgREUkKJRgREUmKlCYYMys2s+fNbIOZLTCzM+K0MzMbYWarwscIM7OI+pFmNtvMtpvZuTFe/2szW2pm68zsETNrVltsX67bzuG3j+OFKWXf6DOKiEgg1T2Y+4AqoANwJnC/mfWJ0e4CYDjQH+gHnARcGFH/KfALYHL0C81sGHAtcAzQFegB/D6R4MrWVnLdqGlKMiIi9SBlCcbMWgCnATe6e4W7vwu8CJwVo/k5wF3uvsjdy4C7gHOrK939Pnf/H7ApzmsfdvcZ7r4GuCXytbWp3LKNO8fOTrS5iIjEkcotk/cFtrr7nIiyT4EjY7TtE9ZFtovV04mlDzA66rUdzKytu6+KbGhmFxD0lsjruM+O8rK1lZSWlib4dk1bRUWFzkUMOi+70jmJLZPPSyoTTCGwLqqsHGgZp215VLtCMzN39wTeJ/q1hO+zU4Jx95HASIBme/bccdySooImvYd2XTT1/cR3l87LrnROYsvk85LKMZgKoFVUWStgfQJtWwEVCSSXeK8lzvvsItuMq4b1SqSpiIjUIJUJZg6QY2Y9I8r6AzNitJ0R1tXWLpZYr10WfXkslpbNctjmTuvmuQm+lYiIxJOyBOPuG4BRwM1m1sLMDgdOBp6I0fxx4HIzKzGzTsAVwGPVlWaWZ2b5gAG5ZpZvZlkRr/25mfU2syLghsjXxtOtVRYTb/wO++xRyPWjplGxeetuf1YREUn9NOVfAAXAcuBfwMXuPsPMhphZRUS7B4GXgGnAdGBMWFbtdaASOIxgDKUSOALA3V8D7gDeAr4CFgC/SyS4ZjnZjDitH0vWbeKO1z7b7Q8pIiKpHeTH3VcT3N8SXT6eYHC++rkDV4ePWMcZWsv7/Bn48+7EOKhrG356WHceeW8+J/brxEHdi3fnMCIiGU9LxcRw5bB96dymgGv+O5VNW7alOxwRkUZJCSaG5nk53H5qP+av3MBf/zc33eGIiDRKSjBxfLtnO35wYGdGvjOP6WXltb9ARER2ogRTg+u/15viFnlc/Z+pbNm2Pd3hiIg0KkowNWjdPJdbTt6fmUvWMfKdeekOR0SkUVGCqcXx+3fke3078tf/zeXz5RW1v0BERAAlmITc9H99KMjN5tr/TmX79kRWqxERESWYBOzRMp8bT+zNxAVreGLCgnSHIyLSKCjBJOi0A0oY0rMdI177jEVrNqY7HBGRBk8JJkFmxh9P6QvAb56fTmILO4uIZC4lmDrYq7g51xy/H+/MWcGoydpWWUSkJkowdXTWIV05sGsbbn55JivWb053OCIiDZYSTB1lZRm3n9aPik1bOOKOcXS/dgyH3z6OF6aoRyMiEimlqyk3FdPLyjEzKrcEd/eXra3kulHTABg+sCSdoYmINBjqweyGO8fOZmvU/TCVW7Zx59jZaYpIRKThUYLZDYvXVtapXEQkEynB7IZORQV1KhcRyURKMLvhqmG9KMjN3qX8R4P3SkM0IiINkxLMbhg+sITbTu1LSVEBBuzZOp/WBTn8d/IiKjZvTXd4IiINgmaR7abhA0t2mjE2Yd4qzvjHBH47ejp//sGA9AUmItJAqAdTTw7p0ZZLju7JqMllPD9lUbrDERFJOyWYenTp0fswuFsbbnh+Ol+u3JDucERE0koJph7lZGdx948GkpOdxaX/nkLVVm2zLCKZSwmmnpUUFTDitL5MXVTOXa/rxksRyVxKMElw/P57cubBXXjwnXm8PWdFusMREUkLJZgkufHE3uzboZArnv1Eqy6LSEZSgkmS/Nxs/vbjA1i/aSuXP/sJ27drgzIRySxKMEnUq2NLbjyxN+PnruShd+elOxwRkZRSgkmyMw/uwvF9OnLHa7P5dOHadIcjIpIySjBJZmbcflpf9mjZjEv/PYX1m7akOyQRkZRIaYIxs2Ize97MNpjZAjM7I047M7MRZrYqfIwwM4uoH2Bmk8xsY/h1QERdMzN7wMyWmdlqM3vJzNK6C1hR8zz++uOBLFy9kd+OnpHOUEREUibVPZj7gCqgA3AmcL+Z9YnR7gJgONAf6AecBFwIYGZ5wGjgSaAN8E9gdFgOcBlwaPi6TsAa4G/J+TiJG9ytmMuO2Zfnp5Qx8ObXtdWyiDR5KUswZtYCOA240d0r3P1d4EXgrBjNzwHucvdF7l4G3AWcG9YNJVik82533+zu9wAGHB3WdwfGuvsyd98EPAPESmIpt1ebArIM1mzcgvP1VstKMiLSFKVyNeV9ga3uPiei7FPgyBht+4R1ke36RNRNdffIeb9Tw/LXgIeBv5pZJ2AtQU/p1VgBmdkFBL0l2rdvT2lpad0+UR3dWrqR6NnKlVu2ccvoTykqn5vU995dFRUVST8vjZHOy650TmLL5POSygRTCKyLKisHWsZpWx7VrjAch4muiz7OXGAhUAZsA6YBl8QKyN1HAiMBevXq5UOHDk3wo+ye1a+NiV2+yUn2e++u0tLSBhtbOum87ErnJLZMPi+pHIOpAFpFlbUC1ifQthVQEfZaajvOfUAzoC3QAhhFnB5MqmmrZRHJJAknGDP7rpm9bGYzzWyvsOw8MzsmwUPMAXLMrGdEWX8g1rSqGWFdrHYzgH6Rs8oIBvSr6wcAj7n7anffTDDAf5CZtUswzqSJt9Xyd/fvmIZoRESSK6EEY2ZnAs8SXH7qDuSGVdnA1Ykcw903EPQmbjazFmZ2OHAy8ESM5o8Dl5tZSTiWcgXwWFhXSnDp69JwSnL15a9x4dePgbPNrLWZ5QK/ABa7+8pE4kym6K2WO7XOp3NRAf/+eCFzl8XqyImINF6J9mCuBs53918DkZvOTyDoMSTqF0ABsBz4F3Cxu88wsyFmVhHR7kHgJYLxk+nAmLAMd68imMJ8NsEg/s+A4WE5wJXAJoJkuAL4HnBKHWJMquEDS3jv2qOZf/sJvH/dMTx38aHk52Zz/uMTKd+omzBFpOlIdJC/J/BBjPJY4yFxuftqguQQXT6eYPC++rkTJLWYvSN3nwIMilO3imDmWKOwZ+sCHjzrAH488kMu+ddkHj13MDnZWmBBRBq/RH+TLSaYZhztCOCL+gsnMw3qWswfhu/P+Lkr+eMrn6U7HBGRepFoghkJ3BOOmwDsZWbnAHcA9yclsgzzg8F7ce5h3Xjkvfk8N3FhusMREfnGErpE5u53mFlr4A0gH3gL2Az8yd3vS2J8GeWGE77F3OXruf756fRoX8igrm3SHZKIyG5L+GK/u18PtAMOAg4B2rv7jckKLBPlZGdx748PYM+ifC56chJLyzelOyQRkd2W6DTlR8yspbtvdPeJ7v6Ru1eE040fSXaQmaRNizz+cfaBbNy8lQuemMimLdvSHZKIyG5JtAdzDsH04mgFBNOFpR7t26Eld/9oINPKyrn2v1PZedk1EZHGocYEE+7f0pZgteI24fPqR3vgRGBZKgLNNMf27sAVx+7LC58sZuQ72m5ZRBqf2gb5VwIePmbGqHfgd/UdlAR+edQ+zFq6nttf+4x9O7bkqF57pDskEZGE1ZZgjiLovYwj2MtldURdFbDA3RcnKbaMZ2bceXo/5q/YwEWPT6R18zxWrN9Mp6ICrhrWi+ED07pRp4hIjWpMMO7+NoCZdQcWuvv2lEQlOzTPy+H7B3bm9y/NZPn6zcDXG5UBSjIi0mAleh/MAoBw4ckuQF5U/Tv1H5pUe2j8/F3KKrds486xs5VgRKTBSijBhInlaYKlYZzgslnk1KZd16CXerN4bWWdykVEGoJEpynfTbBEfm9gIzAE+D4wCzg+KZHJDvE3KstPcSQiIolLNMEcCVzj7p8R9FxWuPso4BrglmQFJ4F4G5X161yU+mBERBKUaIIpIJiyDMFMsur5sjMJdpOUJNplo7KifAZ3bcOr05fyxIQF6Q5PRCSmRPeD+QzYD/gS+AS4yMwWAr8EypISmexk+MCSnQb0t2zbzkVPTOK3o6dT3DyPE/rtmcboRER2lWgP5q9A9cbxNwPHAfMIdqj8TRLiklrkZmdx7xkHcGDXNvzqmSm8OzftO0KLiOwkoQTj7k+5+2Ph95OBbsBgoIu7P5e06KRGBXnZPHTOYPZuX8gFT0zk04Vr0x2SiMgOu7U3b7iq8mRgg5ldW88xSR20Lsjl8Z8dRHGLPM599CM+X16R7pBERIAEEoyZtTOzE8zsODPLDstyzexXBGMyVyY3RKnNHq3yefLnB5OdZZz98IcsKdf9MSKSfrWtpnwYMBd4CXgVeM/M9gOmApcQTFHukuwgpXbd2rXgsZ8exPpNWznr4Y9Ys6Eq3SGJSIarrQdzCzCWYCry3QS7Wb4M3Ab0dPd73X1jUiOUhO1f0pqRZx/IV6s38tPHPmZj1dZ0hyQiGay2BNMfuMXdpwM3EtxkeZ27P+7aBatBOnTvtvztxwOZumgtFz05maqtWp9URNKjtvtgioEVEAzsm9lGYErSo5JvZFifjtx2al+u+e80fvTgByxdv4klazdpmX8RSalEbrRsY2Zb+XqBy1ZmVhzZwN1Xx3ylpM0PB3dh/NwVvDx16Y4yLfMvIqmUyDTlmQS9mOVAIfBx+HwFwfIxK5IWnXwjU75au0tZ9TL/IiLJlsiOltJILV67KU65pjGLSPIltKOlNE6digooi5FMtMy/iKTCbt3JL41DvGX+e7QvRJMARSTZUppgzKzYzJ43sw1mtsDMzojTzsxshJmtCh8jzMwi6geY2SQz2xh+HRD1+gPM7B0zqzCzZWZ2WZI/WoMUvcx/SVE+R/Zsx/i5K7n+hels364kIyLJk+hy/fXlPqAK6AAMAMaY2afuPiOq3QXAcIL7cBx4A5gPPGBmecBoghs//w5cCIw2s57uXmVm7YDXgF8D/wHygM7J/VgNV/Qy/+7OHWNnc3/pF2zZup3bT+tHdpbVcAQRkd2Tsh6MmbUATgNudPcKd38XeBE4K0bzc4C73H2Ru5cBdwHnhnVDCRLj3e6+2d3vIZhCfXRYfzkwNlwBerO7r3f3WUn7YI2MmXH1sF5cdkxPnpu0iCue/YSt23QzpojUv1T2YPYFtrr7nIiyTwm2Y47WJ6yLbNcnom5q1EoCU8Py14BDgGlm9j6wD/Ah8Et3/yr6TczsAoLeEu3bt6e0tHQ3PlbjNDAXTuuZy38/WUzZ0mVc2K8ZOTF6MhUVFRl1XhKl87IrnZPYMvm8JJRgzOyROFUObAI+B55x98U1HKYQWBdVVg60jNO2PKpdYTgOE10XfZzOwAHAscA04A7gX8DhuwTvPhIYCdCrVy8fOnRoDeE3PUOHwrfGz+MPY2ZRVNySe88YSLOcnScFlJaWkmnnJRE6L7vSOYktk89Loj2Y9sAQYDswPSzbn+DS1CTgVOBmMxvi7p/EOUYF0CqqrBWwPoG2rYAKd3czq+04lcDz7v4xgJn9HlhpZq3dPToxZbzzhvQgNzuL3704gwufmMQDPxlEfoyZZyIidZXoGMx7BMv1d3b3I9z9CIKewivA60BXYAzBWEk8c4AcM+sZUdYfiB7gJyzrH6fdDKBf5KwygtWeq+unEvSsqmmqVC3OOawbfzylL2/PWcF5/5xIZdW2dIckIk1AognmMuDmyKX5w+9vBX7t7lXACIKZYTG5+wZgFEFPp4WZHQ6cDDwRo/njwOVmVmJmnYArgMfCulJgG3CpmTUzs0vC8nHh10eBU8KpzLkEq0C/q95Lzc44uAt3nt6f979YybmPfsSGzVrqX0S+mUQvkRUCewLRs7E6hnUQjK/UdrxfAI8QrGu2CrjY3WeY2RDgVXevPtaDQA+CMRSAh8IywqnIw8Oy28OYhodJDncfZ2a/IehRNQfeBWLebyM7O31QZ3Kzjcuf/ZQT7hnP5q3bWVK+iZIJ47QKs4jUWaIJ5nngYTO7mmCxS4DBBAPoo8LnBxFcBosrXHV5eIzy8XydqAhniF0dPmIdZwowqIb3uR+4v6ZYJLaTB5TwyVdrePT9BTvKtAqziOyORC+RXUSws+WTwBfh40mCacG/CNvMAs6v7wAl9V6fuXyXMq3CLCJ1lVAPJhxvucjMrgD2Dou/CMdVqtt8Uv/hSTrEW21ZqzCLSF3U6UbLMKFMTVIs0kDEW4W5uEVeGqIRkcYqoUtkZpZvZteY2etm9omZTY18JDtISa1YqzCbwaoNVTzxwZfpCUpEGp1EezB/B04BngPeR/eWNGnVA/l3jp1N2dpKSooKuPTofXh95jJuHD2DhWsqufb4/cjSIpkiUoNEE8xw4Pvu/mYSY5EGpHoV5shlLk4/cC9+/9IMRr4zj4WrN/KXHw7QXf8iEleis8g2AguTGYg0fNlZxu//rw83nPAtXpuxlB//YwKrKjanOywRaaASTTB3ENxZr2siGc7MOG9ID+4/8wBmLl7HKX9/ny9WVKQ7LBFpgBJNMMcCPwS+NLNXzezFyEcS45MG6vj99+TfFxzChs1bOfXv7/PhvFXpDklEGphEE8xKgrv5xwFLCZZ5iXxIBhrYpQ3P/+Jw2hbmcdbDHzH6k7J0hyQiDUiiN1r+NNmBSOPUpW1zRl18GBc+MYnL/v0Jr89YyicL17J47SY6FRVoDTORDJayLZOl6SpqnsfjPz+IQV2KGDNtKWVrN+F8vYbZC1PUsxHJRHF7MOENlEe6+xozm0YN9764e79kBCeNR7OcbJau27RLefUaZurFiGSemi6R/ReonoP6nxTEIo3c4rW7JpigXGuYiWSiuAnG3X8f63uReOKtYdaqIAd3R7PcRTKLxmCk3sRawyzLoLxyK798ejIV2iVTJKMkNIvMzIoJtkc+BtiDqMTk7q3qPzRpbCLXMFu8tpJORQVcedy+LF+/mTvGzuazpe/ywE8GsW+HlmmOVERSIdG1yB4GBgIjgcVosUuJo3oNs2j99yrikqencPK973H7aX05eYAG/UWaukQTzDHAse7+YTKDkabrkB5teeXSb/PLpydz2b8/YfKCNVx/Qm/ycnSVVqSpSvR/93JAC07JN7JHq3yePv8Qzh/SnX9+sIAfPPiBZpiJNGGJJpjrgZvNrDCZwUjTl5udxfUn9Ob+Mw/g8+UVnHDPeMbPXZHusEQkCRK9RHYD0A1YbmYLgC2RlbrRUurqu333pFfHllz85GTOfuQjju/TkU8XrWWJlpgRaTISTTC60VLqXY/2hTz/y8M4++EPeXX60h3l1UvMAEoyIo1YrQnGzHKBFsB97r4g+SFJJmmel8OSci0xI9IU1ToG4+5bgIsB3YYtSaElZkSapkQH+V8Hjk5mIJK5OhUVxCzPyjImLViT4mhEpL4kmmD+B/zRzO42s7PM7NTIRzIDlKYv1hIzeTlZtGyWw/cfeJ87x35G1dbtaYpORHZXooP894ZfL41R50B2jHKRhMRaYuaqYb045lt7cMvLM7nvrS9467MV/OWHA+jVUcvMiDQWie5oqdutJaniLTFzx+n9ObZ3R64bNZWT/vYuVw7bl59/uwfZWRoSFGnolDikwTu2dwfG/uoIjtqvPX985TN+PHICC1dvTHdYIlKLRC+RYWZtgO8CXYC8yDp3vznBYxQTLJx5HLASuM7dn47RzoDbgfPCooeAa93dw/oB4XG+BcwCfu7un0QdIw/4FGjp7p0T+pDSYLUtbMYDPxnEqMll3PTiDI6/+x1uPLE3zXKy+NPrc3a6tKapzSINQ6LL9R8CjCHY4bI9UAbsGT7/EkgowQD3AVVAB2AAMMbMPnX3GVHtLgCGA/0JxnjeAOYDD4SJYzRwN/B34EJgtJn1dPeqiGNcBawAdNG+iTAzThvUmYN7FHPVc1O5dtQ0sgy2h2t76wZNkYYl0UtkdwJPASXAJoIpy12AicCIRA5gZi2A04Ab3b3C3d8FXgTOitH8HOAud1/k7mXAXcC5Yd1QgsR4t7tvdvd7CO7R2TGN2sy6Az8Bbkvw80kj0rlNc54672BaF+TsSC7Vqm/QFJH0S/QSWT+Cy1BuZtuAZu4+z8yuAZ4mSD612RfY6u5zIso+BY6M0bZPWBfZrk9E3dTqy2WhqWH5a+HzvwG/AWq8U8/MLiDoLdG+fXtKS0sT+BiZpaKiosGel/LK2Dtklq2tTHrMDfm8pIvOSWyZfF4STTCRl56WAV0Jxj4qgE4JHqMQWBdVVk7sS1iFYV1ku8JwbCa6bqfjmNkpQLa7P29mQ2sKyN1HEmyiRq9evXzo0BqbZ6TS0lIa6nkpmTCOshh3+7dslsMhhw8hPzd5s+cb8nlJF52T2DL5vCR6iWwyMDj8vhT4g5mdA9xD0HtIRAUQvbVyK2B9Am1bARVhryXuccLLcHcQ+34daWJi3aCZbcb6zVsZdvc7lM5enqbIRATqth/M4vD7GwgGz/8GtCG8xJSAOUCOmfWMKOsPRA/wE5b1j9NuBtAv7M1U6xeW9yTYVmC8mS0FRgF7mtlSM+uWYJzSSAwfWMJtp/alpKgAA0qKCrjrB/156ryDyTbj3Ec/5pdPTWZpjMU0RST5Er3RcmLE9ysIpivXibtvMLNRBBuXnUcwi+xk4LAYzR8HLjezVwhmkV1BkNAg6EFtAy41sweA88PyccB2YK+I4xxGsArBAQRJUZqYeDdovvqrIYx8ex73vvU5pbOXc/lxvTjn0K7kZOvWL5FUqdP/NjM70Mx+GF6KwsxamFnC99IAvwAKCLZg/hdwsbvPMLMhZha5JfODwEvANGA6wRTpBwHCqcjDgbOBtcDPgOHuXuXuW919afUDWA1sD59vq8tnlcatWU42/++Ynrzx6yMZ3L2YW16eyUn3vsfkr7R4pkiqJHofTAeCe08OIuhR9ATmAX8mmLZ8WSLHcffVBMkhunw8weB99XMHrg4fsY4zBRiUwPuVArrJMoN1aducR88dzGvTl/L7l2Zy2v3v86PBXehb0or73vpCN2iKJFGivY+/EMweawt8FVH+HF9fuhJpkMyM7/bdkyH7tufuN+bw8Lvz+VdEvW7QFEmORC+RHQNc7+7R1xe+ILjhUqTBK2yWww0n9qZ9y2a71OkGTZH6l2iCKWDne2GqtSe4RCbSaKxYvzlmuXbQFKlfiSaYd/h6qRYAN7Ns4BqCzchEGo14O2g68Jvnp7F8vf5mEqkPiSaYq4HzzewNoBnB2mAzgcOB65IUm0hSxLpBMz83iyE92/HsxwsZemcpf3ljDhs2x16KRkQSk+h9MDPNrC9wMcEKyvkEA/z3ufuSJMYnUu/i7aA5fGAJX67cwJ2vz+av/5vLUx9+xa++05MfDt6LXN0/I1JnCd/DEt5X8rvIMjPrambPuvsP6j0ykSSKd4Nmt3YtuO+MAzjv22u47ZXPuOGF6Tzy3nyuOX4/juvdgdGfLObOsbMpW1tJyYRxmt4sUoO63CQZSxHBEvwiTcrALm145sJD+N+s5dz+2mdc+MQkurdtzuLyTWzeuh3Q9GaR2qjfLxKHmfGd3h147bIh3HZqXxas3rgjuVTT9GaR+JRgRGqRk53Fjw/qgnvsek1vFolNCUYkQfGmNzfLyeLThWtTG4xII1DjGIyZvVjL66P3ZRFpsq4a1ovrRk2jcsvX66bmZBlmcPJ973HEvu259Oh9OLBbcRqjFGk4ahvkX5VA/fx6ikWkQYuc3ly2tpKScHrzd3p34IkPFvDQ+Hmc/sAHHNKjmEuP7smhe7dl522LRDJLjQnG3X+aqkBEGoPq6c3R2+BePHRvzjmsK//6aCEPvv0FZzz0IYO6tuGSo/dh6L7td0xv1urNkkm+6TRlEQk1z8vh59/uzpkHd+G5iQu5v/QLfvrox+zVpoBl6zZRtS2YJaDpzZIpNMgvUs/yc7M569BulF51FCNO68vi8q+TSzVNb5ZMoAQjkiR5OVn8cHAXtm+PPb9Z05ulqVOCEUmymlZvvuLZT5leVp7agERSRAlGJMlird7cLCeLb+/TllenL+HEv73LDx74gFenLWHrtu1xjiLS+GiQXyTJalq9ubxyC89NXMhj73/JxU9NpqSogHMO68oPD+xC6+a5vDClTLPPpNFSghFJgXirN7cuyOW8IT346eHdeWPmMh59bz5/fOUz/vLGXA7o0pqJC9ZqcU1ptJRgRBqA7Czj+P07cvz+HZmxuJzH3vuS5yYt2qVd9ewzJRhpDDQGI9LA9OnUmju/3594awBo9pk0FkowIg1UTbPPvv/A+zw3cSEbq7StszRcSjAiDVSs2Wf5OVmc1G9PVlVUcdV/pnLQrf/julFTmfLVGjzefgIiaaIxGJEGqqbZZ+7OxAVreObjhbwwZTH/+mgh+3Yo5AcH7sWpB3TmnTkrNPtM0k4JRqQBizf7zMwY3K2Ywd2K+d1JvXl56hKe+Xghfxgziz++MguA6gUENPtM0kWXyEQauZb5ufz4oC688MvDGfurIyjIyyZ6dZrKLdu4Y+xn6QlQMpYSjEgT0qtjSzZu3hazbvHaTdz2yiyml5VrvEZSIqUJxsyKzex5M9tgZgvM7Iw47czMRpjZqvAxwiJ2bjKzAWY2ycw2hl8HRNRdZWbTzWy9mc03s6tS8NFEGox4s8/yc7J4+N35nPi3dzn6rrf58+uz+Xz5+p3avDCljMNvH0f3a8dw+O3jeGFKWSpCliYq1WMw9wFVQAdgADDGzD519xlR7S4AhgP9CWZlvkGwc+YDZpYHjAbuBv4OXAiMNrOe7l4FGHA2MBXYG3jdzBa6+7+T+9FEGoZYWzsX5GZz26l9GdqrPa9NX8pLUxdz71ufc8+4z9mvY0tO6t+J/Jws/vT6nB2v09iNfFMpSzBm1gI4Ddjf3SuAd83sReAs4Nqo5ucAd7n7ovC1dwHnAw8AQ8O47/agn3+PmV0JHA285u53RBxntpmNBg4HlGAkI9Q0+wzgRwd14UcHdWH5+k28MnUJL01dEndvGq0cIN+EpeparJkNBN5z9+YRZVcCR7r7SVFty4Hj3P3D8PmBwFvu3tLMfh3WfTei/cth/V1RxzFgMvCguz8QI6YLCHpLtG/fftCzzz5bT5+26aioqKCwsDDdYTQ4Te28rNi4naveib9CwKPDmhNxlTqmpnZO6ktTPy9HHXXUJHc/MFZdKi+RFQLrosrKgZZx2pZHtSsME0Z0XU3HuYlgnOnRWAG5+0hgJECvXr08co91CUTvPS+Bpnhe7p46jrI4y9D8ZsJ2vvOtDnyndwcO7dGWvJxdh2+b4jmpD5l8XlKZYCqAVlFlrYD1CbRtBVS4u5tZQscxs0sIxmKGuPvmbxK4SCaINXaTn5vFKQNLWL2hiv9MWsQTExZQ2CyHI3u157jeHRi67x68NXs5d46dTdnaSkomjNNNnbJDKhPMHCAnHIyfG5b1B6IH+AnL+gMfxWg3A7jCzMy/vr7Xj2ACAQBm9jOCcZ0jqsdxRKRmtY3dbNqyjfe/WMkbM5fxxszljJm6BAPMdFOnxJayBOPuG8xsFHCzmZ1HMIvsZOCwGM0fBy43s1cId5YF/hbWlQLbgEvN7AGCwX+AcQBmdibwR+Aod5+XnE8j0jTFWzkAID83m6P368DR+3Xg1uHOp4vWctbDH1GxeecFNyu3bOOWl2dybO8OtGimxUIyWar/9X8BPAIsB1YBF7v7DDMbArzq7tUjYQ8CPYBp4fOHwjLcvcrMhodltwOzgOHhFGWAPwBtgY8jBiWfdPeLkvnBRDJJVpYxsEsbNmyOvZrzqg1VDLj5dQZ1bcOQnu05omd7+nRqRVbW1xMFtFtn05fSBOPuqwnub4kuH08weF/93IGrw0es40wBBsWp614fsYpI7ToVFcScGNCuMI/TB+21Y9HNO8fOprhFHt/epx1DerZjQ9VWRrw6W/fcNHHqv4rIbot3U+cNJ/Rm+MASrv3ufqxYv5n3Pl/JO3NXMH7uSl78dHHMY+mem6ZHCUZEdlvkxICytZWUxLjU1b5lsx1jO+7O7GXrOf7u8TGPV7a2kjdnLmNwt2JaN89NyWeQ5FGCEZFvpDp5JHK/h5mxX8dWlMS5tAZw3uMTMYP9Orbi4O7FHNy9mMHdi2lX2AzQ2E1jogQjIikX79LazSf3oUtxcz6cv5qP5q/mmY8X8tj7XwKwzx6F7NEyj4+/XMOWbcG8aI3dNGxKMCKScrXdc3Nwj7YAVG3dzrSycj6av5oP56/i7dkriF7cqnLLNm4dM4vv9d0z5goDkj5KMCKSFjXdc1MtLyeLQV3bMKhrGy4eujfdrx0Ts92Kis3sf9NY+pa0ZuBeRQzoUsTALm3o1Dp/xxpqurSWekowItJoxJsW3aZ5LqcP6syUr9byxIQFPPTufAD2aNmMAXsVkZeTxeszl1G1dTugS2upogQjIo1GvLGb353UZ0ei2LJtO58tWc+UhWuY8tVapny1hi9XbdzlWJVbtvGHMVpxIJl0VkWk0aht7AYgNzuLvp1b07dza84+NCjrfu2YXcZuAFZWVLH/TWPp3q4F+3dqTZ9Ordi/JPha1DwP0KW1b0IJRkQalUTGbqLFu7TWtkUeZx/ajemLy5m0YM1ON4GWFBVQ3CKXWUvWs3W7Zq3tDiUYEWny4l1au/HE3jslitUbqpixuJwZi9cxvaycV6cvZdv2nfs+lVu2cf3z06jYvJVeHVuy7x4tY94UWt3zyeRtDJRgRKTJS+TSGkBxizyG9GzPkJ7tAeLOWttQtY0bXpi+43nHVvn06tgySDgdWrKkvJL73vqcTVsye1KBEoyIZIT6vLRWUpTPsxcdxpyl65m9bD2zlwaPD+at2jFTLVr1/TpH9dqjxmVwmtKYjxKMiEgc8S6tXTVsP0qKCigpKuCo/fbYUbd123YWrN7IMXe9HfN4Kyo20//m12lXmEePdoXsvUeLnb5OXrCa61+Y0WRWmVaCERGJI9FLa9VysrPYu31h3LXWilvkcdGRPfhi+Qbmraxg7IxlrN6wsMYYKrds47ZXZ/F//TvttJ9OtIbY81GCERGpwe5cWovX8/lt1KQCgDUbqpi3soIvVmzg6v9MjXm8Zes2s9+Nr9G5uIBubVvQpbg5XdsGjy7FLfjkqzXcOLrh9XyUYERE6lki2xhUa9Mij0EtihnUtZi/vjk3Zs+nqCCXHw7eiwWrNrJg9UY+nLeKDVXbdmkXqfpG0gO7taFjq3xysmOv05bMno8SjIhIEtRlG4Nq8Xo+N/1fn51+6bs7qzZUsWDVBhas2sjlz34a83grK6r49oi3yM4yOrbKp6RNAZ3bFNC5qICSNgUsWLWRh9+dz+bdWEKnOjHlddwn5u7CoAQjItJgJDrmY2a0K2xGu8JmDOpazF2vz4l7I+mVw3pRtqaSsrWVLFqzkQlfrGLpuk1sj7W0AV/f57OkfBN7ts6nY+t89mydT4dW+eTnZgNBcolOhLEowYiINCD1OeYTfSNptS3btrO0fBND7ngr5vE2VG1jxGuf7VJe3CKPjq3ymbeigk1xpmNHUoIREWnk6jrbLTc7i72Km8ed7VZSVMDrvz6Cpes2sbR8E0vKN7G0vDL8uomZS9YlFJcSjIhIE1CfPZ+rhvWiRbMc9m5fyN7tC3d53eG3j4u75XUkbf8mIpKhhg8s4bZT+1JSVIAR9FxuO7VvrYnqqmG9KAjHY2qiHoyISAbbnZ5P5CW5JTW0Uw9GRETqbPjAEt679miqln4+KV4bJRgREUkKJRgREUkKJRgREUkKJRgREUkKJRgREUmKlCYYMys2s+fNbIOZLTCzM+K0MzMbYWarwscIM7OI+gFmNsnMNoZfByT6WhERSY1U92DuA6qADsCZwP1m1idGuwuA4UB/oB9wEnAhgJnlAaOBJ4E2wD+B0WF5ja8VEZHUSVmCMbMWwGnAje5e4e7vAi8CZ8Vofg5wl7svcvcy4C7g3LBuKMENone7+2Z3vwcw4OgEXisiIimSyjv59wW2uvuciLJPgSNjtO0T1kW26xNRN9XdIxebnhqWv1bLa3diZhcQ9HgANpvZ9MQ+SkZpB6xMdxANkM7LrnROYmvq56VrvIpUJphCIHoJznKgZZy25VHtCsOxlOi66OPEfW1UUsLdRwIjAcxsorsfmPjHyQw6L7HpvOxK5yS2TD4vqRyDqQBaRZW1AtYn0LYVUBEmiNqOU9NrRUQkRVKZYOYAOWbWM6KsPzAjRtsZYV2sdjOAflEzw/pF1cd7rYiIpEjKEoy7bwBGATebWQszOxw4GXgiRvPHgcvNrMTMOgFXAI+FdaXANuBSM2tmZpeE5eMSeG1NRtb9U2UEnZfYdF52pXMSW8aeF0vllSMzKwYeAY4FVgHXuvvTZjYEeNXdC8N2BowAzgtf+hBwTfVlLjMbGJb1BmYBP3f3KYm8VkREUiOlCUZERDKHlooREZGkUIIREZGkyPgEk+j6aJnGzErNbJOZVYSP2emOKdXM7BIzm2hmm83ssai6Y8zss3A9vLfMLO7NZk1NvPNiZt3MzCN+ZirM7MY0hppS4aSjh8PfI+vN7BMz+25Efcb9zGR8giHx9dEy0SXuXhg+eqU7mDRYDPyBYGLKDmbWjmBG5I1AMTAReCbl0aVPzPMSoSji5+aWFMaVbjnAQoLVSVoDNwDPhok3I39mUnknf4MTsT7a/u5eAbxrZtXro12b1uAk7dx9FICZHQh0jqg6FZjh7s+F9TcBK81sP3f/LOWBplgN5yWjhbdi3BRR9LKZzQcGAW3JwJ+ZTO/BxFsfTT2YwG1mttLM3jOzoekOpgHZab278BfLF+jnptoCM1tkZo+Gf7lnJDPrQPA7ZgYZ+jOT6QmmLuujZZprgB5ACcGNYi+Z2d7pDanBqG09vEy1EhhMsPjhIILz8VRaI0oTM8sl+Oz/DHsoGfkzk+kJpi7ro2UUd//Q3deHWyL8E3gP+F6642og9HMTQ7gNx0R33+ruy4BLgOPMrEn/Eo1mZlkEK5RUEZwDyNCfmUxPMHVZHy3TOcG+OxK13l04lrc3+rmJVn0Xd8b8nglXEnmYYNLQae6+JazKyJ+ZjPmHj6WO66NlDDMrMrNhZpZvZjlmdiZwBMF+Oxkj/Oz5QDaQXX0+gOeB/c3stLD+twR7FDXZwdpI8c6LmR1sZr3MLMvM2gL3AKXuHn1pqCm7H/gWcJK7V0aUZ+bPjLtn9INgyuALwAbgK+CMdMeU7gfQHviYoPu+FpgAHJvuuNJwHm4i+Cs88nFTWPcd4DOgkmAB1m7pjjfd5wX4MTA//L+0hGDh2Y7pjjeF56VreC42EVwSq36cmak/M1qLTEREkiKjL5GJiEjyKMGIiEhSKMGIiEhSKMGIiEhSKMGIiEhSKMGIiEhSKMGINFHh3iynpzsOyVxKMCJJYGaPhb/gox8T0h2bSKpk9H4wIkn2JsHeQpGq0hGISDqoByOSPJvdfWnUYzXsuHx1iZmNCbfQXWBmP4l8sZn1NbM3zazSzFaHvaLWUW3OMbNp4fbFy8zsn1ExFJvZc+GW4POi30MkmZRgRNLn98CLwACCPXceD3eJrF5tdyzBWlYHAacAhxGxTbGZXQg8CDwK9CPYTmF61Hv8FhhNsJLvM8AjZtYlaZ9IJILWIhNJAjN7DPgJwcKHke5z92vMzIGH3P38iNe8CSx195+Y2fnAn4DO7r4+rB8KvAX0dPfPzWwR8KS7x9zeO3yP2939uvB5DsEGexe4+5P192lFYtMYjEjyvANcEFW2NuL7D6LqPgBOCL//FsFy7pEbUr0PbAd6m9k6gt1G/1dLDFOrv3H3rWa2AtgjoehFviElGJHk2ejunyfhuHW57LAl6rmjS+OSIvpBE0mfQ2I8nxV+PwvoG7Xd8GEE/2dnuftyoAw4JulRiuwm9WBEkqeZmXWMKtvm7ivC7081s48JNp86nSBZHBzWPUUwCeBxM/st0IZgQH9URK/oVuAvZrYMGAM0B45x97uS9YFE6kIJRiR5vkOws2OkMqBz+P1NwGkEWwuvAH7q7h8DuPtGMxsG3A18RDBZYDRwWfWB3P1+M6sCrgBGAKuBV5L0WUTqTLPIRNIgnOH1fXf/T7pjEUkWjcGIiEhSKMGIiEhS6BKZiIgkhXowIiKSFEowIiKSFEowIiKSFEowIiKSFEowIiKSFP8f9o4JO8yw41UAAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light",
      "image/png": {
       "width": 408,
       "height": 286
      }
     },
     "output_type": "display_data"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "The schedule function can take the current learning rate as a second argument:",
   "metadata": {
    "cell_id": "00126-e5e71192-defd-4795-9673-299662a5ea3a",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00127-d6ee56b0-b69d-467a-a8ba-e518b5f1c1d8",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "20a0eb85",
    "execution_start": 1659355876803,
    "execution_millis": 4,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "source": "def exponential_decay_fn(epoch, lr):\n    return lr * 0.1**(1 / 20)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "If you want to update the learning rate at each iteration rather than at each epoch, you must write your own callback class:",
   "metadata": {
    "cell_id": "00128-be51a48a-6c92-425a-8526-83abbd3b540b",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00129-310e7671-5ba6-46aa-9027-d472c6675a28",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "aefa5bdf",
    "execution_start": 1659355876818,
    "execution_millis": 323499,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 1254
   },
   "source": "K = keras.backend\n\nclass ExponentialDecay(keras.callbacks.Callback):\n    def __init__(self, s=40000):\n        super().__init__()\n        self.s = s\n\n    def on_batch_begin(self, batch, logs=None):\n        # Note: the `batch` argument is reset at each epoch\n        lr = K.get_value(self.model.optimizer.learning_rate)\n        K.set_value(self.model.optimizer.learning_rate, lr * 0.1**(1 / self.s))\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        logs['lr'] = K.get_value(self.model.optimizer.learning_rate)\n\nmodel = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.Dense(10, activation=\"softmax\")\n])\nlr0 = 0.01\noptimizer = keras.optimizers.Nadam(learning_rate=lr0)\nmodel.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\nn_epochs = 25\n\ns = 20 * len(X_train) // 32 # number of steps in 20 epochs (batch size = 32)\nexp_decay = ExponentialDecay(s)\nhistory = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n                    validation_data=(X_valid_scaled, y_valid),\n                    callbacks=[exp_decay])",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/25\n1719/1719 [==============================] - 12s 7ms/step - loss: 0.7677 - accuracy: 0.7725 - val_loss: 0.8305 - val_accuracy: 0.7626 - lr: 0.0089\nEpoch 2/25\n1719/1719 [==============================] - 11s 7ms/step - loss: 0.6341 - accuracy: 0.8052 - val_loss: 0.5845 - val_accuracy: 0.8136 - lr: 0.0079\nEpoch 3/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.5647 - accuracy: 0.8243 - val_loss: 0.5743 - val_accuracy: 0.8236 - lr: 0.0071\nEpoch 4/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.5512 - accuracy: 0.8310 - val_loss: 0.5167 - val_accuracy: 0.8176 - lr: 0.0063\nEpoch 5/25\n1719/1719 [==============================] - 12s 7ms/step - loss: 0.4752 - accuracy: 0.8497 - val_loss: 0.5107 - val_accuracy: 0.8642 - lr: 0.0056\nEpoch 6/25\n1719/1719 [==============================] - 19s 11ms/step - loss: 0.4148 - accuracy: 0.8681 - val_loss: 0.4661 - val_accuracy: 0.8658 - lr: 0.0050\nEpoch 7/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.3942 - accuracy: 0.8740 - val_loss: 0.4503 - val_accuracy: 0.8648 - lr: 0.0045\nEpoch 8/25\n1719/1719 [==============================] - 12s 7ms/step - loss: 0.3642 - accuracy: 0.8812 - val_loss: 0.4791 - val_accuracy: 0.8554 - lr: 0.0040\nEpoch 9/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.3252 - accuracy: 0.8904 - val_loss: 0.4403 - val_accuracy: 0.8672 - lr: 0.0035\nEpoch 10/25\n1719/1719 [==============================] - 11s 7ms/step - loss: 0.3168 - accuracy: 0.8959 - val_loss: 0.4257 - val_accuracy: 0.8848 - lr: 0.0032\nEpoch 11/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.2874 - accuracy: 0.9029 - val_loss: 0.4471 - val_accuracy: 0.8708 - lr: 0.0028\nEpoch 12/25\n1719/1719 [==============================] - 12s 7ms/step - loss: 0.2674 - accuracy: 0.9093 - val_loss: 0.4681 - val_accuracy: 0.8654 - lr: 0.0025\nEpoch 13/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.2503 - accuracy: 0.9139 - val_loss: 0.4463 - val_accuracy: 0.8764 - lr: 0.0022\nEpoch 14/25\n1719/1719 [==============================] - 12s 7ms/step - loss: 0.2343 - accuracy: 0.9205 - val_loss: 0.4423 - val_accuracy: 0.8812 - lr: 0.0020\nEpoch 15/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.2135 - accuracy: 0.9277 - val_loss: 0.4445 - val_accuracy: 0.8846 - lr: 0.0018\nEpoch 16/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.1977 - accuracy: 0.9318 - val_loss: 0.4533 - val_accuracy: 0.8862 - lr: 0.0016\nEpoch 17/25\n1719/1719 [==============================] - 12s 7ms/step - loss: 0.1854 - accuracy: 0.9359 - val_loss: 0.4733 - val_accuracy: 0.8810 - lr: 0.0014\nEpoch 18/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.1697 - accuracy: 0.9421 - val_loss: 0.4940 - val_accuracy: 0.8860 - lr: 0.0013\nEpoch 19/25\n1719/1719 [==============================] - 12s 7ms/step - loss: 0.1600 - accuracy: 0.9451 - val_loss: 0.4916 - val_accuracy: 0.8872 - lr: 0.0011\nEpoch 20/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.1455 - accuracy: 0.9511 - val_loss: 0.5125 - val_accuracy: 0.8856 - lr: 9.9967e-04\nEpoch 21/25\n1719/1719 [==============================] - 12s 7ms/step - loss: 0.1386 - accuracy: 0.9541 - val_loss: 0.5435 - val_accuracy: 0.8878 - lr: 8.9094e-04\nEpoch 22/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.1295 - accuracy: 0.9565 - val_loss: 0.5668 - val_accuracy: 0.8912 - lr: 7.9404e-04\nEpoch 23/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.1203 - accuracy: 0.9611 - val_loss: 0.5791 - val_accuracy: 0.8878 - lr: 7.0767e-04\nEpoch 24/25\n1719/1719 [==============================] - 12s 7ms/step - loss: 0.1137 - accuracy: 0.9636 - val_loss: 0.6111 - val_accuracy: 0.8886 - lr: 6.3071e-04\nEpoch 25/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.1077 - accuracy: 0.9653 - val_loss: 0.6072 - val_accuracy: 0.8862 - lr: 5.6211e-04\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00130-ede552c8-7349-4776-8be3-a1a85702aa80",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "625beef",
    "execution_start": 1659356200358,
    "execution_millis": 1,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 112
   },
   "source": "n_steps = n_epochs * len(X_train) // 32\nsteps = np.arange(n_steps)\nlrs = lr0 * 0.1**(steps / s)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "scrolled": true,
    "cell_id": "00131-13cbbd99-183f-4ed8-ba39-e99b49ce8879",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "e9610a53",
    "execution_start": 1659356200359,
    "execution_millis": 196,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 488,
    "deepnote_output_heights": [
     288
    ]
   },
   "source": "plt.plot(steps, lrs, \"-\", linewidth=2)\nplt.axis([0, n_steps - 1, 0, lr0 * 1.1])\nplt.xlabel(\"Batch\")\nplt.ylabel(\"Learning Rate\")\nplt.title(\"Exponential Scheduling (per batch)\", fontsize=14)\nplt.grid(True)\nplt.show()",
   "execution_count": null,
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEeCAYAAAC30gOQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABAeElEQVR4nO3dd3wUdfrA8c+T3hOS0DvSe7MCAnY9C+p5nngqoof1vLMcP892Hnoqep7tFMGGig1PsXsoSkBQERCEUKWH0BMISUgIhOf3x0xwXTbJJmR3U5736zWvZOf7ndlnJpt5dub7ne+IqmKMMcbUtLBQB2CMMaZ+sgRjjDEmICzBGGOMCQhLMMYYYwLCEowxxpiAsARjjDEmICzBmDpPREaJSEEVl8kQkf8EKib3PTaIyB0BWO9vRaRK9xd476Pq7LOjISJ/F5GXg/V+Pt5fReS3IXjfSveziNwkIh8HK6ZgsgRTh4nIZPcfx3v6PtSxBUo5B4p3gA4BeK9rRWSRiBSISJ6ILBGRB2v6fUIkIPvMFxFpAtwO1Ol9JyL3i0hmAFb9IjBARIYEYN0hFRHqAMxRmwFc4TWvJBSBhIqqFgFFNblOERkNPA3cCnwFRAI9gRNr8n1CJRD7rALXAj+o6rpAv5GIRKrqgUC/T01S1f0i8iZwC/BNqOOpSXYGU/ftV9VtXlMugIgMFZEDIjKsrLKIXCcie0Wkg/s6Q0SeF5GnRGS3Oz0mImEeyzQSkVfdsiIRmSEiPTzKR7nf8k8VkUwRKRSRmSLS3jNQETlPRBaKSLGIrBeRf4pIlEf5BhG5R0QmujFuFpG/epa7v77rnsls8Hx/j3rHiMiHIrLNjeVHETm3ivv1fOB9VZ2oqmtUdYWqvquqt3lt0zkiMs/dLzki8rGIxHhUiSlve9zlk0VkkojsEJF8EZklIgO96lwpIhtFZJ+IfAI09So/4pt1ZZdmfOyz+92/3e9FZK0bywciku5RJ0JEnvD4nDwhIhNEJKOSfTkS+NUlID8/d1EiMt7db/tEZL6InOlRPsz9HJwjIj+ISAlwJuVrJiKfuuvaKCJ/8IrpERFZ5f4tN4jIo2V/SxEZBfwd6CG/XCkY5ZYlu/thq/vZXiEil3qtu8L/DeAj4HwRiatkX9YtqmpTHZ2AycAnldR5CMgCGgFdgULgKo/yDCAfeMYt/x2QB9zmUedDYCVwMtAL558hC4h1y0cBB3DOpo4DegOLgOke6zgT2AtcDRwDDAdWAf/yqLMByAFuBjoCfwIUONEtb+y+vhZoBjT2eP8Cj/X0Aa53Y+0I3I1zVtfVa7v/U8F+ex5YDXSooM5ZwEGcSz/d3e2+A4jzc3sEmAN86u63jsAD7n5q7tY5HjjkbkNn4Dp3neoRx/1Aplds3vukstf3AwXANHc7TgQ2AhM96twJ7AYuBroAT7mflYwK9lGqG/8gr/kZVP65ewP4Hudz18HdjyVAH7d8mLs/lwJnuHUalxOHuvvtOnc/3u3GNdCjzr3AIKAdcA6wCXjALYsF/oXzf9DMnWLdv+FcYLn7eegAnA1c6O//hlsvDigFTg31caVGj1GhDsCmo/jjOQnmoHtg8JzGe9SJBOYD7wM/Au94rSMD50AqHvPuATa7v3dy/zlP9ihPdg8G17qvR7l1unjUuRzYX7ZeYDZwr9d7j3DjLauzAXjLq87PwD0erxX4rVedUXgcLMvZV997rSeDihNMc+A79/1+BqYAVwKRHnXmAm9XsI4Ktwc4xd3+WK86i4Gx7u9vAl96lb9IYBJMMZDsMe9uYI3H663AnR6vBedLQkYF+6Cvuw/bV/FzdwxOAmjjtdwHwHPu78PcdV/sx/+KAi94zZsBTKlgmeu9tt/Xfj7djbNbOesYRSX/Gx7zc4FrKtuWujTZJbK6bzbOP7Hn9FhZoTrXo0cC5wJNcL7Befte3U+46zugpYgkAd1w/oG+81hnHs63xu4ey+xX1VUer7cAUThnTgADgLvdS2kF7uWZN4F4nG+DZZZ4xbbFjdtvIhLvXt5Y7l56KQAGAm38XYeqblXVE3HOgp7EOZhOBH7wuIzRD6d9piIVbc8AnG+uO732S0+cAyw4+/87r3V4v64pG92/7RGxikgyzt/ph7JC9zPzAxWLdX8W+yir6HPXH2efL/faN7/hl31TZkElMXiu3/v14c+wOL3z5riXVguAJ6j8M9MP2KqqKyqoU9n/Rpkiftlf9YI18td9+1R1TSV1TsBpb0vBucy0p4be2/PgcLCcsjCPn/8A3vWxnp0ev3s30CpVbyv8F87lijtwzhj2Aa/h/FNXiapmApnAsyIyGKcR9nc4Z4/+qGh7woDtgK/eQ3urEOYhnIOxp8gqLF+mJva9t13uz0Y4Z0D+CnPf/1gfcXl3TiisXmi/EJETgLdxPqO34vyPnI/zWTpalf1vlEnl1/8LdZ6dwdRzbmPif4CbgC+BKSLi/cXieBHxPECdAGxR1b3ACpzPyeHeU+43zF4415399SNOG8gaH5P3P2BFDgDhldQZDLymqu+p6hJgM0d+662Osu1NcH8uAk49ivX9iNNgf8jHPtnh1lmB8/fw5P16J9DU62/Y9yjiOoJ7ZrMN54APgPt+x5a7kGMtTrLs7qOsos/dIpyk2czHvsmu5mb42o9lZx6DgGxVfUBV56vqz0Bbr/olHPnZWwQ0F5Fu1YwJcDqmADE4n4l6w85g6r5oEWnmNa9UVXeKSDjwOjBLVSeKyH9xLm39HadBs0wL4EkReQ4ncfwV954FVf1ZRD4EJorIGJxvdv/EOWi8WYU4xwGfiMhGYCrOt7qewHGqOrYK69kAnCois3AuPez2UWc1cKEb9wGc7Y3xUa9cIjIB51LG1zgJqjlOG8E+4Au32j+Bj0VkDc6+EJzG5omqus+Pt5mB047zoYiM5ZcG5LOAGar6DU5X6W9F5G/Af3HaHS70Wk8Gzrffu0TkbbdOIG4qfAoYKyKrcZLtdTj7pdwzE1U9JCIzcJL+f72KK/rcrRaRN4DJInI7zoE3FWfb1qnq+9WI/yIRmY+zv36L8+XgeLdsNc7luctxLp2dCVzmtfwGoK2I9MfpAJCPc4l0HvCeiNzqrqcjEK+qH1QhtiE42/Vz1Ter9rIzmLrvNJx/cM9pkVt2F86H/RoAVc0BrgLudC/3lHkD55vZPOAF4CWc689lrsa51v6R+zMOOEudeyn8oqrTca6fD3fX8QNOr6RN/m8q4NywNxynF9uicurcBuzAuZz1OU4Df1XvL/gS5+AzFeegMc2df7qqrgZQ1c9wDvZnu7HMcmM75M8buO0P5+AksRdwGsyn4vTQ2uLW+R7n73cDTnvORTiNzZ7rWeGWj3HrnI7Te7Cm/QvnC8srOPsUnP3iq33F0yTgUvcLjyd/PnevAI/iJN9PcHqUbaxm/Pfj9IBbgrO/rlbV+QCq+jFO2+WT/LIP7/Na/j3gM5ykshO4TFUP4fz95+J0BFmBk4irejn2Mpx9UK+U9d4xDZR7D0Omqt4c6lhM3SMii4A5qvqnSup9h9P763X3dQb2uQNARHriJK3OXp0s6jy7RGaM8YuItMW5dDQLpxPBH3Hu6/ijH4tfh9PjyhypBXBlfUsuYAnGGOO/Qzj3Aj2Gc3l9OXC2qlbaTdjtbOHdZdsAqvpF5bXqJrtEZowxJiCskd8YY0xA2CUyV0pKinbs2DHUYfhUWFhIfHx8qMPwyWKrHouteiy26glkbAsXLtylqo19FoZ6rJraMnXu3Flrq5kzZ4Y6hHJZbNVjsVWPxVY9gYwNWKA2FpkxxphgsgRjjDEmICzBGGOMCQhLMMYYYwLCEowxxpiAsARjjDEmICzBGGOMCQhLMMYYYwLCEowxxpiAsARjjDEmICzBGGOMCQhLMMYYYwLCEowxxpiAsARjjDEmICzBGGOMCYigJhgRSRWRaSJSKCIbRWRkOfVERMaLSI47jRcR8SifJCKrROSQiIzysfytIrJNRPaKyMsiEh3AzTLGGONDsM9gngVKgKbA5cAEEenho94YYATQB+gNnAdc51H+E3Aj8KP3giJyJnAncCrQFugA/KOywA5pFbbCGGNMpYKWYEQkHrgYuFdVC1R1DvARcIWP6lcBj6vqZlXNBh4HRpUVquqzqvoVUFzOsi+p6jJV3Q084LlseTYXHGLPvpIqbpUxxpjyiPPEyyC8kUg/YK6qxnnMuwMYqqrnedXNA85Q1Xnu64HATFVN9Ko3B3hRVSd7zPsJeEhV33FfpwM7gXRVzfFafgzO2RJRzToOGHnPf7iqR+27mlZQUEBCQkKow/DJYqsei616LLbqCWRsw4cPX6iqA32VRQTkHX1LAPZ6zcsDEsupm+dVL0FERCvPiL6WxX2fXyUYVZ0ETAKIbt5JMzYf5NYLjqd3q5RK3iK4MjIyGDZsWKjD8Mliqx6LrXostuoJVWzBbIMpAJK85iUB+X7UTQIK/Egu5S1LOe/zS6UoQRXu/SCTQ9YgY4wxRy2YCWY1ECEinTzm9QGW+ai7zC2rrJ4vvpbd7n15zFtKtNAsKYafNufx9vwsP9/KGGNMeYKWYFS1EHgfGCci8SIyCLgAeN1H9deA20SkpYi0AG4HJpcVikiUiMQAAkSKSIyIhHkse42IdBeRFOAez2XLEyZwz7ndAHh0+kpyC63B3xhjjkawuynfCMQCO4C3gBtUdZmIDBGRAo96E4GPgaVAJvCpO6/MF0ARcBJOG0oRcDKAqv4PeBSYCWwCNgJ/9ye43/RqzuCO6ezZd4DHpq+s9kYaY4wJcoJR1VxVHaGq8araRlXfdOd/o6oJHvVUVceqaqo7jfVsf1HVYaoqXlOGR/m/VbWpqiap6tWqut+f+ESE+8/vQWS48Pb8LBZt2l2DW2+MMQ2LDRXjpWOTBK4d0sFp8P8wk1Jr8DfGmGqxBOPDn07pSIvkGDKz9/LmvI2hDscYY+okSzA+xEVFcN953QF49H+r2L7X14ABxhhjKmIJphxn9mjGqV2bkL//IP/42N8e0sYYY8pYgimHiDBuRE/iosL5bOk2ZizfHuqQjDGmTrEEU4GWKbHccUYXAO77MJOC/QdDHJExxtQdlmAqcdVJ7ejdKpktecU8/sWqUIdjjDF1hiWYSoSHCQ9f1IvwMGHytxtYnLUn1CEZY0ydYAnGDz1aJHPt4Paowt/eX8qB0kOhDskYY2o9SzB++vNpnWidGsuKrXt5ac76UIdjjDG1niUYP8VFRfDgiF4APDljNRtzCkMckTHG1G6WYKpgaOfGjOjbguIDhxj73yX23BhjjKmAJZgquu+8HqQnRDFvfS5TbBgZY4wplyWYKkqNj+LBET0BeOTzlWTl7gtxRMYYUztZgqmGs3o259zezdlXUmqXyowxphyWYKrpH+f3IC0+iu/W5fDmD5tCHY4xxtQ6lmCqKS0hmnEXOJfKHv5sBZt326UyY4zxZAnmKPymd3PO6dWMwpJS7nxvKR4P3TTGmAbPEsxRGndBTxrFRTJnzS7enp8V6nCMMabWsARzlNITovmHe6nswU+WW68yY4xxWYKpAed5XCq7bepiSq1XmTHGWIKpCSLCP0f0okliNPM37GbS7HWhDskYY0LOEkwNaRQfxWOX9AHg31+uYtmWvBBHZIwxoWUJpgYN7dyYK09sy4FS5dZ3FlN8oDTUIRljTMhYgqlhfzu7Gx3S41m9vYDHptsTMI0xDZclmBoWGxXOE5f2JTxMeGnOeuau2RXqkIwxJiQswQRAn9Yp3HJKJwDuePcn8ooOhDgiY4wJPkswAXLT8GPo0zqFrXnF3DXN7vI3xjQ8lmACJCI8jKcu7Ut8VDifLtnK1AV2l78xpmGxBBNA7dLjefBC5y7/v3+0jDU78kMckTHGBE9QE4yIpIrINBEpFJGNIjKynHoiIuNFJMedxouIeJT3FZGFIrLP/dnXoyxaRJ4Xke0ikisiH4tIyyBsnk8X9mvFRf1aUnzgEDe/uci6LhtjGoxgn8E8C5QATYHLgQki0sNHvTHACKAP0Bs4D7gOQESigA+BKUAj4FXgQ3c+wJ+BE93lWgC7gWcCszn+GTeiJ+3S4li5LZ+HPlsRylCMMSZogpZgRCQeuBi4V1ULVHUO8BFwhY/qVwGPq+pmVc0GHgdGuWXDgAjgSVXdr6pPAwKc4pa3B6ar6nZVLQbeAXwlsaBJiI7gmcv6ExkuvPbdRqYv2xbKcIwxJigkWL2bRKQfMFdV4zzm3QEMVdXzvOrmAWeo6jz39UBgpqomisitbtnZHvU/ccsfd+s+BVwC7AFeBHao6l98xDQG52yJxo0bD5g6dWpNbvIRpm84wFsrS4iPhHEnxZIW619+LygoICEhIaCxVZfFVj0WW/VYbNUTyNiGDx++UFUH+iqLCMg7+pYA7PWalwckllM3z6tegtsO413mvZ6fgSwgGygFlgI3+wpIVScBkwC6dOmiw4YN83NTqmeoKtsmz2fmqp28szGWN/94PBHhlSeZjIwMAh1bdVls1WOxVY/FVj2hii2YbTAFQJLXvCTAV9cq77pJQIE6p1uVredZIBpIA+KB94HPjyryGiIi/OuSPjRJjOaHDbn8+8vVoQ7JGGMCxu8EIyJni8gnIrJcRFq7864VkVP9XMVqIEJEOnnM6wMs81F3mVvmq94yoLdnrzKcBv2y8r7AZFXNVdX9OA38x4lIup9xBlRaQjRPX9aPMIHnMtYyY/n2UIdkjDEB4VeCEZHLgak4l5/aA5FuUTgw1p91qGohztnEOBGJF5FBwAXA6z6qvwbcJiItRaQFcDsw2S3LwLn0dYvbJbns8tfX7s/5wJUikiwikcCNwBZVrTWDgp3QIY2/ntkVgNumLranYBpj6iV/z2DGAn9U1VuBgx7zv8c5Y/DXjUAssAN4C7hBVZeJyBARKfCoNxH4GKf9JBP41J2HqpbgdGG+EqcRfzQwwp0PcAdQjJMMdwLnABdWIcaguO7kDpzWrSl7iw9ywxsL7f4YY0y9428jfyfgOx/zfbWHlEtVc3GSg/f8b3Aa78teK05S83l2pKqLgAHllOXg3GNTq4WFCY9f0odz//MNmdl7GffJch66sFeowzLGmBrj7xnMFqCzj/knA2trLpyGJTkukgmXDyAqIow3523i/R83hzokY4ypMf4mmEnA0267CUBrEbkKeBSYEJDIGoieLZP5x/nOfaB3TVvKqm02Xpkxpn7wK8Go6qM4DfRf4nT9nQk8Dzyvqs8GLryG4ffHtuai/s54ZddPWWjPjzHG1At+d1NW1buBdOA44ASgsareG6jAGhIR4Z8jetG1WSLrdxXyl7cXUXrInh9jjKnb/O2m/LKIJKrqPlVdoKo/qGqB29345UAH2RDERoXzwpUDSYmLZOaqnTxhN2EaY+o4f89grsLpXuwtFqe7sKkBrVPjeHZkf8IE/jNzDZ8t3RrqkIwxptoqTDDu81vScEYrbuS+LpsaA+cCdit6DRrUMZ27zukGwO1Tf2LFVu/h24wxpm6o7AxmF85NkQosx7lxsWzahjNS8XOBDLAhumZwey7s15KiA6WMeX0BBSXWHmOMqXsqu9FyOM7Zy9c4z3LJ9SgrATaq6pYAxdZgiQgPX9SLNTsKWJqdx4Sfwjjr1EN+jbxsjDG1RYVHLFWdpaoZOOOPfei+Lpu+s+QSODGR4Uy8YgBp8VEsyznEQ5+tDHVIxhhTJf7eB7NRVQ+JSAsROUFETvacAh1kQ9UiJZYJfxhAuMDLc9fzxryNoQ7JGGP85tdYZO6Ixm/iDA2jOJfNPBsGwms+NANwXPtURvWI4qXMEu77cBltU+MZ3KlWPHnAGGMq5O9F/SdxhsjvDuwDhuA8kngFcFZAIjOHDWkVyfVDj6H0kHLDGwtZs8OGkzHG1H7+JpihwP+p6kqcM5edqvo+8H/AA4EKzvxi7JldOKtHM/KLDzJ68gJyC0sqX8gYY0LI3wQTi9NlGZyeZE3c35fjPE3SBFhYmPDvS/vQq2Uym3L3cd3rC9h/0J4hY4ypvfxNMCuBru7vi4HrRaQtcBOQHYC4jA9xURG8eNVAmiXFMH/Dbu58bynOo3OMMab28TfBPAU0c38fB5wBrMN5QuVdAYjLlKNpUgwvXjWQ2Mhwpi3K5qmvfg51SMYY45O/3ZTfUNXJ7u8/Au2AY4E2qvpuwKIzPvVsmczTl/UjTODJGT/zzvxNoQ7JGGOOUK1bw91RlX8ECkXkzhqOyfjh9O5NGXdBTwDumpbJ1yttSDhjTO1SaYIRkXQR+Y2InCEi4e68SBH5C7ABuCOwIZry/OGEttw8vCOlh5Sb3ljE4qw9oQ7JGGMOq2w05ZOAn4GPgc+BuSLSFVgC3IzTRblNoIM05bv9jM78dkArig6UMnryfDbsKgx1SMYYA1R+BvMAMB2nK/KTOE+z/AR4GOikqv9R1X0BjdBUqGxgzKGdG5NbWMKVL//Azvz9oQ7LGGMqTTB9gAdUNRO4F+cmy7+p6mtq/WNrjcjwMJ67vP/he2RGT55P4f6DoQ7LGNPAVZZgUnGe/YJ7prIPWBTooEzVxUdH8PKoY2mTGsfS7DzGvL6A4gN2I6YxJnT86UXWyOPJlgokeT3ZMjXAMRo/NU6M5rXRx5GeEM3cNTnc8tYiDpYeCnVYxpgGyp8EU/Ykyx1AAjCfX55qucv9aWqJdunxTLn2OJJjI/li+XbG/ncJhw7Z1UxjTPD580RLU8d0bZbEK1cfyx9enMf7i7JJjIng/vN7ICKhDs0Y04BUmGBUdVawAjE1q3+bRrxw5UCufmU+r363kaTYSG4/o0uowzLGNCD2kPd6bFDHdJ4Z2Y/wMOGZr9cwafbaUIdkjGlAgppg3E4B00SkUEQ2isjIcuqJiIwXkRx3Gi8e13dEpK+ILBSRfe7Pvl7L9xeR2SJSICLbReTPAd60WuvMHs147LfOExUe+mwlr39vj102xgRHsM9gngVKgKbA5cAEEenho94YYATOfTi9gfOA6wBEJAr4EJgCNAJeBT505yMi6cD/gIlAGtAR+CJgW1QHXNS/FeMucHbzvR9k8uY8GxzTGBN4QUswIhIPXAzcq6oFqjoH+Ai4wkf1q4DHVXWzqmYDjwOj3LJhOG1HT6rqflV9GhDgFLf8NmC6OwL0flXNV9UVAduwOuLKE9tx77ndAbhr2lKmzs8KcUTGmPpOgnVDvoj0A+aqapzHvDuAoap6nlfdPOAMVZ3nvh4IzFTVRBG51S0726P+J2754yLyNbAU53ECHYF5wE2qesTXdhEZg3O2ROPGjQdMnTq1Zje6hhQUFJCQkFAj6/p8/QHeWVWCANf0imJwy8haE1tNs9iqx2KrnoYa2/Dhwxeq6kBfZZV1UwZARF4up0iBYmAN8I6qbqlgNQnAXq95eUBiOXXzvOoluO0w3mXe62kF9AdOx0k0jwJvAYOOCF51EjAJoEuXLjps2LAKwg+djIwMaiq2YcOg/ay1PPL5Sl7KLKF7t25c1L9VrYitplls1WOxVY/FdiS/EgzQGBgCHAIy3Xk9cS5NLQQuAsaJyBBVXVzOOgqAJK95SUC+H3WTgAJVVRGpbD1FwDRVnQ8gIv8AdolIsqp6J6YG6fqhx1B6SHls+iruePcnwsOEC/q2DHVYxph6xt82mLk4w/W3UtWTVfVknDOFz3Aa0NsCn+K0lZRnNRAhIp085vUBlvmou8wt81VvGdDbs1cZTkeAsvIlOGdWZew2dh9uGt6R207vzCGFW99ZzAeLskMdkjGmnvE3wfwZGOc5NL/7+z+BW1W1BBgP9C1vBapaCLyPc6YTLyKDgAuA131Ufw24TURaikgL4HZgsluWAZQCt4hItIjc7M7/2v35CnCh25U5EmcU6Dl29nKkW07txF9O6+QkmamL7dHLxpga5W+CSQCa+5jfzC0Dp32lsktuNwKxOOOavQXcoKrLRGSIe+mrzESch5wtxbkk96k7DzeZjQCuBPYAo4ER7nxU9WvgLneZHTgN/T7vtzHwl9M689czu6AK//feUibPXR/qkIwx9YS/bTDTgJdEZCzOYJfg9NJ6FOesBJyHka2uaCWqmouTHLznf8MviQr3WTNj3cnXehYBAyp4nwnAhIpiMb+4aXhHYiPDGffJcu7/eDnFBw9x/dBjQh2WMaaO8zfBXA/8G+fmxrJlDgIvA3e4r1cAf6zR6EzQjB7cnpjIcO7+YCmPfL6SopJS/nJaJxsg0xhTbX4lGLe95XoRuR0o+2q71m1XKauzuObDM8E08vg2xESGcce7P/HUVz9TfKCUO8/uaknGGFMt/p7BAIcb6pcEKBZTC1zUvxXREeH8+e1FTJy9joL9Bxl3QU/CwyzJGGOqxt8bLWNwepKdCjTBq3OAqvau+dBMqPymd3OiI8K48c0feWPeJnbvK+GJS/sSHREe6tCMMXWIv2cwzwEXAu8C32L3ltR7p3Vvyuujj+PaVxfw2dJt7Nk3n4lXDCAx5uiGljHGNBz+JpgRwCWqOiOAsZha5vgOabxz3Ylc9coPfLs2h8te+J5XRh1H48ToUIdmjKkD/L0PZh9gw+82QN1bJPHe9SfRLi2OzOy9XPL8t2Tl7qt8QWNMg+dvgnkU5856a+ltgNqkxfHu9SfRo0USG3L2cdGEb1m+xXvcUmOM+TV/E8zpwKXABhH5XEQ+8pwCGJ+pJRonRvP2mBM4sUMaO/P387uJ3zFr9c5Qh2WMqcX8TTC7cO7m/xrYBuR4TaYBSIyJ5JWrj+Xc3s0p2H+Q0ZPnk5F1INRhGWNqKX9vtLw60IGYuiEmMpynf9+PNqlxPJexlsnLSoj5fCVjz+xCmN0rY4zxELRHJpv6IyxMGHtWVx65qBdhAs/PWsuf3lpE8YHSUIdmjKlFyj2DEZElOI8z3i0iS6ng3he70bJh+v1xbdi16WcmLj3Ip0u3sjWviBeuHEhagnVjNsZUfInsPWC/+/t/gxCLqYN6pofz7g3HMvqV+fy4aQ8XPvctL141kM5NfT0J2xjTkJSbYFT1H75+N8Zb12ZJTLtpENe8Op/M7L1c+Oxcnvx9P07v3jTUoRljQsjaYEyNaJoUw7vXncS5vZtTWFLKmNcX8OzMNTiP9jHGNER+JRgRSRWRCSKyWkT2iMhezynQQZq6ITYqnGcu68dfz+wCwGPTV/GntxZRVGKN/8Y0RP6ORfYS0A+YBGzBBrs05RARbhrekS5NE/nz24v4ZMlWNuQUMumKgbRIiQ11eMaYIPI3wZwKnK6q8wIZjKk/TuvelGk3DeKPry0gM3sv5/9nDs+O7M/xHdJCHZoxJkj8bYPZARQEMhBT/3RumsiHNw1iUMc0dhWUMPLFebwwe521yxjTQPibYO4GxolIQiCDMfVPSlwUr159HNcPPYbSQ8o/P1vBDVN+JL/Yhpgxpr7z9xLZPUA7YIeIbAR+dXSwGy1NRSLCw7jz7K70a5PCHVN/4n/LtrFqez7P/2EAXZrZ/TLG1Ff+Jhi70dIctTN7NKPznxK5YcpCVm7LZ8Szc3n4ol6M6Ncy1KEZYwKg0gQjIpFAPPCsqm4MfEimPmufHs+0Gwdx9wdLef/HbP7yzmIWbMzlnt90JyYyPNThGWNqUKVtMKp6ALgBsKFyTY2IjQrn8Uv68NCFvYgKD2PK95sY8exc1uzID3Voxpga5G8j/xfAKYEMxDQsIsLI49vw/o0n0T49npXb8jnvmblMnZ9lvcyMqSf8bYP5CnhIRHoDC4FCz0JVfb+mAzMNQ8+WyXz8p8Hc90Em7y/KZux7S/hmzS7+eWFPkmIiQx2eMeYo+Jtg/uP+vMVHmQJ28dxUW0J0BP++tC+DO6VzzweZfPzTFhZn7eaZy/rTt3VKqMMzxlSTX5fIVDWsgsmSi6kRF/Vvxae3DKFnyySycov47YRveXbmGkoP2SUzY+oiG03Z1Crt0+N574aTGD2oPQcPKY9NX8XvJn7HxpzCyhc2xtQqficYEWkkIiNF5E4Ruc9zqsI6UkVkmogUishGERlZTj0RkfEikuNO40VEPMr7ishCEdnn/uzrYx1RIrJCRDb7G5+pHaIjwrnvvO68Nvo4miZFs3Djbs5+6hvenLfJOgAYU4f4O1z/CcAa4F/AA8BonOFj7gB+W4X3exYoAZoClwMTRKSHj3pjgBFAH6A3cB5wnRtLFPAhMAVoBLwKfOjO9/RXYGcVYjO1zMmdGzP9LydzXp8W7Csp5a5pSxk9eT479haHOjRjjB/8PYN5DHgDaAkU43RZbgMsAMb7swIRiQcuBu5V1QJVnQN8BFzho/pVwOOqullVs4HHgVFu2TCczglPqup+VX0a5x6dw92oRaQ98AfgYT+3z9RSKXFRPHNZP56+rB/JsZHMXLWTM5+czWdLt4Y6NGNMJcSfSw4ikgccq6qrRWQPcKKqrhCRY4E3VbWTH+voB8xV1TiPeXcAQ1X1PB/vd0bZ4wFEZCAwU1UTReRWt+xsj/qfuOWPe7x+CdgNTFHVVuXENAbnbInGjRsPmDp1aqX7IhQKCgpISKid44wGM7bdxYd4aWkJmTnOA8yObRbOH7pFkxzt+x5g22/VY7FVT0ONbfjw4QtVdaCvMn+7KZd4/L4daAuswBnCv4Wf60gAvJ9+mQf4Gu0wwS3zrJfgtsN4l/1qPSJyIRCuqtNEZFhFAanqJJyHqNGlSxcdNqzC6iGTkZGBxeYYcaYy5fuNPPz5SuZvK+XnvQf4+3ndGdG3JR7NdCGJrSostuqx2KonVLH5e4nsR+BY9/cM4EERuQp4Glji5zoKgCSveUmAr/FBvOsmAQXqnG6Vux73Mtyj+L5fx9QDIsIVJ7Zj+l9OZkindPbsO8Ct7/zE6Mnz2bKnKNThGWM8VOV5MFvc3+/BaTx/BqeRfYyf61gNRIiI5+W0PsAyH3WXuWW+6i0Desuvv672dud3wnmswDcisg14H2guIttEpJ2fcZo6oHVqHK+NPo5Hf9ubpJgIZq7ayRlPzOaNeRs5ZPfNGFMr+Huj5QJVnen+vlNVz1bVJFUdqKpL/VxHIc4Bf5yIxIvIIOAC4HUf1V8DbhORliLSArgdmOyWZQClwC0iEi0iN7vzvwYygdZAX3e6FueSXl8gy584Td0hIvxuYGtm3DaUM7o3pWD/Qe6elsllL3zPmh32AFZjQq1KN1qKyEARudS9FIWbKPxtxwG4EYjFeQTzW8ANqrpMRIaIiOcRYSLwMbAUJ2l86s5DVUtwujBfCezB6TI9QlVLVPWgqm4rm4Bc4JD7urQq22rqjiZJMUy8YgDPjuxPekIU89bncvZTs3lvdQnFB+zPbkyo+JUcRKQpzr0nx+GMPdYJWAf8G6fb8p/9WY+q5uIkB+/53+A03pe9VmCsO/lazyJggB/vlwH47EFm6hcR4Te9m3PSMWk8On0lb/2QxcfrDrD4iVmMO78nw7s2CXWIxjQ4/p7BPIFzqSkN2Ocx/13gjJoOypjqahQfxcMX9ea9G06kdWIYWblFXD15Pte/vtA6ARgTZP4mmFOBu1V1t9f8tTg3XBpTqwxom8r9J8Zwz2+6ERcVzv+WbeO0f8/ihdnrKDl4KNThGdMg+JtgYvn1vTBlGuNcIjOm1gkPE64d0oGvbh/K2T2bsa+klH9+toKznprNzFU7Qh2eMfWevwlmNr8M1QKgIhIO/B/Ow8iMqbWaJ8cy4Q8DeOXqY+mQHs+6nYVc/cp8rn7lB9butN5mxgSKvz3AxgKz3KFhonHGBusBJAODAhSbMTVqeJcmDDomnde+28BTM35m5qqdfPPzbK46qR23nNqJ5Fh7gqYxNcnf+2CWA72Ab4EvgBicBv5+qro2cOEZU7OiIsK4dkgHZv51GJcd15pSVV6as57h/8rgjXkbOVhq7TPG1BS/74Nx7yX5u6qeq6rnqOo9QJSI1M4RIo2pQHpCNA9f1JuPbx7Mce1TyS0s4e5pmZz91DfMWL7dnjtjTA042idapuAMwW9MndSzZTLvjDmB/4zsR+vUWH7eUcC1ry3g0onfs3Cjd6dJY0xV2COTTYMnIpzbuwUzbhvKfed2p1FcJD9syOXiCd9y/esLrSOAMdVkCcYYV3REOKMHt2fW2OHcNPwYYiLD+N+ybZzxxGzumrbUnqRpTBVZgjHGS1JMJH89sysZdwzn98e2RlV5c94mhjw6kwc/Wc6ugv2hDtGYOqHCbsoi8lEly3s/l8WYeqNZcgyPXNyba4e057Hpq5i+bDsvzlnPG/M2cdVJ7bju5A40io8KdZjG1FqV3QeT40f5+hqKxZhaqWOTRCZeMZDM7Dye+HI1X63cwfOz1vL6dxsYPbg91w7uQHKc3UNjjLcKE4yqXh2sQIyp7Xq2TOalUceyOGsPT3y5mlmrd/LM12uY/O0GrhncnqtPam+JxhgP1gZjTBX1bZ3Cq6OP470bTmRwx3Tyiw/y5IyfGTT+ax7+fAU78q0zgDFgCcaYahvQNpUp1x7PO2NOYEindAr2H2TirHUMHj+Tez/IJCt3X+UrMaYeq8rTKI0xPhzfIY3jO6TxU9YenstYw/Rl23n9+428+cMmLujbghuGHkOnpomhDtOYoLMzGGNqSJ/WKUy8YiBf3HoyF/VrCcD7P2Zz+hOzGfPaAuZvyLUhaEyDYmcwxtSwzk0T+felfbn19M5Mmr2OdxZk8cXy7XyxfDt9WiVzzZAOxB2yRGPqPzuDMSZAWqfG8cCInsz5v+HcckpHGsVF8tPmPG55axFjZxcxafZa8ooOhDpMYwLGEowxAdYkMYbbzujCt3eeykMX9qJD43hyi5WHPlvJSQ9/xT8+XsamHOsQYOofSzDGBElsVDgjj2/DjFuHcuuAaAZ1TKOwpJRX5m5g6L9mMnryfGau3EGpXT4z9YS1wRgTZGFhQp/GEfz5khNYvmUvL81Zz8dLtvD1yh18vXIHrVNj+cPxbblkYGtSbSgaU4fZGYwxIdS9RRKP/64P3//tVO48uyutGsWSlVvEw5+v5ISHv+K2qYtZnLXHep+ZOsnOYIypBVLjo7h+6DH8cUgHZq3ewevfbSRj9U7e/zGb93/MpmfLJC49tg3n92lBcqwNR2PqBkswxtQi4WHCKV2bckrXpmzMKeTNeZt4Z0EWmdl7yczO5MFPlnNOr+b8bmBrTuiQioiEOmRjymUJxphaqm1aPH87pxu3nt6Z6cu28c78LL5dm8O0RdlMW5RN27Q4fjewNRf3b0Wz5JhQh2vMESzBGFPLxUSGc0HfllzQtyWbcvbx7sIs3l2wmY05+3hs+ioe/2IVw7o04eL+rTi1WxNiIsNDHbIxgCUYY+qUNmlx3H5GF/5yWmdm/7yTqfOzmLFi++EeaInREZzTqzkj+rXk+PaphIXZJTQTOkHtRSYiqSIyTUQKRWSjiIwsp56IyHgRyXGn8eJxsVlE+orIQhHZ5/7s61H2VxHJFJF8EVkvIn8NwqYZE1ThYcLwLk2Y8IcBfP+3U7n33O70aplM/v6DvLMgi8te+J5B47/mkc9XsmpbfqjDNQ1UsM9gngVKgKZAX+BTEflJVZd51RsDjAD6AAp8ifPkzOdFJAr4EHgSeA64DvhQRDqpagkgwJXAEuAY4AsRyVLVtwO7acaERlpCNNcMbs81g9uzZkc+HyzawrRF2WTvKeL5WWt5ftZaujVPYkTfFvymd3NaNYoLdcimgQjaGYyIxAMXA/eqaoGqzgE+Aq7wUf0q4HFV3ayq2cDjwCi3bBhOYnxSVfer6tM4SeUUAFV9VFV/VNWDqroKJxkNCuCmGVNrdGySyB1nduGbscN59/oTGXl8G5JjI1mxdS8Pf76SweNncsGzc5k0ey2bd9vwNCawJFg3cIlIP2CuqsZ5zLsDGKqq53nVzQPOUNV57uuBwExVTRSRW92ysz3qf+KWP+61HgF+BCaq6vM+YhqDc7ZE48aNB0ydOrWGtrZmFRQUkJCQEOowfLLYqieYsR04pCzZWcq8rQdZvLOUktJfyjokh3FsswgGNg2ncVxY0GOrKoutegIZ2/Dhwxeq6kBfZcG8RJYA7PWalwf4ehJTglvmWS/BTRjeZRWt536cs7RXfAWkqpOASQBdunTRYcOGVbgBoZKRkYHFVnUW2y9Od38WlZSSsWoHnyzdytcrdrAur5R1eSW8swr6tErmnF7NSZaNnGv7rcostiMFM8EUAEle85IAXy2Q3nWTgAJVVRHxaz0icjNOW8wQVd1/NIEbU1/ERoVzdq/mnN2r+RHJ5qfNefy02fnu9uKqWZzevSmndWtKv9Yp1hvNVEswE8xqIMJtjP/ZndcH8G7gx53XB/jBR71lwO0iIvrL9b3eOB0IABCR0cCdwMmqurlmN8OY+sFXsvnfsm18kbmFNTsKWLOjgAkZa0lPiOa0bk04vXtTBnVMt/tsjN+ClmBUtVBE3gfGici1OL3ILgBO8lH9NeA2EfkMpxfZ7cAzblkGUArcIiLPA390538NICKXAw8Bw1V1XWC2xpj6xTPZzPh6D7FtevHl8u18uXw72XuKeHt+Fm/PzyI2MpwhndI5pWsThnVpYiMImAoFu5vyjcDLwA4gB7hBVZeJyBDgc1Uta4WaCHQAlrqvX3TnoaolIjLCnfcIsAIY4XZRBngQSAPme9w6M0VVrw/khhlTX0SECYM6pjOoYzp/P687K7bmM2OFk2yWZucdfvwzQNdmiQzt0pihnRszsG0qURE2QLv5RVATjKrm4tzf4j3/G5zG+7LXCox1J1/rWQQMKKesfU3EaowBEaF7iyS6t0jillM7sTWviBkrdjBr1U6+XbuLldvyWbktn4mz1pEQHcFJx6QxrEsThnZpTMuU2FCHb0LMhooxxviteXIsV5zQlitOaMv+g6Us2LCbWat3krFqB6u3F/zq7KZjkwQGd0znpGPSOL5Dmj1moAGyBGOMqZboiPDDl9LuOqcb2XuKmLVqJ7NW72DumpzDHQUmf7uBMIFeLZM5yU04A9umEhtlnQXqO0swxpga0TIllpHHt2Hk8W0oOXiIxVl7mLtmF9+tzWFR1u7D3aAnZKwlKjyM/m1TOOmYdAZ1TKNXyxRrv6mHLMEYY2pcVEQYx7VP5bj2qdx6OuwrOcgP63P5dm0O367dxbIte/l+XS7fr8vl319CTGQYfVuncFy7VI5tn0r/No2Ij7bDU11nf0FjTMDFRUUwrIvTtRlgd2EJ36/L4du1OXy3zrmcVpZwwBktukeLJI5tl+pOjUhLiA7lJphqsARjjAm6RvFRh++7Acgp2M+CjbuZvz6X+RtyydyylyWb81iyOY+X5qwH4JjG8bSM3s+2uE30a9OIjk0SCLcRBmo1SzDGmJBLS4jmzB7NOLNHMwAK9x9k0aY9/LAhl/nrc1mUtZu1OwtZC8ze7NweFx8VTp/WKfRrk0Lf1o3o2zqFxol2llObWIIxxtQ68dERDO6UzuBO6QCUHDxE5pY83v16AflRaSzatIfsPUVum07O4eVap8bSt3Uj+rVOoU/rZLo3T7beaiFkCcYYU+tFRYTRv00j9raLZNiw/gDs2FvMoqw9LM7aw6JNu1myOY+s3CKycov4+KctAIQJHNM4gV4tk+npTj1aJFkHgiCxvWyMqZOaJMX86rLawdJD/LyjgEWbnISzNDuPn3cUHJ7eX5QNgAh0SI+nZ8tkerVMpkeLZHq0TCIpxm4ErWmWYIwx9UJEeBjdmifRrXkSI49vA0DxgVJWbstnaXYey7LzWJqdx+rt+U57zs5CPly85fDyrRrF0rVZIl2bJdG1eSJdmyXSLi2eiHC7P6e6LMEYY+qtmMhw+rZOoW/rlMPz9h8sZfW2AjK35B1OPCu25bN5dxGbdztjrZWJigijc9MEujRNoltzJ/l0aZZonQn8ZAnGGNOgREeE06tVMr1aJXOZO+9g6SE25BQ6g3duzWfltr2s2JpP9p4iMrP3kpn964fxpsVH0bFJwuGpU5NEdhcfQlXxGMW9wbMEY4xp8CLCw+jYJJGOTRI5t/cv8/cWH2D1tnxWbMtn5da9rHJHj84pLCFnfS7z1uf+aj33ffcFHZok0LFxAp2aOj87NkmgdWpcg7xnxxKMMcaUIykmkoHtUhnYLvXwPFUle0/R4cE81+50fq7I3k3+/oP8lLWHn7L2/Go9URFhtE+Lp21aHO3T42mXHk+7tHjapcfRNDGm3j6S2hKMMcZUgYjQqlEcrRrFHR76BiAjI4NeA0/kZzfxeCafrXnFrNqez6rt+UesLyYyjHZu8mmXHk/7tF8SUNOk6Dp9yc0SjDHG1JC0hGjSEqI5oUPar+bnFx9gw659bMgpZMOuQta7Pzfm7COnsOTwg9u8xUaG06pRLK0axdI6NY7WjeJ+9XtyXO3uWm0JxhhjAiwxJvJwxwJveUUH2JhTyPpdhWzYtc/53U1Au/cdOHwfj+/1RtCqURyt3aTTqlEsrRvF0To1juYpMSG/t8cSjDHGhFBybCS9W6XQu1XKEWV5RQfYvHsfWblFbN69j827i8jK3UeWOy+/+CArtu5lxda9R64YSIiOoHlyDNGlxXy+awnNU2JokRxL85QYmifH0iIlhriowKUBSzDGGFNLJcdGkhzrjDbgTVXJLSwha3fR4SSU5Sahzbn72JJXRMH+g4fPfjJzsny+R1JMBC1SYmmeHEPzlFhaJMfQJCmGpkkxNE2KpkliDI3iIqvVFmQJxhhj6iAROdzm43kjaRlVJa/oAFv2FPPFnB9Ia9OJrXuK2JZXzJa8IrbmFbM1r5i9xQfZW04bUJmo8DAaJ0bTJCmapokxzs+kGJpUcsOpJRhjjKmHRISUuChS4qLY0SSCYSe0PaKOqpJTWOIknT1O0tmSV8TOvfvZnl/Mjr372b7XSULZe4rI3lNUpRgswRhjTAMlIqQnRJOeEE3PlkdehitTfKDUSTb5xWzfW3z49x179/NkBeu3BGOMMaZCMZHhtEmLo01a3BFlT/6+/OVsmFBjjDEBYQnGGGNMQFiCMcYYExCWYIwxxgSEJRhjjDEBYQnGGGNMQAQ1wYhIqohME5FCEdkoIiPLqSciMl5EctxpvHiMUyAifUVkoYjsc3/29XdZY4wxwRHsM5hngRKgKXA5MEFEevioNwYYAfQBegPnAdcBiEgU8CEwBWgEvAp86M6vcFljjDHBE7QEIyLxwMXAvapaoKpzgI+AK3xUvwp4XFU3q2o28Dgwyi0bhnOD6JOqul9VnwYEOMWPZY0xxgRJMO/k7wwcVNXVHvN+Aob6qNvDLfOs18OjbImqqkf5Enf+/ypZ9ldEZAzOGQ/AfhHJ9G9Tgi4d2BXqIMphsVWPxVY9Flv1BDK2Iwc5cwUzwSQA3g8tyAMSy6mb51UvwW1L8S7zXk+5y3olJVR1EjAJQEQWqOpA/zcneCy26rHYqsdiqx6L7UjBbIMpAJK85iUBvsaI9q6bBBS4CaKy9VS0rDHGmCAJZoJZDUSISCePeX2AZT7qLnPLfNVbBvT26hnW26u8vGWNMcYESdASjKoWAu8D40QkXkQGARcAr/uo/hpwm4i0FJEWwO3AZLcsAygFbhGRaBG52Z3/tR/LVmRS1bcqaCy26rHYqsdiqx6LzYsE88qRiKQCLwOnAznAnar6pogMAT5X1QS3ngDjgWvdRV8E/q/sMpeI9HPndQdWANeo6iJ/ljXGGBMcQU0wxhhjGg4bKsYYY0xAWIIxxhgTEA0+wfg7PloNvl+GiBSLSIE7rfIoG+nGUCgiH7htVn7FWdGyFcRys4gsEJH9IjLZq+xUEVnpjvc2U0TaepRFi8jLIrJXRLaJyG01tWxlsYlIOxFRj/1XICL3Bis2t85L7r7OF5HFInJ2bdhvFcUW6v3m1psiIlvdeqtF5NqaWH8gY6sN+82jfidxjh1TPOYF5JhR2bJ+U9UGPQFvAe/g3KA5GOfGzB4BfL8M4Fof83vg3MtzshvLm8Db/sRZ2bIVxHIRzrhtE4DJHvPT3fVfAsQAjwHfe5Q/DHyDMxZcN2AbcNbRLutnbO0ABSLK2aaAxgbEA/e7cYQB57r7vl2o91slsYV0v3l8TqPd37u69QaEer9VElvI95tH/S/c+lMCfcyoaNkqHe8CdSCtCxPOP2QJ0Nlj3uvAIwF8zwx8J5iHgDc9Xh/jxpZYWZwVLetnTA/y64P4GOBbr/1UBHR1X28BzvAof6Dsw3k0y/oZW2X/8EGLzaPeEpxx9mrNfvMRW63ab0AXYCvwu9q237xiqxX7Dfg9MBXnC0RZggnIMaOyZasyNfRLZOWNj+Zz7LIa9LCI7BKRuSIyzJ33qzHUVHUt7h/ZjzgrWrY6vNdXCKwFeohII6A5FY8VV91lq2KjiGwWkVdEJB0gFLGJSFOc/bzsKNcf6NjKhHS/ichzIrIPWIlzEP/sKNcf6NjKhGy/iUgSMA7wvoQWqGNGjR0XG3qCqcr4aDXl/4AOQEucm58+FpFjqHiMtcrirGx8tqqqLBY4crw3f2KpbFl/7AKOxRlgb4C77Bse7x202EQk0n3vV1V15VGuP9Cx1Yr9pqo3umVDcG683n+U6w90bLVhvz0AvKSqm73mB+qYUWPHxYaeYKoyPlqNUNV5qpqvzqMGXgXmAudUEktVx1/zLq+qymKBI8d78yeWypatlDqPeligqgdVdTtwM3CGiCQGMzYRCcO5bFDixnC06w9obLVlv7mxlKrzuI5WwA1Huf6Axhbq/SbOwxRPA57wEW6gjhk1djxp6AmmKuOjBYriPM/mV2OoiUgHINqNsbI4K1q2OrzXF49zjXaZqu7GuXzQx6N+RbFUZdnqUPdnWLBiExEBXsJ5cN7FqnqgBtYf6Ni8BX2/+RBRtp6jWH+gY/MW7P02DKcdaJOIbAPuAC4WkR99rL+mjhk1d1ysaqNNfZuAt3F6TMQDgwhgLzIgBTgTp0dJBM5TPQtxrnn2wDktHeLGMoVf9+ooN87Klq0gngg3lodxvvGWxdXYXf/F7rzx/Lr3yyPALJzeL11x/lHKes5Ue1k/YzsepxE2DEjD6ekyM8ixPQ98DyR4za8N+6282EK634AmOA3VCUA4zv9BIXB+qPdbJbGFer/FAc08pn8B/3XXHbBjRkXLVumYF4gDaV2agFTgA/cDtQkYGcD3agzMxznV3INzIDjdo3ykG0MhzmOhU/2Ns6JlK4jnfpxvZJ7T/W7ZaTiNnUU4Pd/aeSwXjTOm3F5gO3Cb13qrvWxlsQGXAevd7dyKM7hps2DFhnMtXoFinEsJZdPlod5vFcVWC/ZbY5yD6R633lLgjzWx/kDGFur9Vs7/xZRAHzMqW9bfycYiM8YYExANvQ3GGGNMgFiCMcYYExCWYIwxxgSEJRhjjDEBYQnGGGNMQFiCMcYYExCWYIypZ0RklIgUVF7TmMCyBGNMgIjIZPdhVWXTLhH5RES6VmEd94tIZiDjNCZQLMEYE1gzcIZlbw6cAcQC00IakTFBYgnGmMDar6rb3OlHnFFxu4pILICIPCIiq0SkSEQ2iMijIhLjlo0C/o7z/JCys6BRblmyiEwQ5zG/xSKyQkQu9XxjcR7Xm+k+9namiLQP5oYbExHqAIxpKNwh3i8FlqpqkTu7EBgNZAPdcQar3A/cizOwYk+cRx8Pc+vnuSMmf4YzSOLVOKPfdsEZULFMNPA3d93FwKvuus8MzNYZcyRLMMYE1lkeDe7xQBbO838AUNUHPOpuEJGHcIZkv1dVi9xlD6rqtrJKInI6cCLO6LYr3NnrvN43ArhJVVe5y/wLeFlERG0AQhMkdonMmMCaDfR1p+OAr4AvRKQ1gIj8VkTmiMg2N5k8AbSpZJ39gK0eycWX/WXJxbUFiMI56zEmKCzBGBNY+1R1jTvNB67FeTrgGBE5Aee5G9OB83ASxz1AZA2870Gv14cflFUD6zbGL3aJzJjgUuAQzoOkBgHZnpfJRKStV/0SnIdgeVoENBeRbpWcxRgTUpZgjAmsaBFp5v7eCOeZ7gnAx0Ai0FJELge+w2mAv8xr+Q1AWxHpj/Pgp3ycy2zzgPdE5FacRv6OQLyqfhDQrTGmCux02ZjAOg3nSYhbcZLCscAlqpqhqh8DjwFPAkuA04H7vJZ/D6fH2FfATuAyVT0EnA3MxXnU7QrgKZw2FmNqDXuipTHGmICwMxhjjDEBYQnGGGNMQFiCMcYYExCWYIwxxgSEJRhjjDEBYQnGGGNMQFiCMcYYExCWYIwxxgTE/wOnW/qwUi8MMwAAAABJRU5ErkJggg==\n"
     },
     "metadata": {
      "needs_background": "light",
      "image/png": {
       "width": 408,
       "height": 286
      }
     },
     "output_type": "display_data"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "### Piecewise Constant Scheduling",
   "metadata": {
    "cell_id": "00132-914ecd17-d01d-491d-8082-eefb01db9c25",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 62
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00133-f8eefa8c-b624-4829-8f1e-02350392550f",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "55a749aa",
    "execution_start": 1659356200558,
    "execution_millis": 4,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 184
   },
   "source": "def piecewise_constant_fn(epoch):\n    if epoch < 5:\n        return 0.01\n    elif epoch < 15:\n        return 0.005\n    else:\n        return 0.001",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00134-dafa4aa8-feff-4767-9de5-b627eac34bc3",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "9566e95e",
    "execution_start": 1659356200566,
    "execution_millis": 14,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 202
   },
   "source": "def piecewise_constant(boundaries, values):\n    boundaries = np.array([0] + boundaries)\n    values = np.array(values)\n    def piecewise_constant_fn(epoch):\n        return values[np.argmax(boundaries > epoch) - 1]\n    return piecewise_constant_fn\n\npiecewise_constant_fn = piecewise_constant([5, 15], [0.01, 0.005, 0.001])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00135-24f4a160-f44e-4ea2-a737-95c04020bee4",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "5102065b",
    "execution_start": 1659356200581,
    "execution_millis": 264683,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 912
   },
   "source": "lr_scheduler = keras.callbacks.LearningRateScheduler(piecewise_constant_fn)\n\nmodel = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.Dense(10, activation=\"softmax\")\n])\nmodel.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\nn_epochs = 25\nhistory = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n                    validation_data=(X_valid_scaled, y_valid),\n                    callbacks=[lr_scheduler])",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/25\n1719/1719 [==============================] - 12s 7ms/step - loss: 0.8439 - accuracy: 0.7565 - val_loss: 0.8615 - val_accuracy: 0.7550 - lr: 0.0100\nEpoch 2/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.8937 - accuracy: 0.7437 - val_loss: 0.7484 - val_accuracy: 0.7432 - lr: 0.0100\nEpoch 3/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 1.0008 - accuracy: 0.6805 - val_loss: 1.5534 - val_accuracy: 0.6090 - lr: 0.0100\nEpoch 4/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 1.1070 - accuracy: 0.6275 - val_loss: 1.3036 - val_accuracy: 0.5442 - lr: 0.0100\nEpoch 5/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 1.2505 - accuracy: 0.5916 - val_loss: 1.5156 - val_accuracy: 0.5850 - lr: 0.0100\nEpoch 6/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.8833 - accuracy: 0.6522 - val_loss: 0.9401 - val_accuracy: 0.6704 - lr: 0.0050\nEpoch 7/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.8377 - accuracy: 0.6655 - val_loss: 0.9060 - val_accuracy: 0.6448 - lr: 0.0050\nEpoch 8/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.8336 - accuracy: 0.6704 - val_loss: 0.9199 - val_accuracy: 0.6464 - lr: 0.0050\nEpoch 9/25\n1719/1719 [==============================] - 13s 8ms/step - loss: 0.7837 - accuracy: 0.6826 - val_loss: 0.9033 - val_accuracy: 0.6924 - lr: 0.0050\nEpoch 10/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.7817 - accuracy: 0.6880 - val_loss: 0.8809 - val_accuracy: 0.6804 - lr: 0.0050\nEpoch 11/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.7552 - accuracy: 0.7056 - val_loss: 0.9051 - val_accuracy: 0.6660 - lr: 0.0050\nEpoch 12/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.6739 - accuracy: 0.7580 - val_loss: 0.7328 - val_accuracy: 0.7398 - lr: 0.0050\nEpoch 13/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.6415 - accuracy: 0.7691 - val_loss: 0.8016 - val_accuracy: 0.7512 - lr: 0.0050\nEpoch 14/25\n1719/1719 [==============================] - 12s 7ms/step - loss: 0.6325 - accuracy: 0.7723 - val_loss: 0.6973 - val_accuracy: 0.7740 - lr: 0.0050\nEpoch 15/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.6207 - accuracy: 0.7734 - val_loss: 0.7593 - val_accuracy: 0.7408 - lr: 0.0050\nEpoch 16/25\n1719/1719 [==============================] - 13s 7ms/step - loss: 0.4887 - accuracy: 0.8002 - val_loss: 0.6222 - val_accuracy: 0.7988 - lr: 0.0010\nEpoch 17/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.4628 - accuracy: 0.8117 - val_loss: 0.6270 - val_accuracy: 0.8148 - lr: 0.0010\nEpoch 18/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.4006 - accuracy: 0.8731 - val_loss: 0.5410 - val_accuracy: 0.8582 - lr: 0.0010\nEpoch 19/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.3616 - accuracy: 0.8901 - val_loss: 0.5355 - val_accuracy: 0.8660 - lr: 0.0010\nEpoch 20/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.3398 - accuracy: 0.8953 - val_loss: 0.5233 - val_accuracy: 0.8642 - lr: 0.0010\nEpoch 21/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.3255 - accuracy: 0.8993 - val_loss: 0.5222 - val_accuracy: 0.8660 - lr: 0.0010\nEpoch 22/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.3122 - accuracy: 0.9020 - val_loss: 0.5673 - val_accuracy: 0.8658 - lr: 0.0010\nEpoch 23/25\n1719/1719 [==============================] - 11s 6ms/step - loss: 0.3026 - accuracy: 0.9058 - val_loss: 0.5498 - val_accuracy: 0.8594 - lr: 0.0010\nEpoch 24/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.2939 - accuracy: 0.9080 - val_loss: 0.5635 - val_accuracy: 0.8682 - lr: 0.0010\nEpoch 25/25\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.2836 - accuracy: 0.9110 - val_loss: 0.5557 - val_accuracy: 0.8692 - lr: 0.0010\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00136-3e4efb6e-b566-4358-a920-04eefcb8244a",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "791158ff",
    "execution_start": 1659356465271,
    "execution_millis": 171,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 488,
    "deepnote_output_heights": [
     288
    ]
   },
   "source": "plt.plot(history.epoch, [piecewise_constant_fn(epoch) for epoch in history.epoch], \"o-\")\nplt.axis([0, n_epochs - 1, 0, 0.011])\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Learning Rate\")\nplt.title(\"Piecewise Constant Scheduling\", fontsize=14)\nplt.grid(True)\nplt.show()",
   "execution_count": null,
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEeCAYAAAC30gOQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAt1klEQVR4nO3de5hcVZ3v//cn907nRjqBQNA0CjYSgSCKowySERFnPP6M4pwzAyLIKF6G4wUEcUYUUQfDyAziIJIzIqDoT50DBMQBB6FHUbyAQEIgiXIJkHBJAgnppHP/nj/WrrBTqere3emqSnd9Xs9TT7r2WmvX2qsr9e2116q1FBGYmZkNtGGNroCZmQ1NDjBmZlYTDjBmZlYTDjBmZlYTDjBmZlYTDjBmZlYTDjC2C0mnSepqdD16IikkvbfR9bBiJF0t6Sc1OO+U7L0wuw9l2rMyr6v03AaOA0wTyv6zR/bYIulRSV+T1Jpl+SHwikbWsYB9gZtr+QKSxkv6kqSHJHVLelZSp6S/lVSX/zu1/PDry7klHSvp55JWSdog6RFJ10maMND1aoAnSe+n+xtcjyFnRKMrYA1zO3AKMBI4Bvh3oBX4aER0A90NrFuvIuKZWp5f0iTgLmAv4HPA74DNwJ8D5wN3A4/Xsg57CkmHALcC3wI+CawHXgm8GxjduJoNjIjYBtT0/dS0IsKPJnsAVwM/KTv2f4Cns59PA7rK0t8J3AtsBB4DvgKMyqWPAv4JWAZsAh4FPp5LPwS4BVgHPAf8AJiWpR0MRO752Owct+bKfxD4U+55AO/NPf987rWfAa7NpQk4F3iEFDgXAu/rpY2+Sfog3b9C2hhgTPbzXsA1wAvZuW8HZubyngZ0AccBD2bnvBM4IJfnZcB84HlgA7AY+JvcdeYfndnx1wM/A1YBL5KC4RvL6hnAGcCPs9d9NH/d1c5d4Xo/CTxV4H11MHATsDa75ruBQ/PvOeATwPKsvb4DjO3L7ym77tL78D7gHVndZ2fps7PnU3Jl2rNjryv4vHSO44DfZr+Te4DXltXldOCJLP1m4GNANPr/95708C0yK+km9WZ2IekE4Drg34CZpP9Y7yUFlJJrgPcDZwGvBv4OWJOV3xf4BekD9ijgrcA4YL6kYRGxmBQUZmfnehPpQ/NoSaVe9mygs0r9TgQ+TfoPfhDwP0g9jpIvZ/X5e1Kguwi4UtI7qpxvGPA3wHUR8VR5ekRsjIiN2dOrgTcA78qubQNwq6SWXJHRwGdJ7fZGYBKpN1DyTVJQ/QtS+36SrO2ycwK8nXQb5z3Z8/HAd0m9z6NIt3d+KqmtrLqfJwWvw0m3Pq+S9PJezl3uGWCqpL+oko6k/UhBLoDjgdcClwPDc9mOAV5D+v3/L1IP6BO59B5/T5LGkf5IeRR4HXAe8LVqdRoAF2Wv8VpgNXCdJGV1eSOp1385MIsUWL9Yw7oMTo2OcH7U/0FZD4b0QbMK+GH2/DRyPRhScDi/7BxzSH+livShHsDbq7zehcDPy47tlZU5Knv+/wNXZj9/GbiCdAvqjdmxJ9n1r+/3Zj+fBSwBRlZ47VZS8Dym7PilwE+r1Hfv7Pyf6qUdS9f95tyxiaS/4D+Ya8sAOnJ5Tib1tJQ9XwB8ocprtJP767qHugh4ukIbXZR7PoIUAN/Xx3MPJ/U2AniW9Nf6WcDUXJ6vkHqQo6qc4+rsdzg8d+z/ALcX/T2RemNrgHG59PdRux7MCblzHJ0d2z97/gNyPezs2Dzcg9np4R5M83q7pC5JG0m3Mn4B/O8qeY8E/jHL35XNMPs+6UNhGnAEsJ1066da+TeXlX8yS3tl9m8nL/VgZmfn6gRmSzoQ2J8qPRjSLaAxwGOSvi3pryWVxgYOydJuLXv9j+Zeu5yqHC/3atJ13106EBFrSbd2Dsnl2xQRS3LPV5BuKe6VPf868DlJd0v6sqQje3thSXtLulLSUklrSbce9wZeXpZ1Qa5uW4GVWb7CImJbRHyA9Dv4NOm20DnAYkkzs2xHAHdFxOYeTvVQpPGOkhW5uhT5Pb0aWBAR+RmOd1M7C3I/r8j+LdX3YHbuJUO6nWY5HuRvXr8g/UW4BVgREVt6yDuM1P3/cYW0lQVeaxjp1sanK6Q9m/3bCVyRBZPXZc/HAidlr/FIVLhdBRART0rqIN0zfytwCfAFSW/gpZmS7yR9MOZVu+aVpL+UX93LdfUkv0z51ippwwAi4tuSbgP+ilT/X0u6KCIu6OH81wD7AJ8i9fQ2AT8nBa688msM+jl7NCKWk27LfVfS54ClpEBzWsFT9FSX/vyeKtme/Zv/I6Hird8C8q+70+/MinFjNa8NEfGniFjWS3AB+ANwcJa//LGVdP9/GGkMoVr5mcCyCuXXAcRL4zD/SAomz5GCzNGke/qdPVUw0rjILRHxKdJA8Mys7EOkD98ZFV57WZVzbSfdsjtZ0v7l6ZLGSBoDPJxd9xtzaROAQ7PXLSwinoqIeRHxP0njJmdkSaUewfCyIn8OfCO75kWkHsy+fXnNHs5dpL4vkG7JjcsO3Qf8uaTyAFdUkd/Tw8Chuen0AH9Wdp7SHzz5tpjVzzr1ZDHpfZZ3VKWMzcwBxoq4EDhJ0oWSXiPpYEnvlXQxQEQsBX4E/LukEyUdIOkYSadk5S8njU38UNIbJL1C0lslzZM0Pvc6/026p35ndt7HSR8Y76GHAJN9MfSDkg6VdADwAdJfn3/MAtjXgK9JOl3SgZJmSfqIpDOqnZMU6J4AfivpA5JmZmVPIc1imhYRfyQNoF+ZXe+hwPdIExS+X7BtkfR1SW/P2mUWadC9FKCeI41NnCBpH0kTs+NLgfdJOkTS60kBsafbU5VUO3d5/T4s6QpJb5P0yqwt5pIC6Q1Ztm+Sgs2PJL0+a6u/za6nVwV/T98n9QavyupwPOn3lPcn0u3XCyS9StLbSNPMB9plwNsknSPpIEl/R5q0YHmNHgTyo/4PKkxTLks/jV2nKb8N+CVpkPhF0rTNM3Ppo4GLSVNQN5GmmubTDwL+g5em8y4BvsHOU50/wq7Tj68mN7iaO54f5J9Duhe/hjQd9/fA/8jlFWl8qfRX8krgv4Dje2mniaTB68WkabGlXtXfAMOyPIWmKZeddza5geisHf6YvcZKUrCYnsv/QVKw28ZL05QPJ93z787a+hTSLL0LKrVR7tjjwKd7OneFdjgiu8bS9OHVwG+AU8ryzQR+Spr8sQ74NfCaau854ALgwb78nkgz9v6QpT9AuqW2Y5A/y/MmUq+6O3tflKYy93WQv+pEgezY6aRg1k2a+HA20N3o/9970qM0i8XMzHaDpH8F3hoRhza6LnsKD/KbmfWDpHNIPawu0uSMjwD/0NBK7WHcgzEz6wdJPyTdTptIWt3iSuDr4Q/VHRxgzMysJjyLzMzMasJjMJlJkybFgQce2Ohq7HHWr19Pa2tr7xmbjNtlV26TyoZ6u9x7772rImJqpTQHmMw+++zDPffc0+hq7HE6OzuZPXt2o6uxx3G77MptUtlQbxdJFb+wDL5FZmZmNeIAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNVHXACNpsqQbJK2XtEzSSVXySdJcSauzx1xJyqXPk7RE0nZJp1Uo/ylJz0h6UdJVkkb3VrfHX9zO0V+9gxvvW17oWm68bzlHf/UODjjvliFZzsxsd9W7B3M5sBnYBzgZuELSzAr5zgDmAIcDhwHvBD6cS38A+Bjwh/KCkk4AzgOOA2YArwC+WKRyy9d089nrF/b6IXzjfcv57PULWb6mmxiC5czMBkLddrSU1AqcCLwmIrqAuyTdBJxCCgh5pwKXRMRTWdlLgA8B3wKIiMuz4xsrvNSpwLcjYlGW50vAdRVeo6LuLdv4hxsWctefVlXN89OFT9O9ZdugLffPty1hzhHTq5YzMxsI9dwy+VXA1ohYmjv2AHBshbwzs7R8vko9nUpmAvPLyu4jqS0iVuczSjqD1Fti1LQDdxzfsHkbdy6q/lf+hs1R5fjgKLd8TTednZ1Vy+V1dXUVzttM3C67cptU1sztUs8AMw54sezYWmB8lbxry/KNk6SIqPyp2XNZstfZKcBExDxgHsDofQ/acd7pk1r41XlvqfoCR3/1Dpav6d7l+GAqV3SP8KG+n3h/uV125TaprJnbpZ5jMF3AhLJjE4B1BfJOALoKBJdqZanyOrtoGTmcc07o6DHPOSd00DJy+JAtZ2Y2EOoZYJYCIyQdlDt2OLCoQt5FWVpv+SqpVPbZ8ttjlUyf1MJF7zm01/GJOUdM56L3HMr0SS1oEJRrax0FwJRxowqVMzMbCHW7RRYR6yVdD1wo6YPALOBdwJsqZL8WOEvST4EAzga+UUqUNIoUHAWMlDQG2BwR27OyV0u6DlgBfA64urf6tU8Y1uPtpnJzjpjerw/qRpQ7/GWT+IuvdXLeX77awcXM6qbe05Q/BrQAzwE/AD4aEYskHSOpK5fvSuBmYCHwIHBLdqzkZ0A3KTjNy35+M0BE3ApcDNwJPAEsA75Qw2va4+2/VwvDh4llq9c3uipm1kTqOchPRDxP+n5L+fFfkgbnS88DODd7VDrP7F5e51+Af9mNqg4pI4cPY/+9WnhslQOMmdWPl4ppEjPaWlm2ekOjq2FmTcQBpkm0t43l8dXrKTYRz8xs9znANIkZba2s27iVFzZsaXRVzKxJOMA0ifa2sQA87oF+M6sTB5gmMaOtFcAzycysbhxgmsTLJrcgweOrPNBvZvXhANMkRo8Yzn4TW3yLzMzqxgGmiRwwpZXHPVXZzOrEAaaJzGgb6zEYM6sbB5gm0t7WypoNW1izYXOjq2JmTcABponMyKYq+xv9ZlYPDjBNpH1KmqrsgX4zqwcHmCby8snuwZhZ/TjANJExI4ez78Qx7sGYWV04wDSZ9rZWHvey/WZWBw4wTaZ9yljfIjOzunCAaTIz2lpZvX4zL270qspmVlsOME2mtKryE+7FmFmNOcA0mdKqyh7oN7Nac4BpMv6ypZnViwNMkxk7agR7jx/tmWRmVnMOME2ova3Vt8jMrOYcYJpQ+5SxXrbfzGrOAaYJzWhrZeW6TazftLXRVTGzIcwBpgm1ZzPJPNBvZrXkANOEXppJ5nEYM6sdB5gmVAowHocxs1pygGlC48eMZMq4Ue7BmFlNOcA0qRmeqmxmNeYA06TSsv2+RWZmtVPXACNpsqQbJK2XtEzSSVXySdJcSauzx1xJyqXPknSvpA3Zv7NyaaMlfUvSs5Kel3SzpOl1uLxBpb1tLM+8uJHuzdsaXRUzG6Lq3YO5HNgM7AOcDFwhaWaFfGcAc4DDgcOAdwIfBpA0CpgPfA/YC7gGmJ8dB/gE8Mas3H7AC8A3anM5g9eMKWmq8hPPuxdjZrVRtwAjqRU4ETg/Iroi4i7gJuCUCtlPBS6JiKciYjlwCXBaljYbGAFcGhGbIuIyQMBbsvQDgNsi4tmI2Aj8EKgUxJpa+46ZZB6HMbPaGFHH13oVsDUiluaOPQAcWyHvzCwtn29mLm1BREQufUF2/Fbg28DXJe0HrCH1lP6zUoUknUHqLTF16lQ6Ozv7dkWD2Potqfnu+N1CRq9cXDVfV1dXU7VLUW6XXblNKmvmdqlngBkHvFh2bC0wvkretWX5xmXjMOVp5ef5I/AksBzYBiwEzqxUoYiYB8wD6OjoiNmzZxe8lKHhc3f/jOGTpjF79qFV83R2dtJs7VKE22VXbpPKmrld6jkG0wVMKDs2AVhXIO8EoCvrtfR2nsuB0UAb0ApcT5UeTLOb0dbq78KYWc0UDjCS/lLSTyQ9JOll2bEPSjqu4CmWAiMkHZQ7djiwqELeRVlapXyLgMPys8pIA/ql9FnA1RHxfERsIg3wHyVpSsF6No32trGeqmxmNVMowEg6GfgR6fbTAcDILGk4cG6Rc0TEelJv4kJJrZKOBt4FfLdC9muBsyRNz8ZSzgauztI6Sbe+Pp5NSS7d/roj+/f3wPslTZQ0EvgYsCIiVhWpZzNpn9LKirXdbNziqcpmNvCK9mDOBT4UEZ8C8mu8/4bUYyjqY0AL8BzwA+CjEbFI0jGSunL5rgRuJo2fPAjckh0jIjaTpjC/nzSIfzowJzsO8GlgIykYrgT+Cnh3H+rYNNrbWomAp15wL8bMBl7RQf6DgLsrHK80HlJVRDxPCg7lx39JGrwvPQ9SUKvYO4qI+4Ajq6StJs0cs17sWPRy1QYO3LvSXAszs/4r2oNZQZpmXO7NwCMDVx2rp9K+MP4ujJnVQtEAMw+4LBs3AXiZpFOBi4EralIzq7lJY0cyYcwIbzxmZjVR6BZZRFwsaSLwX8AY4E5gE/C1iLi8hvWzGpJE+xSvqmxmtVH4i5YR8Y+SvgIcQur5PBQRXb0Usz3cjLZWHnhyTaOrYWZDUNFpyldJGh8RGyLinoj4XUR0ZdONr6p1Ja12Dmgby1MvbGDz1u2NroqZDTFFx2BOJU0vLtdCmi5sg9SMtla2e6qymdVAj7fIJE0mrVQsYC9J+e/ADAfeATxbu+pZrbVPSVOVl63ewCumjuslt5lZcb2NwawCIns8VCE9gC8MdKWsfmZ4qrKZ1UhvAeYvSL2XO0h7uTyfS9sMLIuIFTWqm9VBW+soxo32VGUzG3g9BpiI+G8ASQcAT0aER4KHGEnMaBvrHoyZDbii34NZBpAtPPlyYFRZ+i8GvmpWL+1trTz0dPlWPWZmu6dQgMkCy/dJS8ME6bZZfkfJ4QNfNauXGW1juW3RM2zdtp0Rw+u5RZCZDWVFP00uJS2RfwiwATgG+GvgYeDtNamZ1U37lFa2bg+Wr+ludFXMbAgp+k3+Y4F3RMRiSQGsjIhfSdoEfIm0hIwNUi8terlhx6wyM7PdVbQH00KasgxpJtne2c8PkXaTtEGsva30XRgP9JvZwCkaYBYDB2c/3w98RNIM4O+B5TWol9XR1PGjaRk53Nsnm9mAKnqL7OvAtOznC4Fbgb8lrah8ag3qZXVUmqrsHoyZDaSi05Svy/38B0ntpB7NE97rfmhob2vlj8+ta3Q1zGwI6dec1GxV5T8A6yWdN8B1sgaYMWUsTz7fzbbt0XtmM7MCeg0wkqZIeoekt0kanh0bKemTwOPAp2tbRauHA9pa2bxtOys8VdnMBkiPAUbSm4A/AjcD/wn8StLBwALgTNIU5ZfXupJWe6XpyV6TzMwGSm89mC8Bt5GmIl8KHAX8BLgIOCgi/i0i/Ik0BJSW7feaZGY2UHoLMIcDX4qIB4HzScvDfDYiro0I36wfQvYZP4bRI4Z5JpmZDZjeAsxkYCWkgX3SMjH31bpSVn/DhpVWVXaH1MwGRpFpyqWdLEsLXE7IdrrcISKer1jSBpUZba3uwZjZgCkSYPI7WQr4fdnzwKspDwntbWP5xdKVbN8eDBumRlfHzAa5IjtaWpOY0dbKpq3beebFjew3qaXR1TGzQa7QjpbWHA6YUlpVeb0DjJntNu8uZTvM2LGqsgf6zWz31TXASJos6QZJ6yUtk3RSlXySNFfS6uwxV5Jy6bMk3StpQ/bvrLLyr5X0C0ldkp6V9IkaX9qQsO/EFkYNH+bvwpjZgKh3D+ZyYDOwD3AycIWkmRXynQHMIX0P5zDgncCHASSNAuYD3wP2Aq4B5mfHkTSFtNrzlUAbcCDws5pd0RAyfJh42eQWlnnZfjMbAHULMJJagROB8yOiKyLuAm4CTqmQ/VTgkoh4KiKWA5cAp2Vps0ljR5dGxKaIuIw0m+0tWfpZwG0RcV2Wvi4iHq7ZhQ0x7W2t7sGY2YAouh/MQHgVsDUiluaOPUDajrnczCwtn29mLm1B2UoCC7LjtwJ/BiyU9GtS7+W3wN9HxBPlLyLpDFJvialTp9LZ2dmPyxpahndv4tGVW7nzzjuRRFdXl9ulArfLrtwmlTVzuxQKMJKuqpIUwEbgT8API2JFD6cZB7xYdmwtML5K3rVl+cZl4zDlaeXn2R94LXA8sBC4GPgBcPQulY+YB8wD6OjoiNmzZ/dQ/ebwxOjH+dmyRcw88o3sPWEMnZ2duF125XbZlduksmZul6I9mKnAMcB24MHs2GtIt6buBd4DXCjpmIi4v8o5uoAJZccmAJV2uSrPOwHoioiQ1Nt5uoEbIuL3AJK+CKySNDEiygOTlWnPVlV+bNV69p4wpsG1MbPBrOgYzK9Iy/XvHxFvjog3k3oKPyUNoM8AbiGNlVSzFBgh6aDcscOBRRXyLsrSKuVbBByWn1VGmghQSl9A6lmVeFHOPmj3sv1mNkCKBphPABfml+bPfv4K8KmI2AzMBWZVO0FErAeuJ/V0WiUdDbwL+G6F7NcCZ0maLmk/4Gzg6iytE9gGfFzSaElnZsfvyP79DvDubCrzSNIq0He591LMfpPGMGKYPNBvZrutaIAZB+xb4fi0LA3S+Epvt9w+BrQAz5HGRT4aEYskHZPd+iq5krTJ2ULSLblbsmNkwWwO8H5gDXA6MCc7TkTcAfxDVuY50kB/xe/b2K5GDB/GyyaPdQ/GzHZb0TGYG4BvSzqXlxa7fD1pAP367PlRpNtgVWWrLs+pcPyXvBSoyGaInZs9Kp3nPuDIHl7nCuCKnupi1aVl+92DMbPdUzTAfAT4F9KXG0tltgJXAZ/Onj8MfGhAa2cN0d7Wyj2Pv4D3lDOz3VEowGTjLR+RdDbwyuzwI9m4SinP/QNfPWuEGW1j6dq0ldXrNze6KmY2iPXpi5ZZQFlQo7rYHqI0k+zxVb5NZmb9V/SLlmNIM8mOA/ambHJARBw28FWzRmnfsWz/BqY0uC5mNngV7cF8E3g38GPg1/i7JUPa9EktDB8mlq1ez5RRja6NmQ1WRQPMHOCvI+L2GtbF9hCjRgxj+qQWHl+9gSMrTU43Myug6PdgNgBP1rIitmeZ0TaWZZ6qbGa7oWiAuZj0zXr1mtOGhPa2Vh5btd5Tlc2s34reIjuetNjl2yU9BGzJJ0bE/zfQFbPGmtE2lnUbt7J+iwdhzKx/igaYVaRv81uTKE1VfnbD9gbXxMwGq6JftPxArStie5ZHVqal4b70m41ctfgOzjmhgzlHTO+13I33Leefb1vCijXd7DepZciWW76mm+m/cbuY9aSeO1raIHHjfcv519tfWlZu+ZpuPnv9QoAeP2xuvG85n71+Id1btrlcE5Uzq0bVBnElLQCOjYgXJC2kh+++DIUvWnZ0dMSSJUsaXY09wtFfvYPla7p3OT5yuDhkv4lVyz20Yi1btu36NnG5wV1u+qQWfnXeW6qWK2nmnRt7MtTbRdK9EfG6Smk99WD+L7Ap+/k/BrxWtsdaUSG4AGzZFkxqGVm1XKUPJ5cb/OWqvR/MelM1wETEFyv9bEPffpNaKvZgpk9q4ZrTj6parlrPx+UGd7n9JrVULWPWk6Lfg7Emcs4JHbSMHL7TsZaRwznnhA6XczmzwooudjmZtD1ytcUuJwx81axRSgO6O2ZLFZxNlC/Xl1lIg7HcUG6X0kB/0eszq6bqIP9OmaQbgCOAecAKygb8I+KamtSujjzIX9lQH6Dsr6HcLld0PsLcWxfzwOffxsSx1cdsyg3lNtkdQ71d+jvIn3cccHxE/HbgqmVme6KDp40HYMmz6zjqgMkNro0NZkXHYJ4DumpZETPbM3SUAswzLza4JjbYFQ0w/whcKGlcLStjZo2378QxTBgzgsXPrGt0VWyQK3qL7HNAO/CcpGXsutjloP+ipZklkjh42gSWOMDYbioaYPxFS7Mm0jFtPDfet5yIwLt0WH/1GmAkjQRagcsjYlntq2RmjdYxbTzrNm1l+Zpu9t9rbKOrY4NUr2MwEbEF+CjgP2PMmsSOmWS+TWa7oegg/8+A3le7M7Mh4VVZgPFAv+2OomMwPwf+SdJhwL3ATpu1R8T1A10xM2ucCWNGMn1Si3swtluKBph/y/79eIW0AIZXOG5mg9jB08Y7wNhuKXSLLCKG9fBwcDEbgjqmjeeRlV1s3upts61/vJqymVXUMW08W7fHju2zzfqqcICRtJekkySdJ+nz+UcfzjFZ0g2S1ktaJumkKvkkaa6k1dljrnKT8SXNknSvpA3Zv7MqnGOUpIclPVW0fmb2koOnpUXSfZvM+qvocv1/BtxC2uFyKrAc2Dd7/jhwYcHXuxzYDOwDzAJukfRARCwqy3cGMAc4nDTG81/AY8C3JI0C5gOXAt8EPgzMl3RQRGzOneMcYCUwvmDdzCznFVNbGTlcnklm/Va0B/PPwHXAdGAjacryy4F7gLlFTiCpFTgROD8iuiLiLuAm4JQK2U8FLomIpyJiOXAJcFqWNpsUGC+NiE0RcRnpOzo7plFLOgB4H3BRweszszIjhw/jlVPHedFL67eis8gOA/4uIkLSNmB0RDwq6TPA90nBpzevArZGxNLcsQeAYyvknZml5fPNzKUtiJ03slmQHb81e/4N4B+AHjcTl3QGqbfE1KlT6ezsLHAZzaWrq8vtUkGztMte2sgDy4pda7O0SV81c7sUDTD5W0/PAjOAh0lL+O9X8BzjgPI/hdZS+RbWuCwtn29cNg5TnrbTeSS9GxgeETdImt1ThSJiHmkTNTo6OmIobwrUX0N9s6T+apZ2eZhHuPvWxRxx1NG9bj7WLG3SV83cLkVvkf0BeH32cyfwZUmnApeReg9FdAHlWytPACrd4C3POwHoynotVc+T3Ya7mMrf1zGzPspvPmbWV33ZD2ZF9vPnSIPn3wD2IrvFVMBSYISkg3LHDgfKB/jJjh1eJd8i4LD8rDLSLbxFwEGkbQV+KekZ4HpgX0nPSGovWE8zy3jzMdsdhW6RRcQ9uZ9XAn/Z1xeKiPWSridtXPZB0iyydwFvqpD9WuAsST8lzSI7mxTQIPWgtgEfl/Qt4EPZ8TuA7cDLcud5E2kVgteSgqKZ9cG+E8cw3puPWT/16YuWkl4n6X9lt6KQ1Cqp6DgOwMeAFtIWzD8APhoRiyQdIyn/ba4rgZuBhcCDpCnSVwJkU5HnAO8H1gCnA3MiYnNEbI2IZ0oP4Hlge/Z8W1+u1cxKm495yRjrn6Lfg9mH9N2To0g9ioOAR4F/IU1b/kSR80TE86TgUH78l6TB+9LzAM7NHpXOcx9wZIHX6wT2L1I3M6vs4GkTuPF+bz5mfVe0B/OvpNljbcCG3PEfA28b6EqZ2Z6jY9p41m3cyoq1GxtdFRtkit7eOg44LiJeKPsL5hHSFy7NbIgqzSRb/PSLTJ/U0uDa2GBStAfTws7fhSmZSrpFZmZDlDcfs/4qGmB+wUtLtQCEpOHAZ0ibkZnZEOXNx6y/it4iOxf4b0mvB0aT1gabCUwEjq5R3cxsD9HhmWTWD0U3HHsIOBT4NfAzYAxpgP+IiHikdtUzsz3Bwd58zPqh8HdYsu+VfCF/TNIMST+KiP854DUzsz1GafOxR1d17dgnxqw3u7uj5STSEvxmNoSVgsrip32bzIrzlslm1itvPmb94QBjZr3y5mPWHw4wZlaIZ5JZX/U4yC/ppl7Ke7TPrEkcPG0C8+9fwdruLUxs6XnzMTPofRbZ6gLpjw1QXcxsD1ZaMmbps+t4ffvkBtfGBoMeA0xEfKBeFTGzPVtHbk0yBxgrwmMwZlaINx+zvnKAMbNCvPmY9ZUDjJkV1jFtPEueXUfaE9CsZw4wZlbYwdMmePMxK8wBxswKK80k8xcurQgHGDMrrLT52MNek8wKcIAxs8K8+Zj1hQOMmfWJl4yxohxgzKxPOrz5mBXkAGNmfXJwbvMxs544wJhZn5Q2H/NtMuuNA4yZ9Ulp8zHPJLPeOMCYWZ948zErygHGzPrMM8msCAcYM+uzjmnjWbF2I2u7tzS6KrYHq2uAkTRZ0g2S1ktaJumkKvkkaa6k1dljriTl0mdJulfShuzfWbm0cyQ9KGmdpMcknVOHSzNrKq/OBvqXPutejFVX7x7M5cBmYB/gZOAKSTMr5DsDmAMcDhwGvBP4MICkUcB84HvAXsA1wPzsOICA92dpbwfOlPQ3Nboes6a0Y/Mx3yazHtQtwEhqBU4Ezo+Iroi4C7gJOKVC9lOBSyLiqYhYDlwCnJalzSbtxHlpRGyKiMtIQeUtABFxcUT8ISK2RsQSUjA6uoaXZtZ0dmw+9rQH+q26HrdMHmCvArZGxNLcsQeAYyvknZml5fPNzKUtiJ03pFiQHb81f5LsttoxwJWVKiTpDFJvialTp9LZ2Vn0WppGV1eX26UCtwvs27Kd3y15is7O1YDbpJpmbpd6BphxQPmfO2uB8VXyri3LNy4LGOVpPZ3nAlIv7TuVKhQR84B5AB0dHTF79uweL6AZdXZ24nbZldsFbl+zkPn3r+DYY49FktukimZul3qOwXQBE8qOTQAq3cQtzzsB6Mp6LYXOI+lM0ljMOyJi027U28wq6PDmY9aLegaYpcAISQfljh0OLKqQd1GWVinfIuCw/Kwy0kSAHeeRdDpwHnBcRDw1AHU3szLefMx6U7cAExHrgeuBCyW1SjoaeBfw3QrZrwXOkjRd0n7A2cDVWVonsA34uKTRWU8F4A4ASScD/wQcHxGP1up6zJqdZ5JZb+o9TfljQAvwHPAD4KMRsUjSMZLyS7NeCdwMLAQeBG7JjhERm0lTmN8PrAFOB+ZkxwG+DLQBv5fUlT2+VesLM2s2pc3HFntNMquinoP8RMTzpOBQfvyXpMH70vMAzs0elc5zH3BklbQDBqKuZtY7LxljPfFSMWbWb958zHriAGNm/ebNx6wnDjBm1m8dO2aS+TaZ7coBxsz67RVTxjFyuDyTzCpygDGzfhs1orT5mAOM7coBxsx2S8e08V700ipygDGz3VLafGz9lug9szUVBxgz2y2lJWOWd3mqsu3MAcbMdktHtrvlU+scYGxnDjBmtlv2yzYfc4CxcnVdKsbMhp75969g05bt3PHkdo7+6h2cc0IHc46Y3mu5G+9bzj/ftoQVa7rZb1LLkC23fE03038zdNtl1LQDKy7bBQ4wZrYbbrxvOZ+9fiGbt6Xey/I13Xz2+oUAPX5Ilcp1b9nmckOgXDXaeefh5tXR0RFLlixpdDX2OM28G19P3C7J0V+9g+Vrunc5PnrEMN7wiraq5X776Go2VVi/zOUGX7mnr/kkm57+oyrlcw/GzPptRYXgArBp63Ze7N5StVylDzWXG/zlyjnAmFm/7TeppWIPZvqkFm78+6OrlqvW83G5wV2unGeRmVm/nXNCBy0jh+90rGXkcM45ocPlmqxcJe7BmFm/lQaCd8yWKjgLKV+uL7OXBmO5od4uT/eQz4P8GQ/yV+bB7MrcLrtym1Q21NtF0r0R8bpKab5FZmZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNeEAY2ZmNVHXACNpsqQbJK2XtEzSSVXySdJcSauzx1xJyqXPknSvpA3Zv7OKljUzs/qodw/mcmAzsA9wMnCFpJkV8p0BzAEOBw4D3gl8GEDSKGA+8D1gL+AaYH52vMeyZmZWP3ULMJJagROB8yOiKyLuAm4CTqmQ/VTgkoh4KiKWA5cAp2Vps0n72FwaEZsi4jJAwFsKlDUzszqp54ZjrwK2RsTS3LEHgGMr5J2ZpeXzzcylLYidN7JZkB2/tZeyO5F0BqnHA7BJ0oPFLqWpTAFWNboSeyC3y67cJpUN9XaZUS2hngFmHPBi2bG1wPgqedeW5RuXjaWUp5Wfp2rZsqBERMwD5gFIuqfapjnNzO1SmdtlV26Typq5Xeo5BtMFTCg7NgFYVyDvBKArCxC9naensmZmVif1DDBLgRGSDsodOxxYVCHvoiytUr5FwGFlM8MOK0uvVtbMzOqkbgEmItYD1wMXSmqVdDTwLuC7FbJfC5wlabqk/YCzgauztE5gG/BxSaMlnZkdv6NA2Z7M6/tVNQW3S2Vul125TSpr2nZRPe8cSZoMXAUcD6wGzouI70s6BvjPiBiX5RMwF/hgVvTfgc+UbnNJOiI7dgjwMPB3EXFfkbJmZlYfdQ0wZmbWPLxUjJmZ1YQDjJmZ1UTTB5ii66M1G0mdkjZK6soeSxpdp3qTdKakeyRtknR1WdpxkhZn6+HdKanql82GmmrtIqldUuTeM12Szm9gVesqm3T07exzZJ2k+yX9ZS696d4zTR9gKL4+WjM6MyLGZY+ORlemAVYAXyZNTNlB0hTSjMjzgcnAPcAP6167xqnYLjmTcu+bL9WxXo02AniStDrJROBzwI+ywNuU75l6fpN/j5NbH+01EdEF3CWptD7aeQ2tnDVcRFwPIOl1wP65pPcAiyLix1n6BcAqSQdHxOK6V7TOemiXppZ9FeOC3KGfSHoMOBJoownfM83eg6m2Ppp7MMlFklZJ+pWk2Y2uzB5kp/Xusg+WR/D7pmSZpKckfSf7y70pSdqH9BmziCZ9zzR7gOnL+mjN5jPAK4DppC+K3SzplY2t0h6jt/XwmtUq4PWkxQ+PJLXHdQ2tUYNIGkm69muyHkpTvmeaPcD0ZX20phIRv42IddmWCNcAvwL+qtH12kP4fVNBtg3HPRGxNSKeBc4E3iZpSH+IlpM0jLRCyWZSG0CTvmeaPcD0ZX20ZhekfXesbL27bCzvlfh9U670Le6m+ZzJVhL5NmnS0IkRsSVLasr3TNP84ivp4/poTUPSJEknSBojaYSkk4E3k/bbaRrZtY8BhgPDS+0B3AC8RtKJWfrnSXsUDdnB2rxq7SLpDZI6JA2T1AZcBnRGRPmtoaHsCuDVwDsjojt3vDnfMxHR1A/SlMEbgfXAE8BJja5Tox/AVOD3pO77GuA3wPGNrlcD2uEC0l/h+ccFWdpbgcVAN2kB1vZG17fR7QL8LfBY9n/padLCs9MaXd86tsuMrC02km6JlR4nN+t7xmuRmZlZTTT1LTIzM6sdBxgzM6sJBxgzM6sJBxgzM6sJBxgzM6sJBxgzM6sJBxizISrbm+W9ja6HNS8HGLMakHR19gFf/vhNo+tmVi9NvR+MWY3dTtpbKG9zIypi1gjuwZjVzqaIeKbs8TzsuH11pqRbsi10l0l6X76wpEMl3S6pW9LzWa9oYlmeUyUtzLYvflbSNWV1mCzpx9mW4I+Wv4ZZLTnAmDXOF4GbgFmkPXeuzXaJLK22extpLaujgHcDbyK3TbGkDwNXAt8BDiNtp/Bg2Wt8HphPWsn3h8BVkl5esysyy/FaZGY1IOlq4H2khQ/zLo+Iz0gK4N8j4kO5MrcDz0TE+yR9CPgasH9ErMvSZwN3AgdFxJ8kPQV8LyIqbu+dvcZXI+Kz2fMRpA32zoiI7w3c1ZpV5jEYs9r5BXBG2bE1uZ/vLku7G3hH9vOrScu55zek+jWwHThE0ouk3UZ/3ksdFpR+iIitklYCexeqvdlucoAxq50NEfGnGpy3L7cdtpQ9D3xr3OrEbzSzxvmzCs8fzn5+GDi0bLvhN5H+zz4cEc8By4Hjal5Ls35yD8asdkZLmlZ2bFtErMx+fo+k35M2n3ovKVi8IUu7jjQJ4FpJnwf2Ig3oX5/rFX0F+FdJzwK3AGOB4yLiklpdkFlfOMCY1c5bSTs75i0H9s9+vgA4kbS18ErgAxHxe4CI2CDpBOBS4HekyQLzgU+UThQRV0jaDJwNzAWeB35ao2sx6zPPIjNrgGyG119HxH80ui5mteIxGDMzqwkHGDMzqwnfIjMzs5pwD8bMzGrCAcbMzGrCAcbMzGrCAcbMzGrCAcbMzGri/wEZkTtb50UzSQAAAABJRU5ErkJggg==\n"
     },
     "metadata": {
      "needs_background": "light",
      "image/png": {
       "width": 408,
       "height": 286
      }
     },
     "output_type": "display_data"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "### Performance Scheduling",
   "metadata": {
    "cell_id": "00137-44633281-8c58-4822-8071-aceac8ca2479",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 62
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00138-70369719-1775-4d5f-80dc-6f8dd82d6939",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d690901b",
    "execution_start": 1659356465466,
    "execution_millis": 1,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "source": "tf.random.set_seed(42)\nnp.random.seed(42)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00139-89807479-4c90-48d8-b0e4-ee6f42c236aa",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "4a5895e4",
    "execution_start": 1659356465478,
    "execution_millis": 172746,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 930
   },
   "source": "lr_scheduler = keras.callbacks.ReduceLROnPlateau(factor=0.5, patience=5)\n\nmodel = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.Dense(10, activation=\"softmax\")\n])\noptimizer = keras.optimizers.SGD(learning_rate=0.02, momentum=0.9)\nmodel.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\nn_epochs = 25\nhistory = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n                    validation_data=(X_valid_scaled, y_valid),\n                    callbacks=[lr_scheduler])",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.5904 - accuracy: 0.8058 - val_loss: 0.4759 - val_accuracy: 0.8480 - lr: 0.0200\nEpoch 2/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.4946 - accuracy: 0.8404 - val_loss: 0.5246 - val_accuracy: 0.8340 - lr: 0.0200\nEpoch 3/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.5096 - accuracy: 0.8422 - val_loss: 0.4641 - val_accuracy: 0.8612 - lr: 0.0200\nEpoch 4/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.5048 - accuracy: 0.8469 - val_loss: 0.5777 - val_accuracy: 0.8232 - lr: 0.0200\nEpoch 5/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.5286 - accuracy: 0.8476 - val_loss: 0.4906 - val_accuracy: 0.8536 - lr: 0.0200\nEpoch 6/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.5237 - accuracy: 0.8503 - val_loss: 0.5818 - val_accuracy: 0.8624 - lr: 0.0200\nEpoch 7/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.5291 - accuracy: 0.8524 - val_loss: 0.6646 - val_accuracy: 0.8402 - lr: 0.0200\nEpoch 8/25\n1719/1719 [==============================] - 8s 5ms/step - loss: 0.5261 - accuracy: 0.8537 - val_loss: 0.5651 - val_accuracy: 0.8448 - lr: 0.0200\nEpoch 9/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.2930 - accuracy: 0.8971 - val_loss: 0.4157 - val_accuracy: 0.8776 - lr: 0.0100\nEpoch 10/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2445 - accuracy: 0.9103 - val_loss: 0.4095 - val_accuracy: 0.8864 - lr: 0.0100\nEpoch 11/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.2275 - accuracy: 0.9171 - val_loss: 0.4111 - val_accuracy: 0.8904 - lr: 0.0100\nEpoch 12/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2099 - accuracy: 0.9221 - val_loss: 0.4649 - val_accuracy: 0.8712 - lr: 0.0100\nEpoch 13/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.1961 - accuracy: 0.9277 - val_loss: 0.4395 - val_accuracy: 0.8930 - lr: 0.0100\nEpoch 14/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.1897 - accuracy: 0.9295 - val_loss: 0.4660 - val_accuracy: 0.8812 - lr: 0.0100\nEpoch 15/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.1810 - accuracy: 0.9327 - val_loss: 0.4725 - val_accuracy: 0.8792 - lr: 0.0100\nEpoch 16/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.1230 - accuracy: 0.9517 - val_loss: 0.4385 - val_accuracy: 0.8960 - lr: 0.0050\nEpoch 17/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.1089 - accuracy: 0.9574 - val_loss: 0.4591 - val_accuracy: 0.8912 - lr: 0.0050\nEpoch 18/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.1009 - accuracy: 0.9603 - val_loss: 0.4857 - val_accuracy: 0.8922 - lr: 0.0050\nEpoch 19/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.0949 - accuracy: 0.9626 - val_loss: 0.4869 - val_accuracy: 0.8930 - lr: 0.0050\nEpoch 20/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.0887 - accuracy: 0.9657 - val_loss: 0.5145 - val_accuracy: 0.8952 - lr: 0.0050\nEpoch 21/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.0692 - accuracy: 0.9741 - val_loss: 0.5187 - val_accuracy: 0.8934 - lr: 0.0025\nEpoch 22/25\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.0633 - accuracy: 0.9769 - val_loss: 0.5398 - val_accuracy: 0.8932 - lr: 0.0025\nEpoch 23/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.0601 - accuracy: 0.9779 - val_loss: 0.5382 - val_accuracy: 0.8952 - lr: 0.0025\nEpoch 24/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.0572 - accuracy: 0.9789 - val_loss: 0.5602 - val_accuracy: 0.8946 - lr: 0.0025\nEpoch 25/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.0545 - accuracy: 0.9801 - val_loss: 0.5758 - val_accuracy: 0.8940 - lr: 0.0025\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00140-f53134f9-49fa-4c7c-9a9e-a71c5c19e29c",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "c0fe98e",
    "execution_start": 1659356638229,
    "execution_millis": 926,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 607.875,
    "deepnote_output_heights": [
     281.875
    ]
   },
   "source": "plt.plot(history.epoch, history.history[\"lr\"], \"bo-\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Learning Rate\", color='b')\nplt.tick_params('y', colors='b')\nplt.gca().set_xlim(0, n_epochs - 1)\nplt.grid(True)\n\nax2 = plt.gca().twinx()\nax2.plot(history.epoch, history.history[\"val_loss\"], \"r^-\")\nax2.set_ylabel('Validation Loss', color='r')\nax2.tick_params('y', colors='r')\n\nplt.title(\"Reduce LR on Plateau\", fontsize=14)\nplt.show()",
   "execution_count": null,
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 2 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdMAAAEeCAYAAADRiP/HAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABgq0lEQVR4nO2dd5hU5fX4P4fepImiouxawIICUVGjIlhiTKIRW2I0lhjBEkuMsXeRWPI1UWPFioJdiS0af0ZWsWABBEGKioLIohQFlrbAnt8f5w57d3Zm9s7utJ09n+d5n5n79nt3ds6873uKqCqO4ziO49SfZvmegOM4juM0dlyYOo7jOE4DcWHqOI7jOA3EhanjOI7jNBAXpo7jOI7TQFyYOo7jOE4DcWHqNDlEpEJETs33PIoNEflaRP6a73k4Tj5wYeoUJCLyiIhokNaLyDwRuUdEuuR7bplARAYH99YtSfm1ofuvEpEFIjJGRLbJ9VyD+Zwamo+KSLmIPC0i2zawz4pMztNx8oULU6eQeQPYEigFTgeOAO7O54RyzCzs/rcGfgvsBjydx/msCuazFXAC0B94UUSa53FOjlMQuDB1Cpm1qrpQVeer6uvAU8Ch4Qoi8gcR+UxE1ojIbBG5QESahcp3EJGyoHyWiBwe1740WGntGZevInJs6HqrYGW4RERWicgnInJgqPwIEZkYjPOViIwQkVYNvP/1wf0vUNXxwP3APiLSMVUjETlaRD4VkbUi8o2IXCEiEir/WkSuFJH7RGS5iMwXkYsizEeD+ZSr6jjgOmBXYIck8/iLiEwVkZUi8q2IPCAinYOywcDDQPvQavfaoKyViNwczGuViHwkIj8P9dtcRB4MnvNqEflcRC6O+7s/IiIvx83nWhGZFuE+HSdtWuR7Ao4TBRHZDjgMWBfKGwpcD5wLTMS+2O8P6twZfLmOBX4Afgq0A24HWqc5dnvgLeB7YAiwAOgXKv85MAY4H3gb6AncG4yTkTNEEdkCOBrYEKRk9fYAngFuCOY0ALgPWA78K1T1AuAa4O/AL4A7ROQdVX0/jWmtDl5bJimvAv4MzAFKgvH/BZwEvBeU/Q3YPqgf2/J9OMg7AZgP/BJ4SUQGqOoUbBHwLfAbYBGwFzASWAI8mMb8HSdzqKonTwWXgEeA9dgX7GpAg3RBqM484KS4dn8GPgveH4oJnp6h8v2Dfk4NrkuD6z3j+lHg2OD9UGAF0C3JXN8GrorLGxLMXZK0GRyMkazPa4O5V2Dbq7H7v72O5zYGeDNBX/ND118DT8TV+Ry4MkW/pwIVoeutgfeBb4BWoX7/mqKPw4C1QLNEfQZ522NCuGdc/r+Bu1P0fRPwRtzn5+UEz2Favj/bnooz+crUKWTeBoYBbTGBtj1wB4CIbAZsA9wnIveE2rQAYluaOwPfquq8UPkH2Jd1OvwEmKqqi5OU7wHsJSKXhPKaBfPeAihPc7wYX2KrstbAkcAxwOV1tNkZeCUu7x3gGhHpqKrLg7ypcXUWAJvX0Xf7QGFIsFX+JOBoVa1MVFlEDgIuC+bUCWgOtMKeyYIkY+we9P9ZaGca7Bm8Ger7TOwcvQR7zi2BuXXM33GyhgtTp5BZpapfBO/PE5FxwFXYCiN2PnYmtmVYX2KCNXymmGzbMhnNsPPDZxKULarnvAAqQ/c/XUR6AXdhK7r6EA4RtS5BWV06FKswpaMq4DtVXZmsooiUYEL9fuBqbAt2d+AJTKAmo1kwlwEJ5rg66Pu3wG3YFvp72Bb2n4CjQnWrCP1NA9L9uzpOZFyYOo2J64BXRWSkqi4QkQXA9qr6aJL6M4AeIrKNqn4T5O1FTaERE3ZbhvL6x/UzGThJRLolWZ1OAnYKCb5scQMwS0T+paoTk9SZAewXl7c/ts27ooHjaxr3uCcmNC9Q1Q0A8cpfQCW2Wg0zGROCW6gpOSVif+ADVb0zliEi28fVWUTtv2P8teNkDNfmdRoNqloGfAZcGWRdA1wcaPDuKCK7isjJInJZUP4GMBN4VET6i8hPgX9iZ7GxPlcDE4BLRKSPiOwL/F/c0I9jykcviMhAEdlORH4d0ua9HjhBRK4P5rCTiBwrIrdEuK1dg7mFU8L/S1X9EngBGJ6iv1uBQYHmam8RORG4EIgyl0zyOfb98mcR2VZEfoedZ4f5GmgjIj8TkW4i0k5VZ2Pnvo8Ez3A7EdlTRP4qIkcH7WYDu4vIL0Skl4hcBQyK6/tN4CcicpqYRvfF1P6R4TiZI9+Htp48JUokUCAJ8k/AlFhKguvfYSvDNZjW7jvA8aH6vTFN3LXYF/yvMaWeU0N1dgbexbYxPwUGElJACupsjZnm/BjUmwwMDpUfCowPypYDHwPnpLi/wVQrFcWnDiRRlgH2Dersm6Lvo4P7qMQUhK4gpAhFAkUhoAy4M0WfpxKnLJSgTo1+gfMwrdvVwP8w7VsFSkN17gEWB/nXBnktg/ufE9zDQuBFYI+gvBWmtftD8Pd4ENtK/jpuPtdi59XLMPvkvyV6pp48ZSKJavgYxXEcx3GcdPFtXsdxHMdpIC5MHcdxHKeBuDB1HMdxnAbiwtRxHMdxGojbmUakWbNm2rZt23xPo6CoqqqiWTP/PRaPP5fE+HNJTDE/l7Zt27JkyZLFqrpZvueSbVyYRqRVq1asXJnU4UuTpKysjMGDB+d7GgWHP5fE+HNJTLE/FxFpl+855ILi/DnkOI7jODnEhanjOI7jNBAXpo7jOI7TQFyYOo7jOE4DcWHqOI7jOA0kp8JUhK4ijBVhpQhzRTghST0R4WYRlgTpZhGLTShCbxFeEGGRCEtF+K8IO8a1v0CEhSIsF+EhEVqHykpFGCfCKhFminBIlLmvXduc0lIYMybavY4ZA6Wl0KwZRdcu1uaggwalPdaArct5SwYxYJuFWb83x3GcnJFLr/qgT4A+BdoBdH/QZaB9EtQ7A3QW6NagPUA/Az0zKNsL9I+gXUFbgg4HnRlq+3PQ70D7gHYBLQO9KVT+Pug/QNuCHgP6I+hmdc+9nYJqu3aqo0drSkaPtnpQnYqlXUPHuouzdD3N9E7Ozuq95ZNx48blewoFiT+XxBT7cwFWagFEdcl2ylnUGBHaYyGTdlVldpD3GPCtKpfG1X0PeESVkcH1H4GhquyToN+uwBKgmypLRHgc+FqVy4Pyg4ExqmwhQm8sNFU3VVYE5eOD8ntTz7+9gtmZduoE552XvO4dd8CyZbXzi6FdQ8Zqu6ycOWxHW9awirZsxxzWdNqiXnMsKYGvv07eLp8Uu91gffHnkphify4iskpV2+d7Htkml8L0J8C7qrQL5f0VGKTKEXF1lwGHqvJBcL0nME6VTRL0OwS4R5Utg+spwN9UeSq47gYsAroBBwRlO4fa34kt0M9N0PcwYJhdtd8jJkxBEUl+r/ZIE1Vo/O0aMtZd/ImhjKQlG6ikJfczlHO4s15zFFHefPOt5A3zSEVFBR06dMj3NAoOfy6JKfbncuCBBzYJYZrLLd6BoAvj8oaCliWouwF0p9B1r2CLT+LqbQ36LejvQnlfgh4Wum4ZtC0FPQl0QlwfI0AfqXv+7TZuM5aUaEpKSrTGtmQxtavvWHv2WKCraFOj0Ura6p5bl2fl3vJJsW/b1Rd/Lokp9udCE9nmzaUCUgXQMS6vI9h2ax11OwIVqmxcRouwGfA6cLcqT9TRlmCcdOaQkHbtYMSI1HVGjLB6xdiuvmON2Wk4QlWNvGZsYPSOwzM+R8dxnJyTK6kN2h60ErRXKO/RsHJQKP890KGh69PCK8pAsWhykraPg44IXR8UWxGD9gZdA7pJqPxtAuWm1PNvpyUl0RVfRo+21ZOIFl276jZV0cfq3z/xErN//0jj9exp1Tt1KmzlI9XiX2nUF38uiSn250ITWZnmdjD0SUyjtz3ofiTX5j0TdAamybsV6HSqtXk7gn4IemeSMQ4DXQi6C2hn0Depqc07AfT/QNuAHkVEbd7WrVsn/7Q0UdL+Enj5ZfvI7bKLapcuqlVVaTXfZhvVk09Ob8h8UOxfjvXFn0tiiv25NBVhmmunDWcDbYHvgSeAs1SZLsJAESpC9e4DXsI0b6cBrwR5AEcBA4A/iFARSj0BVHkNuAUYB8wD5gLXhPo+HtgT0yy+CThWlUVZuVunJhMmQPPm8Ic/wA8/wPz5aTUvLS1cDV7HcZo2OQ3BpspSYEiC/PFAh9C1AhcHKb7uKGBUHeP8A/hHkrKvgcHRZ+1kjAkToG9f2CewcJo6FbbZJnLzkhIYPz5Lc3Mcx2kA7k7QyQ1VVfDhhyZId9vN8qZOTauL0lJbzK5fn/npOY7jNAQXpk5umDkTli+Hvfc2Dw+lpTBlSlpdlJbChg1p7w47jlPIiHRFZCwiKxGZi0hCN7NB3d0ReRuRCkS+Q+T8UNnXiKwOyioQeT0X04/hwtTJDRMm2Gtsi7dv37RXpiUl9jp3bgbn5ThOvrkLqAS6AycC9yDSp1YtkW7Aa5j+zKbADph5ZJgjUO0QpEOzOus4XJg6ueGDD6BLF+jVy6779YNZs2DNmshdlJbaqyshOU6RINIeOAa4CtUKVN8BXgROSlD7L8B/UR2D6lpUV6A6I5fTTYULUyc3TJgAe+1loV/AVqZVVTB9euQuYrpKLkwdp2joDaxHdXYobwpQe2UK+wBLEXkPke8ReQmRnnF1xiCyCJHXEemXrUknwoWpk30qKmDatOotXjBhCmlt9bZuDVtt5du8jtOY6AYtEPk4lIaFijsAy+OaLIPaftiBrYFTgPOBnsBXUMP73YlAKVCCmUb+F5HOGbmJCOTUNMZponz8sa1Cw8J0++3NL2A9NHp9Zeo4jYfFtvLcM0lxOi5eVwNjUf0IAJHrgMWIdEJ1GarvhureiMgpwEDMZ0HW8ZWpk31iykd77VWd17w57Lpr2hq9JSW+MnWcImI2tnLtFcrrByQ6/5kK1f7Z494nQkkc4ioruDB1ss+ECdC7N3TtWjM/ptGr0cMAlpbCvHlmIuM4TiNHdSXwPHA9Iu0R2Q84EngsQe2HgaMQ6Y9IS+Aq4B1UlyHSE5H9EGmFSBtELsLCbr6boJ+s4MLUyS6qpsm79961y/r1gyVLoLw8cnelpea0YcGCzE3RcZy8UsvNLKrTERmISLWbWdU3gcsx97LfY6YxMZvUTYB7MDex3wKHAb9AdUmubsLPTJ3sMm8eLFxY87w0RkwJacoU0yyKQNjWNA1PhI7jFCqqCd3MolrDzWyQdw8mNOPrTgf6ZmN6UfGVqZNd4p01hKmHRq/bmjqOU4i4MHWyywcfQJs21f54w3TuDD17piVMewZWZa6E5DhOIeHC1MkuEybAnntCy5aJy/v2TUujt21b6N7dV6aO4xQWLkyd7FFZCZMmJd7ijdGvnznBX7s2crdua+o4TqHhwtTJHlOmmJBMpMkbo29fs3OZEd3FptuaOo5TaLgwdbJHKuWjGGGN3oiUlpowraqq/9Qcx3EyiQtTJ3tMmAA9esDWWyev06uXKSiloYRUUmI7yAsXZmCOjuM4GSCnwlSEriKMFWGlCHNFSBgEVgQR4WYRlgTpZpFqt1AijBRhlghVIpwa1/ZeESpCaa1ItZ9HEcpEWBMqn5W1G27qJHPWECbmVrAe5jG+1es4TqGQ65VprSCwIglD7QzDjHj7YYa4RwBnhMqnYF4zJsU3VOVMVTrEEuZR45m4aueE6uzYwHtyErFoEXz5Zeot3hgxjd6IbgXd1tRxnEIjZ8JUhI1BYFWpUCVVENhTgFtVma/Kt8CtUL0CVeUuVf4HpIwsHRpzVGbuwonMBx/YaxRh2q+fCd/vvovUddgLkuM4TiGQS3eCvYH1qsQHgR2UoG6foCxcL9EKti6OARYBb8fl3yjCTcAs4ApVyhI1FmEYtkqmRQuhrCxhtSZLRUVF0mdS+vTTlDRrxviVK6mq47l1rqqiPzDlscf4YcCASGN36rQv7723mLKy2XVXzjGpnktTxp9LYvy5FAe5FKbpBIHtEJSF63UQQVTrDLsT5hTg0bg2lwCfYdvNxwMvidBflS/jG6syEhgJ0KaN6uDBg9MYuvgpKysj6TMZMQL69eOAww6ru6O+feGCC+gHEPEZ77ADVFZuxeDB0Xz65pKUz6UJ488lMf5cioNcnpmmEwQ2vm5HoCIdQSpCT2Aw8Gg4X5UPVFmhylpVRmEhen4ZtV8nAlVV8OGH0bZ4wUKzbb112kpIvs3rOE6hkEthOhtoIUKUILDTg7K66qXiJOBdVebUUS+nAWSbBDNnwvLldWvyhknTrWDMC1IaoVAdx3GyRs6EqSobg8CK0F6EVEFgHwX+IkIPEbYCLgQeiRWK0EqENpgQbClCG5Fa93JyuE3QrrMIPw/qtxDhROAA4LXM3KUDRHPWEE/fvuYFqbIyUvWSElizBr7/vh7zcxzHyTC5No2pFQRWlekiDBShIlTvPuAl4FNgGhYM9r5Q+evAamBf7ExzNSYUARDhp8DW1DaJaQncgCklLQbOBYbEKUU5DWXCBOjSxRwyRKVfP4v6PXNmpOpua+o4TiGR0+DgqiQMAqtKjSCwwdnoxUFK1M/gOsZ5H2ifIH8REE1d1Kk/H3wAe+0FzdL4rRZ2K9i37hi/YVvTvfZKe4aO4zgZxd0JOpllxQqYNi29LV6A3r2hdevISkhua+o4TiHhwtTJLB9/bNq86QrTFi2gT5/IwrRjR9tJdi9IjuMUAi5MncwS83xUn73XNDV6S0pcmDqOUxi4MHUyy4QJtmXbtWv6bfv2NZeCEd0Kuq2p4ziFggtTJ3OomjBNd4s3Rr/AtPjTTyNVd1tTx3EKBRemTuaYN89Wlek4awiz2272GnGrt6QEVq6EJUvqN5zjOE6mcGHqZI76OGsIs9lmsOWWkZWQ3NbUcZxCwYWpkzkmTIC2batXmPWhX7+0zWNcCclxnHzjwtTJHB98AHvsAS1b1r+Pvn3hs89g3bo6q/rK1HGcQsGFqZMZ1q6FSZPqv8Ubo29f8887a1adVTt3NntTX5k6jpNvXJg6mWHKFBOoDRWmMY3eCFu9Im5r6jhOYeDC1MkMMWcN9dXkjbHjjrZNHFGj121NHccpBFyYOplhwgTo0cOCfDeEli1hl13SUkJyW1PHacSIdEVkLCIrEZmLyAkp6u6OyNuIVCDyHSLnh8pKERmHyCpEZiJySC6mH8OFqZMZGuKsIZ40NHpLSy0O+Y8/ZmZox3Fyzl1AJdAdOBG4B5E+tWqJdMNiT98HbArsgIXjjPEEMDkouwJ4FpHN0p6NSL00KF2YOg1n0SKYM6fhW7wx+vaFBQtg8eI6q7pGr+M0YkTaA8cAV6Fageo7wIvASQlq/wX4L6pjUF2L6gpUZwT99AZ2B65BdTWqz2HxsI+pY/zzEDkmdP0gsBqRWYjsmM6tuDB1Gk7svDRTK9NYPNMIq1O3NXWcwqYbtEDk41AaFiruDaxHdXYobwpQe2UK+wBLEXkPke8ReQmRnkFZH2AOqisi9BPmPGARACIHAL8BTgA+AW6NdodGToODO0XKhAnQvLnZmGaCsEbvQQelrOorU8cpbBabsNwzSXEHYHlc3jJgkwR1t8ZWnz/DVp23YFu7+wX9LEvQT486ptcD+Cp4fwTwDKpPI/IpML6OtjXwlanTcD74wFaT7dplpr/NN4fu3SNp9G66KbRv7ytTx2mkVAAd4/I6AisS1F0NjEX1I1TXANcB+yLSKc1+wiwHNg/e/wz4X/B+HdAm0h0ERBamIvxChJdF+EyEbYK800U4OJ0BnSJjwwYTppna4o0RUQnJbU0dp1EzG9sG7hXK6wdMT1B3KhDW2w+/nw5sh0h4RZusnzCvA/cj8gCm0PRqkN+H6hVrJCIJUxFOBJ4GPge2BWLaTs2Bi6MOJkJXEcaKsFKEuSIkVIEWQUS4WYQlQbpZBAmVjxRhlghVIpwa1/ZUETaIUBFKg0PlpSKME2GVCDNFyKn6dNExcyasWJE55aMYffvC9Omwfn2dVd3W1HEaKaorgeeB6xFpj8h+wJHAYwlqPwwchUj/QOP2KuAdVJcFZ66fANcg0gaRo4C+wHN1zOBPwLvAZsCxqC4N8nfHtpAjE3VlejEwVJULgPC32wSgfxrj1VKBFkl4QDwMGIL9suiL7WWfESqfApwNTEoyzvuqdAilslBZLfVpEdJXn3aMTCsfxejb1zwqzZ5dZ1VfmTpOo+ZsoC3wPfb9fBaq0xEZiEjFxlqqbwKXA68EdXeAGguy44E9gR+AmzDhuCjlyKrLUT0X1SNRfS2Ufw2qf0vnJqIK017A+wnyE+1TJ0SEjSrQqlSokkoF+hTgVlXmq/ItplV1aqxQlbtU+R+wJuL8Y3PYqD6tympVoqlPO8mZMAG6dIFevequmw5puBUsLYUffjB7U8dxGhmqS1Edgmp7VHui+niQPx7VDnF170G1B6pdUD0C1W9CZV+jOhjVtqjuiOobdY4tsksNExiRnyEyGpHLEGmezm1E1eZdgKkwx2+mHQB8GbGP3sB6VeJVoAclqNsnKAvXq0vFOcxPRFgMLMW2C25UZX3QxxxVIqlPizAMWyXTooVQVlaWxhSKn4qKCir+9z8qd9iBqW+/ndG+Zd06BjZvzjcvvcRXW2yRsu7KlZsBfXj22Y/YbruVGZ1HfaioqPDPSgL8uSTGn0teeQi4DZiFyDbAC0AZtv3bEbgsakdRhelI4A4RTg+utxFhIKaafG3EPtJRgY5Xc14GdBBBVKnLcdzbwK6Y4O8DPIVtTd+YoN9Y3wnVp1UZid07bdqoDh48uI6hmxbj//MfOnz9NZx0Ell5NrvsQsmPP1JSR99t28L118Nmmw2gEP5EZWVl2XkejRx/Lonx55JXdqL6uPBY4ANUf4nIgdgZbWaFqSq3iNAJ+H+YuvA4YC3wf6rcFXGsdFSX4+t2BCoiCFJUmRO6/FSE64GLMGFaX/VpJwGbzJoFVVWZPy+N0a8fRPjF7ramjuPUk+aYHg/AwcB/gvdfYro9kYlsGqPKFUA3YC/ME8VmqlyVxlizgRYiRFGBnh6U1VUvCgobNYGnA9uJkK76tJOAjp99Zm/22is7A/TtC/Pnw9KlKattvjm0aeNKSI7jpM004CxEBmLCNKaE1AOo259piKimMQ+JsIkqq1T5WJUPVakQob0ID0XpQ5WNKtBBu1Qq0I8CfxGhhwhbARcCj4Tm00qENpiQbClCGxG7l8AetnvwfidMffqFYA4b1aeDNlHVp514ysvZ+umnYbvtoGvX7IwR0a2g25o6jlNPLgGGYuekT6D6aZD/a+DDdDqKujI9BVNdjqctcHIa49VSgVZluggDRagI1bsPeAnTtJ2GqULfFyp/HfOGsS92prkaU4YC+3UxVYSV2JL9eSCs4lxLfVqV1OrTTm2uv56Wy5dbyLRskaZGr2/zOo6TFqpvYzam3VA9LVRyH3BWOl2lPDMVoSu2+hOgi0gNG9PmwK+A76IOpspSzH40Pn88phwUu1bMtjWhQwjVaicMCcr+Cvw1RfnXkLy9E4Hycnj4Yds7nzMHFi6EOjRu60X37rDZZpHcCpaUwMSJmZ+C4zhFjuoGRFYjsit2LPglql+n201dK9PF2CpSgc8w7/qxtBB4ALg73UGdRs7w4eZGMHydDURsqzfiynTxYliZf8sYx3EaCyItEPk7tlM5BdsN/QGRW9KNa1qXMD0Q2zYVTG34oFDaH+ipyog0p+9ki/JyGDTIVorZHOPhh6vd/K1bZ9fZGrNfP5g2rabwTkAsFJtv9TqOkwa3AL8HzsR8IfTCtndPwixAIpNSmKryVuCKb1vgheA6lt5XZUF9Zu9kiSuugHfeyd5KEazvqqqaeRs2ZG/Mvn1hzRr4/POU1WLmMa6E5DgFQvDjvkW1NUUhcgLwR1RHofplkB4BTsdc3kYmkgKSKnNVqRJhKxH2EeGAcEp//k7GefttWyFWVWV3pfj++1BZWTOvshLeey8740XU6HVbU8cpMIYPh3feYavqwCiFSCcSe/H7EuicTkdRTWO2EqEMmI952C/DHDfEkpNvrruu+n02V4qTJ8NXFplo9gUXgKqlyZOzM94uu1jg8TqE6RZbQKtWvjJ1nIKgvBweegiqqtg0uqe9fDAFOC9B/vmYGWVkoprG3AZsAHYBVgEDgeOAGcBh6QzoZIHycluZxqiszO7qNFCbXdG7d3b6D9O6Ney0U50avc2aQc+eLkwdpyD4058s6hOFvceLWYycgsgsREYFaRZ2jnpROh1FFaaDgEtUmYlp9i5S5XnM4DWLB3ROJHJ9jjlpErRowcrttstO//GkodHr27yOk2dGjoSxYzdeFrQwNTvT3sCzmHlmB+AZYEdU30mnq6jCtC3VrpWWApsH7z/DPAg5+eT992sL02yeY06cCH36UNWqVXb6j6dfP5g3D378MWU194LkOHlEFW65Bc44w8zaGguqC1C9AtVjgnQl0BKRp9PpJqownYl51wfbRz5ThBIsTM236QzoZIEPPzRPRIccYtdPP529c0xVW5nusUfm+05GGkpI330Hq1dnf0qO44RYvx7OPBMuuQQ6d7bvicZNZ9KMcx1VmN4OxFzcXA8cCszB3ANens6AThaYM8fsPX/zG9PC+eij7I01fz4sWgS77569MeKJKExjtqbz5mV5Po7jVLN8ORx+uG3vXnYZLFlSrZioykTTsyl6oprGjFE1R/OqTAJKgQGY04ZnsjY7JxozZthrv36WsilMYz77crky3Wor6NIFRoxIqVTltqaOk2O++Qb23x/eeAPuvx/+9jfTBmyC1Ouug+gxk4CVIlya4Tk56TJzpr3uuCMMGGACL/4MNVNMmmT/LH1zeFQuYhHAFy5MqVTltqaOk0MmT7ZYxnPnwquvwumn53tGeaVO+x8RugF7A+uA/6myQYSW2HnpZZjD+5uyOksnNTNm2OqtUycTpnffDbNmwc47Z36siRPN9rNdu8z3nYzycjsMBTP5ueqqhI71t9oKWrTwlanjZJ1XXoHf/tbCL77zDuy2W75nlB4iL9ZRo2O6XaZcmYqwL/A5Fg7tVeDdIEboVOAczCymZ7qDOhlm5kyzxQQTppC9rd5Jk3J7Xgo1V6Pr1yddnTZvDtts48LUcbLKXXfBr39tO2ETJjQ+QWosqSN9hcXVjkxdK9PhwH+BG4A/ABcAL2NKSI8FodKcfKJqK9OTg7CyO+0E7dubMI3lZYoFC2yrNZfnpTHH+jFH9zHH+klWpyUlvs3rOBmnvNxWojvvbIpGRxwBjz8OHTrU3bYQUf1Dprus68y0HzBclWnAVZjDhstUedQFaYFQXg4rVlRv6TZvbsIuGyvTSZPsNZcr0zQdUpSW+srUcTLO1VfD+PEmSM8915wyNFZBmiXqEqZdsdilqLIKU3HOkhNWp17ENHlj27xgW72ffFLbIX1DmTjRlIH6989sv6lI07F+aan9vgg8mTmO01BmzoQHH7T3LVvC5Zfbj3anBlG0ebuI0FWETbGVacfgemPK8hydVMSEaVjZaMAAkybTpmV2rEmTTGjn8hfp5MnVNmtbbAF//GNKhxQlJVb8zTe5m6LjFC0bNsChh1Y7YRDJbojHRkwUYfoZtjr9HvNb+FFwvQhzMbgo6mCB8B0rwkoR5opwQpJ6IsLNIiwJ0s0i1S4eRRgpwiwRqkQ4Na7tKSJMFGG5CPNFuEWk+mxYhDIR1ohQEaRZUedfkMycCZtsAltuWZ2XLSWkiRNzr3wUJsKBqNuaOk4GOeecmr9Msx1EoxFTlzA9EDgolJJdR+UuoBLojgVevUeEPgnqDQOGYGe2fYEjgDNC5VMw70uTErRtB/wZNpr0HAz8Na7OOap0CNKOacy/8Jgxw1alYV+Y225rKuuZFKbffQfffptb5aN4evaMLExdCclxGsijj8K999Z2wpDNIBqNmJTavKq8lamBRGiP+TrcVZUK4B0RXgROglqOH04BblVlftD2VmAocG8wr7uC/DUJ5nxP6PJbEcZgQr84mTmz2idvDBHYc8/MCtN8KB/FU1ICL75oCklJvKz06GFFvjJ1nAbw4YcwbJgd6VRU1CzLZhCNfCDSDuiPBXCp+cWi+nzUbnIZtLU3sF6V2aG8KVh4t3j6BGXheolWsFE4AJgel3ejCDcBs4ArVClL1FCEYdgqmRYthLKyhNXyRvOKCgYuWMCc1q2ZFze3bTffnJ5vvMH4116jqk2bBo/V87nn2A4YX1HBhmCsioqKnD6THpWV9Fq7lvf+/W8quyY/qt9ss3344IMfKSubmbO5hcn1c2ks+HNJTKE9l1aLF7PHmWdS1aULk+69l3WdOiWuWEBzrjcihwBPAJsmKFXMKVE0VDUnCXQg6MK4vKGgZQnqbgDdKXTdK9BCkbh674CemmLM00Dng3YL5e0Nugloa9BTQFeAbl/X/Fu3bq0Fx4QJpprz73/XLvv3v63s3XczM9ZRR6n26lUja9y4cZnpOyovvmj3NGFCymoDB1rKFzl/Lo0Efy6JKajnsnq16l57qbZvrzp1aka6BFZqqu9X6KowVmGlwlyFE5LUu1ZhnUJFKG0XKtegj1jZAynHtTbTFR5R2KrOunWkXHokrqC2i6aOwIoIdTsCFenYtoowBLgR+IXqxlisqPKBKitUWavKKOBd4JdR+y0oYj55w2YxMTKthJTrsGuJiBgWxm1NHaceqNrW7ocfwmOP5dKzUS1dGkSS7UQ+hWqHUJoTV94vVBbFWXApMBzVBfWefUAuhelsoIUIvUJ5/ai9BUuQ1y9CvYSIcBhwP3CEKp/WUV0p8GDwSZkxw+y+ttuudtlWW1nKhDBdssQ0evJ5XgrVwjSCEtK335qzJMdxIvKPf5gQve46OOqo3IwpEtOluQrVClTfgY26NLngXciMEmrOhKkqK4HngetFaC/CfsCRwGMJqj8K/EWEHiJsBVwIFgIOQIRWIrTBhGBLEdqI2L2IcBAwBjhGlQ/DnYrQWYSfB/VbiHAidqb6WsZvOBfMnAk77GACNREDBmRGmOYj7FoiOnWCjh3rFKYlJaajNH9+jublOI2d//4XLr4YjjkGrrwyo113gxaIfBxKw0LFvYH1qMbr0iRbmR6ByFJEpiNyVoLytxFZiMjziJRGmN69wP8hcjoieyOye42UBpEUkER4KEmRAmuAL4CnVKlrqXw28BBms7oEOEuV6SIMBF5VJeYN4D5gO9i4qnwgyIvxOtWKS/sCIzGN3TLM7WEn4D8ha5HxqvwCaIn5Gd4J2ADMBIbEKUU1HmbMgF13TV4+YAC88AL8+CN07lz/cWKavD/5Sf37yBRp2ppuu23WZ+Q4jZvZs83v7m67wahRGY9HutiE5Z5JijsAy+PylgGbJKj7NPZd/x1m9vgcIj+i+kRQPgiYgJlH3gC8jEh/VNenmN6zwevIBGVpKSBF1ebdDBgIVAExtzq7YivDicDR2IpzoCqfJOtElaWY/Wh8/njYKEgJzkYvDlKifganGCOpGYwqi7Cg5o2fykr48ks47rjkdWLnphMnwsEH13+siRNtK7lLl/r3kSkiCNOIu8GO4yxbZhFgWra0H97t2+d6BtF1aVQ/C129h8jtwLGYNi6ovh2UVSJyPiakd4aUR30Z+7kdVZi+i930H9V89CJCO+xccgqmwPMocCvmJMHJNl98YcbTiZSPYuwZ/Bj86KOGCdNCUD6KUVJiDrdTsM02ZmrrSkiOk4ING+B3v7Mf5f/7X/Wv0NwyG9sG7oXq50FeVB2ZuvRd6taHUc3YT+6o6/nzgetjgtTmwCpgBHCBKpXAzZjhq5MLEvnkjadrV9h++4adm/7wA8yZk3/loxglJfZretmypFVatzbdK1+ZOk4KLr8cXn0V7rwTDjggP3NQ3ahLg0h7RJLr0ogciUgXRASRvYDzgBeCsj6I9EekOSIdsIXdt8CMOucg0heRR4Pz3I8QGYVIivOzxEQVph2ALRPkb0H19uxycusEomkTM4vZsQ5FtIYqIcUcyhfSyhTqNI8pKfGVqVPElJfDoEH195E7ZgzccgucdRaccUbd9bPL2UBbTJfmCeAsVKcjMhCRsPul4zH9nBXYTujNqI4KyroDT2FyaA5m8nI4qql1+kV+jbml3QZ4FVNG7QlMRuSIdG4iqvAbCzwowsWYo3uws8dbsF8VAHtBI1XkaYzMmGH7mXVFcBkwAJ580nzrdu+e/jgxTd5CWZn27Gmvc+emtIMrLS0uj2eOU4Phw+Gdd+z1rruitysvh1/9Cj77zITx7bdnb45RUU2oS4NqDV0aVH+Xoo83qZ+Jyw3ACFSvqZErcn1Q9lLUjqIK0zOBfwCjQ23WY5q5MSfyMzD/uU4umDkz9RZvjPC56eGHpz/OpEm2zNs0kbetPBBRu6ikBJ5+Gtavhxa+X+IUE9OmwX33mf3XffeZs4XttrMfy5tvbq/du0O3brXN5i691HabOnSAZ55JblbXdOhNYvPMx0iiAJuMSF8zwfnomSJcCGwfZH8Z2I7G6nySzsBOA6iqMmH6xz/WXXf33U3V/eOP6ydM8x12LZ7u3aFVq0jmMevXw4IF1YtZx2n0rFljgS2qqux6w4ZqwZqITTetFrIdO8JLwUJr3Tpr63wP7IFtH4fZAzPBiUxav9kD4Tk1nTZOFpg/H1aujLYy7dDB6tXn3HT5cvj8czjllPTbZotmzdIKxfb11y5MnSKhqgqOP96ObMK0bg1Tgrgg330H33+f+PWtt6qDfKumv0VcnNwP3IfIDkDsYGg/bMf17+l0FNVpQxtMo/dgEoSpUaVvOoM6DSSVT95EDBgAr7xi/0CSWlO8BjHlo0JamYLt4UZQQALX6HWKiGuuMVvQ5s1rrio3bIDbbjPB2KtX4rbl5TXdjsaCfF91FWyxRVanXeDcgJl9XgjEgrQuAK4B7kino6javHdjMUe/Bv4NPBeXnFwSxSwmzIABsGhRnQKoFoUQwzQRERw3xFajrtHrFAWjRsENN5i5W/z2bJT4osOH194K9iDfsbBp/0R1a8xzXidUt0b19iDMWGSibvMOAY5T5Y00p+pkg5kzzT3g5ptHqx+OIJOOYfbEiRZtuz5awNmkZ0/7pb12rW1xJaBtW/vB7StTp9FTVgZDh8JBB8Frr9VPaej9903ohim2IN8NRTVRBLPIRBWmq4BvGjKQk0FmzLBVadQt27597R/wo4/g2GOjjzNxYuHYl4aJ/SD45htz9J+imq9MnUbNrFlw9NHmfOXZZ+uvfRs7snFAZCowCNUfEPkUUoT2VI18hBlVmN6CRXE5M52Yok6WmDnTbMWi0ro19OuXnhJSRYX9I/8uuWlX3ggfiKYQpqWlpsTsOI2SxYvt/7xFC9N5KATf2MXBc8Da0PuMyLSowvRnmKP7w0T4DKjhVUKVX2diMk4EfvjBNPOiKh/FGDDAvJ5UVUWLCvHJJ6awVGjnpZCWrenzz0e/ZccpGNautZii8+fDuHGJYxY79UP1utD7azPVbdSvmMWYF6Q3gYVY+LRwcnJFuspHMQYMMFOX2RGdVMWUjwpxm3frrW2LO4J5zLp1drzqOI0GVTjtNPNwNGoU/PSn+Z5R8SLyJiKdE+R3ROTNdLqK6rThD+l06gSUl5td2FNPZU79PF2zmBhhJaQobSdOtDlvmcglc55p1co82dehnRy2Ne3RI+uzcpzMcN118Pjjpr3729/mezbFzmCgVYL8NthubGR88yubhP1nZooZM+wMNN2o1zvvDO3aRT83LaSwa4mI4LjBbU2dRsfo0SZMTznForo42UFkd0RiZ1h9N15bGgAMw6LORCbpylSEqcAgVX4QIaXGkzttSMCnn8I999j7TBpHz5wJvXub4XY6NG9u559RhOmqVeYI+6ij6jfHXFBSAh9+WGcVcI1ep5Ewfry5CB08GEaOTM/BipMuH2MyTYHXE5SvBs5Np8NU27xhjadn0+nUAc48s/p9zDg6E667Zsyov1LQgAEm4NetS61iP3Wqae0U8sq0pASeey6ldlH79ubr21emTsHz+ecwZIidTTz3nB1lONlkWyxw+Bws4tmiUFkl8D2q6TkvNgcQuUmgXUHHgq4EnQt6QpJ6Anoz6JIg3QwqofKRoLNAq0BPTdD+AtCFoMtBHwJtHSorBR0Hugp0JughUebeunVrjcyCBaoiqqZKYKltW9Xy8uh9JGL1atVmzVSvvrp+7R9/3OYyeXLqenfeafXmzUtZbdy4cfWbRya4+26b4/z5SauMHq3aqpVVKymx6yiMHm31RerbrirH4xV2uxh5/bwUKgsW6I+77KK67baqm26q+sUX+Z5RxgFWag7lTL5SroXpE6BPgXYA3R90GWifBPXOCITl1qA9QD8DPTNU/ifQg0E/jhemoD8H/Q60D2gX0DLQm0Ll74P+A7Qt6DGgP4JuVtfc0xKmxx1XU5CCfauffXb0PhIxdar19fjj9Wv/+efWfuTI1PVOO011s81Uq6pSVsvrl+Mrr9i9vPtuwuLRo1Xbtav5J2jXrm4B4O0y2y6MC9MEDBumVWA/ksePz/dsskLBC1NoobCvwvEKJ9dIafQT1dF9V2AEyR3dd4zQR3vgGGBXVSqAd0R4ETgJ8/sb5hTgVlXmB21vxWKl3huMd1eQvybBUKcAD6oyPagzHBgDXCpCb2B34FBVVgPPifDnYF731nUPkSkrq52XCddd9TWLibH99mb4/dFH5p4sGbGwa4V8ZhM7EJ03D/bdt1bxFVfY0W+YVavgrLOqrX4Scf/93i6ddldcASeemLydk4TKSnj9dXjgAQTMMUMKByROlhDZCQsAHtv23YAdf67DjjkfjdpVVKcNDwI/AUZiHvXr4zGiN7BelbCh4xRgUIK6fYKycL0+EcfpA7wQ17a7CJsGZXNUWRFXnrBvEYZhWl20aCGUJRKS8W02bOCn69ezbP/9mT58OLtdeiltv/2WDx8L4s9G6CMZJa++SqkI4xcupKqe/fTdfntajhvHxCTtm1VWsv+0aXyz6658VccYFRUVkZ5JNmi+ahUDgS/HjeObBIpd8+YNwv43arJihXLPPcmPQlavbu7t0mg3b55SVvZW0nZh8vl5yTfNV62i4/TpdPr0Uzp9+ikdZ8yg+dq1G8urqqooP/NMPv/zn/M3yabJbcBEoD/mQ6E/5vD+HuDKtHqKsnwNzh73bshSGnQg6MK4vKGgZQnqbgDdKXTdK9hakrh67yTY5v0S9LDQdcugbSnoSaAT4uqPAH2krvlH3uaNbT+OHWvXd91l17NmRWufiuOPVy0tbVgfl1+u2ry56qpVics//NDm+9xzdXaV9227Ll1UzzorYVFJSc0tyVgqKUndpbfLbLswef+8ZJMFC1QPOKBaL2LhQtVnn1U9/3zVPfawbdzYdu4ee6iefrpqy5Y1H2Ym9CoKEAp5mxeWKOwavF+msGPwfpDC1HT6impn+j0W860hVECt7eCOUGOVmKxuR6DC7jHtcWLvV6Q5h/oxapRFt//lL+065kP35Zcb3vfMmfXf4o0xYIBpF3/ySeLyQg27logUodhGjDCz2jDt2ll+KrxdZts1GS65xExbDjzQTNe22MKCSowcCZtsYvvhr78OP/5oDqNbtqx9jOIh0fKBYIFcwDR6Y+5d5gPp7btHkbigvwV9DrRDfX8BgLYHrQTtFcp7NKwcFMp/D3Ro6Pq0+BVlkJ9oZfo46IjQ9UGxFTFob9A1oJuEyt8mpNyULEVamf7wg2rr1qrnnFMzf7fdVA88sO72qdiwwX65XnBBw/qZP99+Bd9+e+LyoUNVu3atU/lItQBWGkceqdqnT9Li/Gm7Fq82b8+e9vHp0MG1eTfy7rs1V5iHHqp6yy2q77+vunZt4jb9+yde6vfvn9u55wAKe2X6tsJRwfvHFf4brEpHp7syjSoIPwVdgZmTzACdGk6RB0OfxDR624PuR3Jt3jODcXqAbgU6nZravK1A24C+G2wVtwFtFpQdhpnF7ALaGfRNamrzTgD9v6DNUWRSm/e+++yRfvRRzfzLL1dt0cKEbX2ZM8f6rksTNwpbbqn6+98nLtt9d9VDDonUTd6/HM87z77VIwj+XJL355JlfvEL1V12Sb9dUT6XDRvs/ykmDOuhtV+UzyVEgQvTnyscHbzfTmGGQpXC9wqD0+kr6jbvs8D/ATcDT2IOHcIpKmcDbbFt4yeAs1SZLsJAkRrbyPdhGlafAtOAV4K8GK9jHir2xZSiVgMHAKjyGhYybhwwD5gLXBNqezywJ/ADcBNwrGoNg936M2qUbcPGOzs4/HBYvx7++9/6911fn7yJGDAgsSekykrz3FTIzhrClJRYqLgff8z3TJoUgwaZg6zvv8/3TAqAW26pGUmhstI8ni1cmL85OdFR/S+qzwfv56C6M9AN6I5qWTpd1anNK0JLoD1wlyoN8iWjylJgSIL88UCH0LUCFwcpUT+D6xjnH8A/kpR9Danb14svvjDTl5tuqn0Wstde5orn5Zfr77i6oWYxYQYMgBdfhGXLoFOn6vxp08w7UmM4LwXzzwt2buqxHnPGoED//u2304s1X3TMm2duQkVsXRojkx7PnNyjurQ+zeoUpqqsE+Es4O76DNBkePRR+6dKZHTXvLkpJL38sv2jpetXF2xl2q2bpYay5572OnEiHHRQdX4hh11LRNiTff/+eZ1KU2KPPcxV41tvNWFhqgpnnGH/z2FBCpmxKXeyh8g4iGjeqXpQ3ZWMqNu8rwORO21yVFWZMD3kEIu1mYjDD4elS+H99+s3xowZmdnihWphGr/VO3GirVQbSyBiDwuTF1q2hP32M2HaZHnsMXjtNbjttkRqRDB5cr5n6CRnGjA9SDOBPTAt3vlB2irIm5FOp1GdNvwP+JsIfTED15XhQlWeT2fQomP8ePtCT2Un8POfm5eTl1+G/fdPf4yZMzMXxaVbNwvh9vHHNfMnTSp8z0dhNtsM2rZ1YZoHBg0ya4/FizOzWdKoWLgQ/vxn+0Vxzjn5no2TLqrV0WBE/gmMAs4PNFRj+beRyEtJCqKuTO/E3AieFwz8bCg9k86ARcmoUdChg0V9SEbHjvYNVB9708WLLWVqZQq1lZDWrYMpUxrPeSmY0I8Q19TJPLFz0/Hj8zuPvPCnP5kvxQcfTBqxyGk0nAzcWUOQGndjrm4jE+mToEqzFKkeB4BFxKpV8MwzcNxxdpCUisMPh+nT4auv0hsjk8pHMQYMMCG0aFH1GGvXNp7z0hglJaYI4uSUAQNsU6DJbfU++yw8/zxcey3suGO+Z+M0HAF2S5CfKC8l/rOqoYwda+YZp5xSd93DD7fXV15Jb4xMmsXEGDDAXmOr04kT7bUxClNfmeacVq3gpz9tYsJ0yRJble6+O/z1r/meTfEg0hWRsYisRGQuIickqXctIusQqQil7ULl/RGZiMiq4LV/hNEfAh5A5FJEBgfpUuB+4OF0biPqmSkidAF+AfQEakSuVeX6dAYtKkaNsi/0gQPrrrvDDiYQX3opvbOWGTNsGRBTuMkEsbPRjz4yTeNJk8ztWWOLXNGzpxk8rl5tz8jJGYMHwzXXwA8/NBHLpD//2ZQIX3/d9B+cTHEXFpC7O+Zo/hVEpqA6PUHdp1D9fa1ckVZYgJPbsC3aM4AXEOmFamWKsS/G/B6cD/wtyCvHfBDcms5NRFqZirAP8AXmuGE4cBpwBfBXoKkqx8O338Ibb8DJJ0c/Ozn8cIscsyINd8AzZ9qWUibPZzbZxLaNwyvTn/yk8Z0BhUOxOTll0CBTXG0S56b/+Q+MHg2XXQb9+uV7NsWDSCw051WoVqD6DmwMzZkOg7HF4W2orkX1DmwLN7UVimoVqreg2gPoDHRGtUeQlzyMUgKifnP+HYsJ2gNYE0ywJ/Ax5hWpaTJ6tH2bnHxy9DaHH252aG+8Eb1NJs1iwsSUkNavN8f3jUn5KIabx+SNvfaC1q2bwFbv8uVmU7rLLqbC7KRFN2iByMehNCxU3BtYj2p8aM5kITePQGQpItMROSuU3weYGqdINDVFP7VRXY7q8sj144gqTPsCdwaeiTYArVX5DrgEuLa+gzdqVG2Ld99909sa3Xdf6Nw5ulbvqlUmKDKpfBRjwADbIn3jDdsmbWznpeDCNI+0aQP77NMEhOnFF8OCBfDQQ/brwUmLxSYs9wylkaHiDkC8AFsGbJKgq6eBnYHNgKHA1Yj8LtTPskj9iExFpEvw/tPgOnFKg6gb/+E95++AEsygtQIzcG16TJxoK8b77qu7bpiWLeGww0wJqaqq7m3V2bNNcGdrZQpw77322hhXpj16mEcp3+bNC4MHm+e8eM+URcO4cfY/fuGFsPfe+Z5NMRI9LKbqZ6Gr9xC5HTtmfCKtfsyffCwy+7PpTzkxUYXpJGAAMBsoA24QoTvwe2wp3fQYNcp+pf7mN+m3PeIIePJJc5qw116p62bDLCZG377VjiTatWucqv4tWphA9ZVpXhg0CK67Dt55pzp0b9GwciWcfrrtPF3fdHUss8xsbBu4F6qfB3n9MO9EdaFUO1aYDlyIiIS2evtiyk1xrfS6hO8bSNRt3iuABcH7K7Egqv8CugDDkjUqWior4Ykn4Mgjbcs2XQ47zFakUbZ6Z860ur16pT9OXbRpYwJ1wwYTSjGb08aGO27IG/vsY2YyRbnVe9VVMGcOPPBA7cjoTmZQXQk8D1yPSHtE9gOOBB6rVVfkSES6ICKI7IU5EXohKC3DjiDPQ6Q1IjFziTezfQsxojpt+FiVccH7Rar8QpWOquypyqfZnWIB8p//mM1ZFNvSRHTtaq7IogjTGTPM9V+bNvUbqy5iW73Ll9t+XWPEbU3zRtu2trlSdMJ0wgTzu3vmmdXunpxsUSs0J6rTERmISDg05/GYVckK4FHgZlRHAQTmL0Mwj0Y/YhYnQxKaxdR1TprlM9NgDuwJbA+8rMpKEdoDa1VZn04/jZ5Ro6B7dzj00Pr3cfjhcMklZl7To0fyejNnZmeLN0Z4xfvww/ZrfIstsjdeNigpsW3z9evd/i8PDB4MN95o1l6bJFIbaWysXQunnWZBK25uusYKOcNCng1JkF8jNCeqv6tVp2b9yZiD+rrI2DlpmEjfPMH56AvAXtg+dS9gDhYzdA1m8No0WLzYlIfOPbdhX9wxYfryy6Z2n4gNG0wB6ec/r/84dfHpp9XxGBtrHMaSEpv7ggXVMU6dnDFoENxwA7z7rp1gNGrKy23vet48ePVV86ntFBcZPCcNE/XM9J+YFu+mwKpQ/jNAA5ZnjZAnnzSn8PXd4o2x884W6izVVu9XX9mv5GytTMvL4amnquMxVlba6nThwuyMly3ccUNe+elP7XdlQW31lpeblE/3s3z++fY52nHHIvhl4OSSqEurg4GDVfkhLjrXl5jzhqbDqFHmAaVv34b1I2Kr05EjzZY0kYJDNnzyhhk+3MxzwjTG1WnY1rQ+4e2cBtG+vR29F5QwHT7cVIyvv962apcuNb+HS5fWfB/OW7jQltdgn6WFCxvfkYeTPiJ/AH5HAle5qEYO7hx1ZdqWmramMTbDtnkjIUJXEcaKsFKEuSIkdGgsgohwswhLgnSzSHVsORH6izBRhFXBa/9Q2asiVIRSpUi1kpQIX4uwOlT+etT589lnZs7S0FVpjMMPhzVrzJYtEdk0iwELVF4Z92etrIT33svOeNlim23s1ZWQ8sagQeZMa+XKuutmnW++MdvQqiq45x7bqi0tNXeZBx9sEZ6GDYNLL4V//tN8Zc+YAV9+WR3Lt6qq8SrkOdERuQjzwTsRKAX+jQUP74o5wY9MVGH6NnBq6FpFaI55QPpfGuOFHRqfCNwjktDd0zDsQLofZit0BOa4GBFiDo1HY6Y5o4AXgnwCTeMOsQS8R+2Yq0eE6kTfpn70UXMQcELioAZpc8ABFgc12VbvzJmm6JQtL+KTJ9sWb3yaPDk742WL9u0tQrUL07wxeLDpfxXE77DjjqvecWne3LyOPfigRXgqK4OpU2H+fJP8a9bYlvAbb8CPPzb+Iw8nXYYCw1C9DFiHxTb9NSZg04osElWYXgwMFeH/Aa2DgT4D9gMui9JBoPl7DHCVKhWqpHJofApwqyrzVfk2GO/UoGwwgUNjVdaqktShsQilwEBMjbpBCMBjj9k5SvfuDe3OaN3alItefrn6nzhMtnzyFiNuHpNX9t3X5Fbet3rffBM++KD6esMG+3H4y1/CkCG2hN5tN9Ogb9eueiWa6sjDKWa2Bj4M3q+m2ovSE5i8ikykM1NVPhNhN+AszA1TG2y1d5cq5RHH6g2sVyXeoXEiI64+QVm4Xp9Q2dTAT3CMmEPj1+L6ORkYr8rXcfljRGgGTAYuUq0x1kZEGEbglKJLc4UFC5g+dCiLysoS32E92GKHHdjpuef4+MEHqQj7+FVlv2nT+P7AA/k8g+NlkoqKCsoKZG592rWj3YwZfFQA8ymk55JLevXanRdfrOKQQz5JWJ7t5yLr1vHTY4+lJdVucQCq1q2j/Mwz+fzPf07ado//9//YJMGRx4rXX2dilv+WTfXzUiAsBLoB84C5wE+BT4AdgAQrnBSoar0TaAno0xHrDgRdGJc3FLQsQd0NoDuFrnsFe5ACehXok3H1x4Bem6CfL0BPjcvbD7QtaDvQy0AXgnaua/7bN2um2rmz6urVmlEWLlQVUR0+vHY+qN52W2bHyyDjxo3L9xSqueAC1XbtVKuq8j2TwnouOeSii1RbtVJduTJxedafy9VXJzq0sNS/f3bHbgDF/nkBVmoD5ExWEzygcG3w/kyF1QrjFJYp3J9OXw0NXtmZ6EvhdBwRx9ftCFQEq9FI/YiwP7AFcQa6qryrympVVqlyI+Yto87I3p2rqsynbqY9EXXvbi5k4s9Ns618VGyUlJhW9JIl+Z5Jk2XwYDtqnDAhD4NPnAgjRlg4xETitLHpATjZReSQ4N0w4AYAVO/FjhM/xVzonp1Ol7mMBD0baCFC2MlsMofG04OyRPWmA33D2r2YklJ8P6cAz6tSQWrCzpJTky1VxcMPhw8/hO++q86LmcW4MI1GzFmDn5vmjf33NzfSOT83XbvWNOy32AJuvz3HgzuNlNcRmYPp/Gy+MVf1KVTPQ/VOVNel02HOhKkqGx0ai9BehOQOjU1h6C8i9BBhK+BC4JGgrIzAobEIrUWo5dBYhLbAb0JtYvk9RdhPhFYitBHhImy//N1IN/Hqq9nR7jviCPv1/J//VOfNmGFaqltvnfnxihGPa5p3OnY065OcC9NrroHp080hfX0CTzhNkT6YPDoXmIvIK4gchUjz+naYy5UpJHBorMp0EQaK1FhB3ge8hC23pwGvBHmoktChcZAfY0hQFm/AuQlwD/AD8C1wGPALVaLtDWZLu69vXxOa4a3emTNNk1eiLZqbPC5Mc0MdnoUGDbJt3jWRrc8byPvvw9//DkOHusciJzqqM1D9K6bN+1tsh/Jp4FtEbkYk7XiUKbV5RXixjvZpOa5UJaFDY1VqODQOzkYvDlKiflI6NFblCUxYx+dPx7aE60fM9izTzuBj3pBGj7Ytq9atbWV6wAGZG6PY6drVVvIuTLNLzLNQEi9ZgwfDP/5h1ilZD7ayapVt726zDdx6a5YHc4oS1fXYCvV5RLbCzkz/APwVkXdRjfwlXNfKdEkd6SsyYMPZqMjW6vTww6GiAt5+216/+cbPS9NBxG1Ns80778D995s9ZhKHBgMH2p8iJ1u9l18On39ucymKcDVOXlFdANwN3IHtbO6XTvOUK1NV/lDviRUr2XK3d9BBFhzypZdslQXusCFdSkrc2X2m+f57C4YwerQpycVI4sO5c2dzXZ11YfrWW6ZsdO65cOCBWR7MKXpMu9dioJqL3CeAB9LpItdnpo2Waa1bZ1fNvm1bOOQQOzd1s5j60bOnr0wzwcqV8Pjj5jVoq63gvPMsLxxyMIW7vUGD7Chz7doszW/FCvjDH2CHHSyQquPUB5GeiFyDyFfA68CWmKnMVqj+CYuPGhkXpoXE4Ydb2LXnnzffbGGPSE7dlJSYnWlBeFsvcOIVidavh9dfNzvN7t3hxBNh2jS46CKLeXvAAWb3Emb9+oRHHoMGwerV5vg+K1x0EXz9NTzyiJ2TO066iLyBxeQ+A3gS6I3qgaiORrVe6nMNiG7tZJxf/cpe//1v6NULWrVKWd2JI6zRu8su+Z1LupSXw/HH25ZqLsJ+xRSJzjvPNMmfeMIEa6dOFsjh97+vNhyFxBGG1q1LeOQR05t7660sRMR7/XWLCPPXv8J+aR1pOU6YlcDRwCuobshEh74yLSR69DBDPVVbYXnEivRozOYxYS3ZbFNeblFUqqrgmWfgzjstwvdzz9lnbuTI2ivR+AhDl15qmkaPPFKr+003NV/yGT83/fFH+OMf7fjDHdA7DUH1SFRfzJQgBRemhccRR9jrkiX+hZEujVWYlpebw4EUWrIZ5S9/qV5lNm8OJ51kRwtHHx3dXebFF9sq9oorEhYPGmRxttel5UOmDi64wJ7VqFGZd+vpOA3EhWmhse++1e89nmJ6bLmlKck0No3e4cOrpU5lZXZ/RH3zjW0lx9iwoXqLNx26dIFLLoFXXrEVdRyDBpkZ6McfN3C+MV580VbBl10GAwZkqFPHyRwuTAuNF16Ali3tvcdTTI/mze38rzGtTMvL4aGHqq83bLAt2Gz9iDruuNqxc+v7OTvvPPsBc+mltfoMn5s2mCVLYNgws7m56qoMdOg4mceFaSFRXm6r0fAqxVen6dHYHDcMH25asWHWrjV/s5lm6tSagbNj1Nd2ul07uPpq288N+5UGNt/cdMAaLEzLy+2MdMkS2951pTynQHFhWkgMH27nZmF8dZoejU2Yvv++/Y3jef75zI5TWVlt9rJoUeZClP3xj7D99uaNKO6zO2iQ7QDH/1ZIi1NPtfnuvrutTB2nQHFhWkgkMj/IlselYqWkBBYsyLDmSxYZM8Ze//WvasH25z/D4sW1VnsNYvhwmDLFNHW7dctcvy1bWt9Tp8KTT9YoGjTIPGNOmlTPvt96y0xhwGxdfYfGKWBcmBYS8eYHHtg4fUpKbIU0f36+ZxKNZ54xE5NjjqnOu/FGsy35wx/MnV9D+fBD6/PUU+HXv254f/H89rfV55mhH4MxR/f12uotLzcnJjF8h8YpcFyYOsVFzDymsWj0Pv20aetsuWV1Xps2tmJdtsy2UeMVhtJh9WqLrLLVVnDbbQ2ebkKaNTNhPWeOmfgEbLEF7LhjPYRpRQUceqi9xnD9AafAcWHqFBc9e9prYzg3nT4dPvvMNGzj2W03uPlm89V83331H+PKKy027kMPmV1otjjsMAsZM3x4DXeOgwbB+PGJj4UTsn49/OY35sqwRZyDNl+dOgWMC1OnuGhMwjTRFm+Yc8+1Fdpf/mICMV3efhv++U84+2wLopBNRGx1unAh3HHHxuxBg2D5cvjkkwh9qNpcX33VTJziNZdcf6A4EemKyFhEViIyF5ET6qjfCpEZiMyPy9egj4ogpRX1paG4MHWKizZtTGO1MQjTp582aZPMF2+zZuaooF07czwfr5yWiooKOyPdbjtb4eaC/fYzD1433wxLlwJpnpveeKPFS738cnMu4foDTYW7gEqgO3AicA8ifVLUvwhYlKSsH6odgnR6hueZEhemTvHRGMxjpk+3UHuJtnjDbLmlOXGYNMlsOqMSjqzSoUNDZpoeI0bYUvSWWwBzN73DDhGE6ejR5prwhBPghhuyP0+nMBBpDxwDXIVqBarvAC8CJyWpvy3we6DgYu+5MHWKj8YgTJ9+2laeRx9dd90jj4ShQ01AlZXVXf+//4V774ULL8xC2JY62G03W0Xffjt8+y1QfW4ab0K9kXHj4LTTYPBgO9sVydl0nbzTG1iP6uxQ3hQg2cr0X8DlwOok5W8jshCR5xEpzdw06yanwlSEriKMFWGlCHNFSLg3LoKIcLMIS4J0swgSKu8vwkQRVgWv/UNl14qwToSKUNouSlunSCgpMW3ehmjBZhPVai3eqOHW/vlPC8t30knwww/J6xVCZJXrrquhLNSypU354IMHUVpabVoLmKLRUUfZvY0dC61bbywaMwZKS+03R612KWgs7ZoK3aAFIh+H0rBQcQdgeVyTZcAmtToSOQpojurYJEMNAkqBnYAFwMuI5C7MqKrmLIE+AfoUaAfQ/UGXgfZJUO8M0FmgW4P2AP0M9MygrBXoXNALQFuDnhdctwrKrwUdnWT8lG1TpdatW6tTk3HjxuV7Com54w47YVu4MC/D1/lcPv3U5nf33el1/NFHqi1aqP7mN6pVVYnrnHyyavPmVjefnHOOavPm+sL/zda2bWsefLZtq3rvvapLPv1W1/fYRjd031KXfjJXFy/WjeneezVpu3C9+FQI7dq1Ux09OvqjKtj/owwBrNRk363wE4VVcXkXKrwUl9de4XOFXsH1YIX5KfptrrBSYbekdTIt33I2ENoetBK0dyjvMdCbEtR9D3RY6PqPoBOC94eCfgsqofJ5oIdp3cI0ZdtUyYVpbQr2S+CFF+yj/cEHeRm+zudy1VWqzZrVT9iPGGH3NmpU7bKxY63sqqvS7zfTLFyo2r69vtDu+ARaRKodWK6T6K8raK/9mZSwTmNOJSXRH1XB/h9liDqEaXuFyo1C0vIeVbgprl5/hXUKC4O0VGFD8L40Qb/NFSoU+iYdO8Mpd0vgYG9clfi98UEJ6vYJysL1+oTKptrz2sjUIP+14PoIEZYC5cCdqtyTRtuNiDAMGAbQooVQFuW8qglRUVFRkM+k/fffMwCY/p//sGjVqpyPn/K5qDJg1Cgq+/VjyowZpoSUDnvvTf++felw1ll83LIlawJnDy2XLWPAaaexdocdmDRwIFoAf5dtjzqKX48eTX8u5hN+sjG/Bet4huPoK58y8oj7GFjSgYF8XqPtv/61A5Do7FQ599wvko5ZKO3mzVPKyqJ5qyjU/6OcoLoSkeeB6xE5HegPHAnsG1dzGrBN6Hpf4E5gd2BRoP3bEvgUaAvcAHwLpPkP1gByJbVBB4IujMsbClqWoO4G0J1C172CX3wCehXok3H1x4BeG7zfBXQr0Oag+4KWg/4uKEvZNlXylWltCvYX9Q8/2PLg73/Py/Apn8vUqTa3e+6p/wBff63aqZPqvvuqrltnW77HHqvasqX1Xyj8+KMubdZV/8NhoVVbld7PH+3i/vuTNi0pqd+Kr7G0C1Ow/0cZglQrU1vZdFX4t9q27DyFE4L8gQoVSdrU3OaFgxRmBX18H/TXK+W4GU65VECqADrG5XUEVkSo2xGoCFaUKftR5TNVFqiyQZX3gNuBY+sxB6ex0rkzdOxYmBq96WjxJqOkBO6+2xwYXHaZxTp79lm4/nrTpi0UOnXiq99exi94jSE8TxmDuJFLOZ0H+fTIK+H05GaAI0aYeW2Ydu0sPxWF0K5t27rbOSFUl6I6BNX2qPZE9fEgfzyqie26VMtQ3Tp0/SaqOwZ9bB7093nCttkiV1Kb6jPTXqG8R0l+Zjo0dH0aNc9M51Pz3HMuSc49QS8Bfb4+bcPJV6a1Kehf1LvtpvrrX+dl6KTPpapKdccdVQ86KDMDnXhi9VKoe3dbpRYaq1bpyi499Ltm3XUDogr65f4nJVegCjF6tK3wROw1qlJPPtuB6nHHRWsXo6D/jzIAda1MiyTldjD0SUyjtz3ofiTX5j0TdAamybsV6HRqa/Oej2nknkNNbd4jQbsEW8J7YQpHp0Rpmyq5MK1NQX8J/OpXqv365WXopM9lyhT7l7v33swMNHNm9Td469aq5eWZ6TfT3HJLtdBv1kx17tx8zyhr/PKXqptuqrpiRfQ2Bf1/lAGaijDNtdOGs7HD4e+BJ4CzVJkuwkARQiEiuA94CTtMnga8EuShSiUwBDgZ+BE4DRgS5AMcD3yBbd0+CtysyqiIbZ1ioRAdNzzzjG3xHnVUZvq7/XYz4gQTVYXqBP7LL6vft2iRO/eGeeDqq2HJEtuFd5oWYj8cnLpo06aNrlmzJt/TKCjKysoYPHhwvqeRmFtugUsusTBmHeOPybNLwueiCjvtBNtsA2+80fBBysvN7274M9m2rYVBi+oIIhc0lnlmkMMOM++PX30F7dvXXb+g/48ygIisUtUIT6Jx4+4EneIkFte0UFanU6fC7NkWXiwTDB9e2z9fIYYoayzzzCDXXAOLFplHR6fp4MLUKU4KTZg+8ww0b565Ld73368dRaYQQ5Q1lnlmkJ/+1CLe3XIL5MHM2ckTLkyd4iQmTOfNy+88wLZ4n34aDjwQNtssM31OnpzIvLHwQpSF5lk2blzhzjPDXHMNfP89jByZ75k4ucKFqVOcdO8OrVoVxsp06lT4/PO6w605RcP++9tvp5tvhtXJ4ps4RYULU6c4adbMlH0KQZg+/XRmt3idRsHVV8PChfDAA/meiZMLXJg6xUshmMfEtngPOihzW7xOo2DwYIuyd9NNNZWZneLEhalTvBSCMJ0yBb74wrd4myhXXw0LFljMc6e4cWHqFC8lJWbnuHZt/ubgW7xNmoMOgv32gxtvzO/H0Mk+Lkyd4iWm0Tt/fn7GVzWTmIMOgm7d8jMHJ6+I2Op0/nx45JF8z8bJJi5MneIl37amn3xiW7yZctTgNEp+9jPYZx/4299qm9w6xYMLU6d46dnTXvMlTH2L16F6dTpvHjz6aL5n42QLF6ZO8bLNNvZNlg9hGtviPfhg2HTT3I/vFBSHHQYDBtjqdN26fM/GyQYuTJ3ipVUr2HLL/AjTyZMtWopv8TpUr06/+gpGj873bJxs4MLUKW623BLGjjXr+Vzy9NMWbmzIkNyO6xQsv/oV7L47jBgB69fnezZOpnFh6hQ3y5ZZymWUEt/idRIQW51++SU8/ni+Z+NkGhemTvFSXg5ff23vH3ggZyYyHWbPtnidvsXrxPHrX0O/fnDDDRaJzikeXJg6xUt4NVpZCb16wdlnw/jxtWNsZpDN33rLt3idhMRWp59/Dk8+me/ZOJnEhalTnJSXw8MP1zycWrfO8g44wMxmLrwQPvrItmUzxYIF9Hj2WRg4ELp2zVy/TtEwZAjsuquvTouNnApTEbqKMFaElSLMFeGEJPVEhJtFWBKkm0WQUHl/ESaKsCp47R8qu0iEaSKsEOErES6K6/trEVaLUBGk17N2w07+GD689uqzeXP4/e/twGqPPeBf/4K99rIV65VXwrRp1XXLy2HQoPQVl847j+Zu++CkoFkzW53OnAlbbAEHHTSI0lIYMyZa+zFjoLTU+inUduE20LZttJEaOaqaswT6BOhToB1A9wddBtonQb0zQGeBbg3aA/Qz0DODslagc0EvAG0Nel5w3Soovxh0d9AWoDsGZceH+v4a9JB05966dWt1ajJu3Lh8TyE5/fsnCp1t+TGWLlV98EHVn/1MtVkzK991V9UbblA94QTLO/ts1ZUrVefMUX3/fdV//1v1vvtUr79e9ZxzVI87TvWAA1R33FG1Y8fqcdq0US0vz9/9FyAF/XnJMY89pipS86PZrp3q6NGp240ebfUKuV3tNu1Ucyhn8pVEM7nFlQIR2gM/ALuqMjvIewz4VpVL4+q+Bzyiysjg+o/AUFX2EeFQ4GFga1U0KJ8HDFPltQTj3gGIKucG118Dp6vyRjrzb9Omja7xOEo1KCsrY/DgwfmeRmb47jt49lk7yHrnnWhtOneGzTe3QOTdu9tSY8YM27tr1QpOPx3uuiur025MFNXnpYGUliY2f27RAnr3Tt5u9uzEZjWF1K52m/aorpTEtYuHFjkcqzewPiZIA6YAgxLU7ROUhev1CZVNjQnSgKlBfg1hGmwNDwTui+t/jAjNgMnARao1xgq3HwYMA2jRQigrK0t6c02RioqK4nomffrA8OHs9Le/sfn//kezqiq0WTOW77gj5UccQWXnzqzr2pXKLl2o7NwZbdVqY9NWS5aw9wkn0Dx2CFZZyYYHH+SDgw+m0s9OgSL8vDSAefMGAbXly/r1ymabLUra7rPPNiv4dsnaFD25WgKDDgRdGJc3FLQsQd0NoDuFrnsFWwYCehXok3H1x4Bem6Cf60CngLYO5e0H2ha0HehloAtBO9c1f9/mrU1RbtstWGBbtOF9rbZt696yPess1VatarZr1cq2iR1VLdLPSz0pKan5UYmlkpLG3652m6axzZtLBaQKoGNcXkdgRYS6HYEKtdVopH5EOAc4GfiVKhsjCaryriqrVVmlyo3Aj9jq1XESKy5t2FC304f3368dEqSyEt57L7Pzc4qCESOgXbuaee3aWX5jb5eoTVMgl8J0NtBChF6hvH7A9AR1pwdliepNB/qGtXuBvuF+RDgNuBQ4WJW6LPWVJrkn4SSkvkJx8uSNP8TLxo2r/lE+eXL25uo0Wk48EUaOtCiBIkpJiV2feGI67SjIdvFtMmt7VsDkchkM+mSg0ds+2G5Nps17JuiMQJN3K9DpCbR5zw+0ec+J0+Y9Mdi63TlBvz2DcVuBtgG9CHQR6KZ1zd23eWvj23aJ8eeSGH8uiSn25wKs1FTfr9BVYazCSoW5CifUUb+VwgyF+XH5/RUmKqwKXvun7KcRb/MCnA20Bb4HngDOUmW6CANFqAjVuw94CfgUmAa8EuShSiUwBNvC/RE4DRgS5APcAGwKfBSyJb03KNsEuAfTKv4WOAz4hSpLsnO7juM4Th3cBVQC3YETgXsQ6ZOi/kVATe0nkVbAC8BooAswCnghyM8JudTmRZWlmCCMzx8PdAhdK3BxkBL1MxnYI0nZtinGn45tCTuO4zj5RqQ9cAywK6oVwDuIvAicBDVNJoP62wK/B/4C3B8qGYzJs9uCbcg7EPkrcBDUNpnMBu5O0HEcx8ka3aAFIh+H0rBQcW9gParxJpPJVqb/Ai4HVsfl9wGmBoI0RsxkMifkdGXqOI7jNC0Wm7DcM0lxB2B5XN4y7EiuJiJHAc1RHYvI4AT9LIvUT5ZwYeo4juPki2gmk7YdfAvwywb1k0VcmEZk7dq1KiLxWwtNnRZAAmdjTR5/Lonx55KYYn4uq+oon41tA/dC9fMgL5HJZC+gFBgf2Nu0AjohshDYJ6h/ISIS2urtiyk35YSc+eZt7IjIx5p8q6JJ4s8kMf5cEuPPJTFN/rmIPInZ+58O9Af+A+yL6vRQnRZAt1CrfYE7gd0xzd7mwOfAP4B7gaGY1m8vVOMMx7ODKyA5juM4+aSWySSq0xEZiIiZTKquR3XhxgRLgargekMgMIcQZzKZK0EKvs3rOI7j5BPVhCaTqNYwmYwrKwO2jstLajKZC3xlGp2R+Z5AAeLPJDH+XBLjzyUx/lyKAD8zdRzHcZwG4itTx3Ecx2kgLkwdx3Ecp4G4MHUcx3GcBuLCtA5EpKuIjBWRlSIyV0ROyPecCgERKRORNSJSEaRZ+Z5TrhGRc0TkYxFZKyKPxJUdLCIzRWSViIwTkZI8TTPnJHsuIlIqIhr6zFSIyFV5nGrOEJHWIvJg8B2yQkQ+EZFfhMqb7OelWHBhWje1wgNJ6vBATYlzVLVDkHbM92TywAIs5N9D4UwR6QY8D1wFdAU+Bp7K+ezyR8LnEqJz6HMzPIfzyictgG+AQUAn4Erg6eAHRlP/vBQFbmeaAgmFB9IgPJCkCg/kNClU9XkAEdmTmjZvRwPTVfWZoPxaYLGI7KSqM3M+0RyT4rk0WVR1JXBtKOtlEfkKs4vclCb8eSkWfGWamt7Aeo0eHqipcaOILBaRd6V2FIemTB/scwJs/CL9Ev/cxJgrIvNF5OFgVdbkEJHu2PfLdPzzUhS4ME1N9PBATY9LgO2AHpjR+Usisn1+p1Qw5D0cVIGyGBgAlGArsk2AMXmdUR4QkZbYfY8KVp7+eSkCXJimJu9hfQoVVf1AVVeo6lpVHQW8S/LwSE0N/9wkQFUrVPVjVV2vqt8B5wCHikiTERoi0gx4DNPDOCfI9s9LEeDCNDWzgRYi0iuUlyg8kGNRHyTfkygQpmOfE2Dj2fv2+Ocmnpj7tSbxPSQiAjyIKTMeo6rrgiL/vBQBTeJDXF+Cs4vngetFpL2I7Accif2ybLKISGcR+bmItBGRFiJyInAA8Fq+55ZLgntvg4V/ah57HsBYYFcROSYovxqY2lSUSZI9FxHZW0R2FJFmIrIpcAdQpqrxW5zFyj3AzsARqhqOjdykPy/FggvTuqkVHkjDcfaaJi0x04dF2DnYucCQOEWtpsCVwGpMs/v3wfsrVXURpgU+AvgB2Bs4Pl+TzAMJnwt2xv4atn05DVgL/C5Pc8wpgd3oGVi8zoUhO9sT/fNSHLije8dxHMdpIL4ydRzHcZwG4sLUcRzHcRqIC1PHcRzHaSAuTB3HcRyngbgwdRzHcZwG4sLUcRzHcRqIC1PHaYIEcUWPzfc8HKdYcGHqODlGRB4JhFl8mpDvuTmOUz88nqnj5Ic3sLi4YSrzMRHHcRqOr0wdJz+sVdWFcWkpbNyCPUdEXhGRVSIyV0R+H24sIruJyBsislpElgar3U5xdU4RkU9FZK2IfCcio+Lm0FVEnhGRlSIyJ34Mx3Gi48LUcQqT64AXMV+uI4FHRWRP2BhV5L9Y6K69gKOAfYGHYo1F5AzgPuBhoC8WHm9a3BhXAy9gEUueAh4SkZ5ZuyPHKWLcN6/j5BgReQRzAL8mruguVb1ERBR4QFWHhtq8ASxU1d+LyFDg/4CtVXVFUD4YGAf0UtUvRGQ+MFpVL00yBwVuUtXLgusWwHJgmKqOztzdOk7TwM9MHSc/vA0Mi8v7MfT+/biy94FfBe93xkJ0hYNHvwdUAbuIyHKgB/C/OuYwNfZGVdeLyCJg80izdxynBi5MHSc/rFLVL7LQbzpbTevirhU/+nGceuH/OI5TmOyT4HpG8H4GsJuIbBIq3xf7f56hqt8D3wIHZ32WjuMAvjJ1nHzRWkS2iMvbEASKBjhaRD4CyoBjMcG4d1A2BlNQelRErga6YMpGz4dWuyOAf4rId8ArQDvgYFW9NVs35DhNGRemjpMfDgHK4/K+BbYO3l8LHAPcASwC/qCqHwGo6ioR+TlwG/Ahpsj0AnB+rCNVvUdEKoELgZuBpcB/snQvjtPkcW1exykwAk3b41T12XzPxXGcaPiZqeM4juM0EBemjuM4jtNAfJvXcRzHcRqIr0wdx3Ecp4G4MHUcx3GcBuLC1HEcx3EaiAtTx3Ecx2kgLkwdx3Ecp4H8f3BU203IxFunAAAAAElFTkSuQmCC\n"
     },
     "metadata": {
      "needs_background": "light",
      "image/png": {
       "width": 467,
       "height": 286
      }
     },
     "output_type": "display_data"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "### tf.keras schedulers",
   "metadata": {
    "cell_id": "00141-42e61f2d-ef12-4712-a54e-f88b1015b36e",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 62
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00142-dd3ef043-8507-4c7f-aa14-06a01cb5681f",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "ce5a7315",
    "execution_start": 1659356638501,
    "execution_millis": 155514,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 912
   },
   "source": "model = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.Dense(10, activation=\"softmax\")\n])\ns = 20 * len(X_train) // 32 # number of steps in 20 epochs (batch size = 32)\nlearning_rate = keras.optimizers.schedules.ExponentialDecay(0.01, s, 0.1)\noptimizer = keras.optimizers.SGD(learning_rate)\nmodel.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\nn_epochs = 25\nhistory = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n                    validation_data=(X_valid_scaled, y_valid))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.4893 - accuracy: 0.8275 - val_loss: 0.4092 - val_accuracy: 0.8602\nEpoch 2/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.3819 - accuracy: 0.8649 - val_loss: 0.3739 - val_accuracy: 0.8690\nEpoch 3/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.3487 - accuracy: 0.8766 - val_loss: 0.3736 - val_accuracy: 0.8676\nEpoch 4/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.3265 - accuracy: 0.8834 - val_loss: 0.3493 - val_accuracy: 0.8804\nEpoch 5/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.3104 - accuracy: 0.8895 - val_loss: 0.3430 - val_accuracy: 0.8788\nEpoch 6/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.2958 - accuracy: 0.8950 - val_loss: 0.3412 - val_accuracy: 0.8802\nEpoch 7/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2854 - accuracy: 0.8987 - val_loss: 0.3352 - val_accuracy: 0.8814\nEpoch 8/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2760 - accuracy: 0.9014 - val_loss: 0.3359 - val_accuracy: 0.8818\nEpoch 9/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.2678 - accuracy: 0.9055 - val_loss: 0.3260 - val_accuracy: 0.8854\nEpoch 10/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.2608 - accuracy: 0.9066 - val_loss: 0.3236 - val_accuracy: 0.8852\nEpoch 11/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.2551 - accuracy: 0.9085 - val_loss: 0.3246 - val_accuracy: 0.8856\nEpoch 12/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2497 - accuracy: 0.9126 - val_loss: 0.3296 - val_accuracy: 0.8814\nEpoch 13/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2449 - accuracy: 0.9136 - val_loss: 0.3214 - val_accuracy: 0.8862\nEpoch 14/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2415 - accuracy: 0.9145 - val_loss: 0.3218 - val_accuracy: 0.8862\nEpoch 15/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2375 - accuracy: 0.9167 - val_loss: 0.3203 - val_accuracy: 0.8872\nEpoch 16/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.2343 - accuracy: 0.9179 - val_loss: 0.3179 - val_accuracy: 0.8890\nEpoch 17/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.2317 - accuracy: 0.9184 - val_loss: 0.3193 - val_accuracy: 0.8898\nEpoch 18/25\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.2291 - accuracy: 0.9196 - val_loss: 0.3165 - val_accuracy: 0.8896\nEpoch 19/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2270 - accuracy: 0.9204 - val_loss: 0.3193 - val_accuracy: 0.8888\nEpoch 20/25\n1719/1719 [==============================] - 7s 4ms/step - loss: 0.2250 - accuracy: 0.9218 - val_loss: 0.3165 - val_accuracy: 0.8898\nEpoch 21/25\n1719/1719 [==============================] - 5s 3ms/step - loss: 0.2229 - accuracy: 0.9220 - val_loss: 0.3175 - val_accuracy: 0.8904\nEpoch 22/25\n1719/1719 [==============================] - 5s 3ms/step - loss: 0.2216 - accuracy: 0.9224 - val_loss: 0.3160 - val_accuracy: 0.8918\nEpoch 23/25\n1719/1719 [==============================] - 5s 3ms/step - loss: 0.2201 - accuracy: 0.9232 - val_loss: 0.3168 - val_accuracy: 0.8898\nEpoch 24/25\n1719/1719 [==============================] - 5s 3ms/step - loss: 0.2188 - accuracy: 0.9240 - val_loss: 0.3162 - val_accuracy: 0.8914\nEpoch 25/25\n1719/1719 [==============================] - 5s 3ms/step - loss: 0.2179 - accuracy: 0.9244 - val_loss: 0.3161 - val_accuracy: 0.8910\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "For piecewise constant scheduling, try this:",
   "metadata": {
    "cell_id": "00143-9feca270-e8db-4b1b-8c42-dce7d31ebcdb",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00144-6703d788-e0df-497f-9c7b-6c375133f649",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "bacabd8a",
    "execution_start": 1659356794016,
    "execution_millis": 12,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 112
   },
   "source": "learning_rate = keras.optimizers.schedules.PiecewiseConstantDecay(\n    boundaries=[5. * n_steps_per_epoch, 15. * n_steps_per_epoch],\n    values=[0.01, 0.005, 0.001])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "### 1Cycle scheduling",
   "metadata": {
    "cell_id": "00145-b94bbcfd-fbfe-4648-84fb-514d3bad542b",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 62
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00146-7a3478d6-23ce-4af5-a56b-468eeee9b767",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "e07980b7",
    "execution_start": 1659356794038,
    "execution_millis": 0,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 634
   },
   "source": "K = keras.backend\n\nclass ExponentialLearningRate(keras.callbacks.Callback):\n    def __init__(self, factor):\n        self.factor = factor\n        self.rates = []\n        self.losses = []\n    def on_batch_end(self, batch, logs):\n        self.rates.append(K.get_value(self.model.optimizer.learning_rate))\n        self.losses.append(logs[\"loss\"])\n        K.set_value(self.model.optimizer.learning_rate, self.model.optimizer.learning_rate * self.factor)\n\ndef find_learning_rate(model, X, y, epochs=1, batch_size=32, min_rate=10**-5, max_rate=10):\n    init_weights = model.get_weights()\n    iterations = math.ceil(len(X) / batch_size) * epochs\n    factor = np.exp(np.log(max_rate / min_rate) / iterations)\n    init_lr = K.get_value(model.optimizer.learning_rate)\n    K.set_value(model.optimizer.learning_rate, min_rate)\n    exp_lr = ExponentialLearningRate(factor)\n    history = model.fit(X, y, epochs=epochs, batch_size=batch_size,\n                        callbacks=[exp_lr])\n    K.set_value(model.optimizer.learning_rate, init_lr)\n    model.set_weights(init_weights)\n    return exp_lr.rates, exp_lr.losses\n\ndef plot_lr_vs_loss(rates, losses):\n    plt.plot(rates, losses)\n    plt.gca().set_xscale('log')\n    plt.hlines(min(losses), min(rates), max(rates))\n    plt.axis([min(rates), max(rates), min(losses), (losses[0] + min(losses)) / 2])\n    plt.xlabel(\"Learning rate\")\n    plt.ylabel(\"Loss\")",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "**Warning**: In the `on_batch_end()` method, `logs[\"loss\"]` used to contain the batch loss, but in TensorFlow 2.2.0 it was replaced with the mean loss (since the start of the epoch). This explains why the graph below is much smoother than in the book (if you are using TF 2.2 or above). It also means that there is a lag between the moment the batch loss starts exploding and the moment the explosion becomes clear in the graph. So you should choose a slightly smaller learning rate than you would have chosen with the \"noisy\" graph. Alternatively, you can tweak the `ExponentialLearningRate` callback above so it computes the batch loss (based on the current mean loss and the previous mean loss):\n\n```python\nclass ExponentialLearningRate(keras.callbacks.Callback):\n    def __init__(self, factor):\n        self.factor = factor\n        self.rates = []\n        self.losses = []\n    def on_epoch_begin(self, epoch, logs=None):\n        self.prev_loss = 0\n    def on_batch_end(self, batch, logs=None):\n        batch_loss = logs[\"loss\"] * (batch + 1) - self.prev_loss * batch\n        self.prev_loss = logs[\"loss\"]\n        self.rates.append(K.get_value(self.model.optimizer.learning_rate))\n        self.losses.append(batch_loss)\n        K.set_value(self.model.optimizer.learning_rate, self.model.optimizer.learning_rate * self.factor)\n```",
   "metadata": {
    "cell_id": "00147-56f70cea-3495-4c64-b0bc-2db157e12a59",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 467.03125
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00148-f29b1e88-d4a7-4ea7-bb39-4f2098f9c97a",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "f1d9958b",
    "execution_start": 1659356794054,
    "execution_millis": 39,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 274
   },
   "source": "tf.random.set_seed(42)\nnp.random.seed(42)\n\nmodel = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.Dense(10, activation=\"softmax\")\n])\nmodel.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n              metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00149-7bfc4cb4-ef02-43c4-90ab-51965530d1bc",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "72fda6d5",
    "execution_start": 1659356794093,
    "execution_millis": 3587,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 449.1875,
    "deepnote_output_heights": [
     null,
     275
    ]
   },
   "source": "batch_size = 128\nrates, losses = find_learning_rate(model, X_train_scaled, y_train, epochs=1, batch_size=batch_size)\nplot_lr_vs_loss(rates, losses)",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "430/430 [==============================] - 2s 5ms/step - loss: nan - accuracy: 0.3855\n",
     "output_type": "stream"
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAERCAYAAACO6FuTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmC0lEQVR4nO3deXyc1X3v8c9Pki1ZluVNsiQsywvGeAW8YJywJZDg2oS0xCxlSygUkjTc5LbNTWhCGichaUNu09ery01CC2EtYQ1JzGIIEPbNNrbBgMEG75u827KtbX73jxmBENo153lGo+/79ZoXszwz8zsW9lfnnOecx9wdERGR1uTEXYCIiGQuhYSIiLRJISEiIm1SSIiISJsUEiIi0iaFhIiItCkv7gLSqaSkxMeMGRN3GZJl9h2uZ8PuQ4wrGcjA/Kz6K5MV3t1+kP55OYweXhh3Kb3W0qVLd7p7aWuvZdX/8WPGjGHJkiVxlyFZpqa2gek/epyLThrNP54zOe5ypIW5//oMY0sG8svLZsZdSq9lZuvbek3DTSIdGJifx8lHD+fxt7ahxaeZx9HPJCSFhEgnfHZyORt3H2b19gNxlyKtMIu7guylkBDphM9MGgHA46u2x1yJtKTOXVgKCZFOGFFcwAmjhvD4WwqJTKSeRDgKCZFOOmtKGSs37WPz3sNxlyLNqCMRlkJCpJPOnlYBwEMrt8RcibRkqCsRSmQhYWb5ZnaTma03swNmttzM5rVx7FQzW2xmO81MvyhIRhg9fCDHVQ5m0cqtcZcizeiMs7Ci7EnkARuB04HBwHXAPWY2ppVj64F7gCsjq06kEz53XAUrN+1j/a6auEuR5tSRCCaykHD3Gndf6O7r3D3h7ouA94GPrYBx99XufhOwKqr6RDrj7OOOAlBvIoOoHxFWbHMSZlYGTEBBIL3IyCEDOGHUEB59Y1vcpUgz6kiEE0tImFk/4E7gVnd/u4efdbWZLTGzJdXV1ekpUKQdc6eU8/pmneWUMdSVCCrykDCzHOB2oA64pqef5+43uvssd59VWtrq/lQiaTV3ShkAj61SbyJTmBZKBBNpSFjyJ3kTUAYscPf6KL9fJB3GlRZxzIgiHtPq64ygjkRYUfckfgFMAs5x9zb76pZUAPRPPS4ws/yIahTp0Nwp5byybjd7auriLkXQnERIUa6TGA18GTgB2GZmB1O3S8ysKnW/KnX4aOAwH05qHwZWR1WrSEfmTimnMeH8Udt0xE7rJMKK7HoS7r6e9gO/qNmx6zo4ViRWU0cWc9TgAhav2s75s0bFXU6fpymJcLQth0g3mBlnTSnn2XerOVTXEHc5fZr6EWEpJES66awpZdQ2JHh6tU69jps6EuEoJES6afaYYQwt7Mdjb2peIk6akghLISHSTXm5OZw5qYwn3tpOfWMi7nL6NK2TCEchIdIDc6eUs/9IAy+9tyvuUvosXeM6LIWESA+cekwJA/rlslirr2OlfkQ4CgmRHijol8vpE0p5bNV2Egn9RhsHzUmEpZAQ6aG5U8vYcaCW5Zv2xl1K36WuRDAKCZEeOuPYMvJyTHs5SVZSSIj00ODCfnzi6OE8tmqbtoiIgf7Iw1JIiKTBWVPKeW9nDWt2HIy7lD7JNN4UjEJCJA3Ompy8xoTOcpJso5AQSYOy4gKmVw1hseYlYqG1dOEoJETS5KzJuqxpHDQPFJZCQiRNdFnT+KgjEY5CQiRNxpUWMaGsiEffUEhESf2IsBQSImk0b2oFr6zbzY4DR+IupU/RnEQ4CgmRNDr7uArcYbF6E5HRlERYCgmRNJpQNojxI4p4+HWFRJS0TiIchYRIms2fWs7L7+9i58HauEvpE7RVeFgKCZE0m39cBQlHE9gR0pxEOAoJkTQ7tmwQ40oH8vDrW+MupU/QnERYCgmRNDMzzp5WwUvv7WKXhpwioZ5EOAoJkQDmTU0OOWmbjvDUkQhLISESwKSKQYwt0ZBTdNSVCEUhIRKAmTF/WjkvvreL3TV1cZeT1TQnEZZCQiSQeVMraEy49nKKgOYkwlFIiAQy5ahiRg8v5CENOQWmrkRICgmRQJJDThW8sHYXezTkFJQ6EuEoJEQCOntaasjpTQ05haI5ibAUEiIBTTmqmFHDBmgvp8A0JxFOZCFhZvlmdpOZrTezA2a23MzmtXP835rZNjPbb2Y3m1l+VLWKpEvTkNPza3ay95CGnEJQRyKsKHsSecBG4HRgMHAdcI+ZjWl5oJnNBa4FzgRGA+OAH0RWqUganT2tgoaE89ibWlgXinaBDSeykHD3Gndf6O7r3D3h7ouA94GZrRz+JeAmd1/l7nuAHwGXR1WrSDpNGzmYyqEDtLBOeqXY5iTMrAyYAKxq5eUpwIpmj1cAZWY2PIraRNKp+ZDTvkP1cZeTdVwz10HFEhJm1g+4E7jV3d9u5ZAiYF+zx033B7XyWVeb2RIzW1JdXZ3+YkXSYP60Cuobncff0pBTCJq4DifykDCzHOB2oA64po3DDgLFzR433T/Q8kB3v9HdZ7n7rNLS0rTWKpIux1cOZuQQDTmFoH5EWJGGhJkZcBNQBixw97b63quA45s9Ph7Y7u67ApcoEoSZMW9qOc++W82+wxpySjd1JMKJuifxC2AScI67H27nuNuAK81sspkNIXkm1C3hyxMJZ/5xySGnJzTklFaakggrynUSo4EvAycA28zsYOp2iZlVpe5XAbj7o8ANwFPABmA98P2oahUJYfqoIRw1uEBDTgGYJiWCyYvqi9x9Pe33CotaHP9z4OdBixKJkJkxb1oFt7+4nv1H6iku6Bd3SVlBZzeFpW05RCI0f1o5dY0JDTlJr6GQEInQ9FFDKS8u0F5OaaR+RFgKCZEI5eQY86aV8/Q71Rw4orOc0kVTEuEoJEQiNn9aBXUNCZ58e0fcpWQHdSWCUkiIRGxm1VDKivN5aKXOckoXbfAXjkJCJGI5OcbZ047iT6urtZdTGqgjEZZCQiQG504fSV1jQte/ThPNSYSjkBCJwdSRxYwfUcSDr22Ou5ReT+skwlJIiMTAzDh3+kheWbebjbsPxV1Or6eORDgKCZGYfP74owD4/YotMVfSu6kfEZZCQiQmo4YVMnvMMB5YtklDJj2kOYlwFBIiMTp3xkjWVtfwxub9cZfSaylfw1JIiMRo/tQK+ufm8FtNYPeIdoENRyEhEqPBhf04Y+II/rByC40J/UrcHa5ZiaAUEiIxm39cBdUHanltw564S+m11I8IRyEhErMzJo6gf24Oj7yhnWG7Q3MSYSkkRGJWlJ/HqceU8Ogb23SWU3epKxGMQkIkA8ydWs7mvYdZtUVnOUlmUUiIZIDPTiojN8f4w0otrOsq9b3CUkiIZIChA/vzqQmlPPjaZp3l1A3aKjwchYRIhlgws5Lt+2t5fs3OuEvpXZSpQSkkRDLEmZNGMHhAP+5ftinuUnodraULRyEhkiHy83I55/gKFq/aputfd4EW04WlkBDJIAtmVHKkPsHDuhhRl6gjEY5CQiSDnDBqCONKB3L/Uu3l1FlaWhKWQkIkg5gZC2ZU8sq63azfVRN3Ob2G5iTCUUiIZJgvzBiJGdy/VBPYnaGORFgKCZEMUzF4AKeML+H+ZZtJaM1Ep2idRDgKCZEMdN7MSjbvPczL7++Ou5SMp/2uwlJIiGSgsyaXU5Sfx30acuoUzUmEo5AQyUAD+ufyueMqeOSNrdTUNsRdTkZTPyKsSEPCzK4xsyVmVmtmt7RzXL6Z/auZbTGzPWb2/8ysX4SlisRuwcxKDtU16joTnaCORDhR9yS2ANcDN3dw3LXALGAqMAGYAVwXtjSRzDJr9FBGDy/kvqUb4y4lo2lKIqxIQ8LdH3D3B4FdHRx6DvBv7r7b3auBfwOuCF2fSCYxM86bUclL7+1m4+5DcZeT2TQpEUwmz0lYi/uVZjY4rmJE4nDujJEAPLBMK7AlHpkaEo8C3zCzUjMrB76eer6w5YFmdnVqnmNJdXV1pEWKhFY5tJBPHj2c+5dt0qme7VA/IpxMDYkfA68By4EXgAeBemB7ywPd/UZ3n+Xus0pLS6OsUSQSC2ZUsmH3IV5dtyfuUjKOgjO8HodEiLOO3P2wu1/j7iPdfRzJOYyl7p5I93eJZLp508oZ2D9XE9jt0JREOF0KCTP7upktaPb4JuCwma02s2M78f48MysAcoFcMysws7xWjhtpZkdZ0hzge8D3u1KrSLYo7J/H/GkVPPz6Ng7Vac1Ec+pIhNfVnsTXgWoAMzsNuAC4mOSw0L904v3XAYdJnuJ6aer+dWZWZWYHzawqddzRJIeZaoBbgWvd/bEu1iqSNRbMrORgbQOLV2nNRGu0d1M4H/stvgMjgfdT988B7nX3e8zsdeDZjt7s7guBhW28XNTsuGeAMV2sTSRrzR4zjFHDBnD/0s2cO70y7nIyhjoS4XW1J7EfGJG6/1ngidT9eqAgXUWJyEfl5BhfmF7J82t3smXv4bjLyRhNE9c56kgE09WQeAz4LzP7b2A88Ejq+Sl82MMQkQAWzKjEHX77mtZMNGlsCgmlRDBdDYmvAc8DpcB57t60j/EM4K50FiYiH1U1vJDZY4dx31KtmWiSSJ3vmKPTm4Lp0pyEu+8H/lcrz+vMI5EInDezkm/dt5JlG/Ywc/SwuMuJXVNPIjdTV3xlga6eAju5+amuZvZZM7vDzP7BzHLTX56INDd/WgUD+uVy31INOQEkPpiTUE8ilK7m783AdAAzGwX8DhhGchjq+vSWJiItFeXnMW9qOYtWbOFIfWPc5cSu6fKuColwuhoSE4FlqfvnAS+7+3zgMuCidBYmIq07b2YlB2obeOzNj+1S0+c0JpqGmxQSoXQ1JHKButT9M4GHU/fXAmXpKkpE2jZn3HBGDhmgS5sCqYzQ2U0BdTUk3gC+amankgyJR1PPjwR2prMwEWldTo7xhRkjee7darbu69trJhJaJxFcV0Pi28BVwJ+Au9z99dTznwdeSWNdItKO82eOIuFw96t9e9O/D4abNCcRTJdCIrVdRilQ4u7NrxT3K+Cr6SxMRNpWNbyQ0yaUcverG2lo7LubIzeFhIabwuny2cXu3khy59epZjbFzArcfZ277whQn4i04eLZVWzdd4SnVvfdi201rSlUTyKcrq6TyDOznwF7gBXA68AeM7shxHUlRKRtZ04aQVlxPne+vD7uUmLz4bYcMReSxbr6R3sDyS2+vwJMAI4hOcx0GfBP6S1NRNrTLzeHC2eN4ul3qtm4+1Dc5cSiUeskgutqSFwMXOnut7r72tTtFuCvgUvSXp2ItOvC2VUY8JtXN8RdSizctU4itK6GxGCSayJaWgsM6XE1ItIlI4cM4IyJI7j71U3U98EJ7EZtyxFcV0NiBcmr07X0jdRrIhKxi0+qYufBWh5b1fdWYGu4KbyuXpnuW8DDZvYZ4KXUc3OAo4B56SxMRDrn9AkjGDlkAHe8tJ6zj6uIu5xINW0VruGmcLqzTmICcB/Jy40WAfcCc2m9hyEigeXmGJfMqeLF93axZseBuMuJlFZch9eddRJb3P277r4gdbsOqAEWpL88EemMC2aNon9uDne81LcmsHVluvB0drFIFigpymf+tHLuX7qJmtqGuMuJTELbcgSnkBDJEpfOGc2B2gZ+t3xL3KVEpmkXWM1JhKOQEMkSM0cPZWL5oD61ZqLp7CZ1JMLp1NlNZvb7Dg4pTkMtItIDZsYFs0bxw0Vv8tbW/UyqyP6/lk0T1xpuCqezPYldHdzeB24LUaCIdN6500fSPzeHe5b0jS3EdWW68DrVk3D3vwpdiIj03NCB/fnslDJ++9pmrp03kfy83LhLCqqpJ2HqSQSjOQmRLHPhrFHsPVTPI69vi7uU4BLauyk4hYRIljllfAnjSgZyywvr4i4luKbtqjQnEY5CQiTL5OQYX/zEaJZv3MuKjXvjLieohK4nEZz+aEWy0IKZlQzsn8utWd6bSGiDv+AUEiJZaFBBPxbMrGTRyq3sOlgbdznBNGpOIrhIQ8LMrjGzJWZWa2a3tHOcmdn1ZrbZzPaZ2Z/MbEqEpYr0epfNGU1dY4J7lmyKu5RgmlZcqycRTtQ9iS3A9cDNHRx3PnAFcCowDHgRuD1saSLZ5ZiyQcwZN4w7Xlr/wXqCbPPhcFPMhWSxSEPC3R9w9wdJLsBrz1jgOXd/z90bgTuAyaHrE8k2X/zEGDbvPcxTb++Iu5QgtJguvEydk/gNcLSZTTCzfsCXgEdbO9DMrk4NYS2prq6OtEiRTPfZyWWUFedz+0vr4y4lCF2+NLxMDYmtwHPAauAwyeGnv23tQHe/0d1nufus0tLSCEsUyXz9cnO4aHYVT79TzbqdNXGXk3auievgMjUk/hE4ERgFFAA/AJ40s8JYqxLphS6eXUVejnFHFvYmmhbTqScRTqaGxAnA3e6+yd0b3P0WYCialxDpshHFBcydWs49SzZm3QWJGrWYLrioT4HNM7MCIBfINbMCM2ttk8FXgfPNrMzMcszsMqAfsCbKekWyxRUnj2H/kQbuX5Zdp8O6tgoPLur8vY7kHMO1wKWp+9eZWZWZHTSzqtRxPwVWAMuBvSTnIxa4+96I6xXJCjOqhnL8qCH8+vl1H5w2mg0ateI6uKhPgV3o7tbittDdN7h7kbtvSB13xN2/5u4V7l7s7jPcvdWzm0SkY2bGlaeM5f2dNTyRRafDfhASmrgORiN5In3EvKnlVA4dwH8+teaDYZreTluFh6eQEOkj+uXm8NVPHc3yjXt5+f3dcZeTFh9uyxFvHdlMISHShyyYUcngAf2yZnGd5iTCU0iI9CEF/XK5YFYli9/Yxo79R+Iup8cS2pYjOIWESB9zyUmjaUg4d72yMe5SeqxpuEmnwIajkBDpY8aUDOS0CaX8zyvrqW9astxLNS2mU0aEo5AQ6YMumzOa7ftreeKt7XGX0iOJhJNjyVN8JQyFhEgfdMbEEYwcMqDXT2A3ums+IjCFhEgflJtjXHxSFc+v2cWaHQfjLqfbEu7qRQSmkBDpoy48cRT983K46bn34y6l2xIJ16R1YAoJkT6qpCif82ZWcv+yTew40DtPh21M6PTX0BQSIn3YVaeOo74xwa0vrIu7lG5JuGu1dWAKCZE+bGzJQOZOLuf2F9f3ymtNJNy1uV9gCgmRPu7Lp49j/5EGfvNq71tc16g5ieAUEiJ93PSqocweO4ybnn2v1y2uS7i2CQ9NISEifPm0cWzZd4SHVm6Nu5QuaVpMJ+EoJESETx87gmNGFPHLp9f2qmtNNLqGm0JTSIgIOTnG1aeN4+1tB3jm3Z1xl9NpiYQmrkNTSIgIAH9+wkjKivP51dNr4y6l05KnwCokQlJIiAgA/fNyuOLksbywdhevb9oXdzmd0uhaTBeaQkJEPnDRSVUMys/jV8/0jt6EJq7DU0iIyAeKC/px8ZwqHn59Kxt2HYq7nA4ltAtscAoJEfmIK04eS26OcdNz78VdSocaE5qTCE0hISIfUVZcwF+cMJK7l2xkd01d3OW0SxPX4SkkRORjrj5tHEfqE9z24rq4S2lXY0LDTaEpJETkY44pG8RnJo3g1hfWcbiuMe5y2pRwNHEdmEJCRFp11anj2HOont++tjnuUtqkXWDDU0iISKtmjx3GlKOK+fXz72fsVh3aBTY8hYSItMrMuOLksby74yDPrcnMrTrUkwhPISEibfrc8RWUFOVzc4ZeBzuR0JxEaAoJEWlTfl4ul80ZzVOrq1lbfTDucj6mUYvpgos0JMzsGjNbYma1ZnZLO8f90swONrvVmtmBCEsVkZRL5lSRn5fDvz/xbtylfIwW04UXdU9iC3A9cHN7B7n7V9y9qOkG3AXcG0WBIvJRJUX5XHnKWB5cvoXlG/fGXc5HaDFdeJGGhLs/4O4PArs6+x4zGwgsAG4NVZeItO9vPj2ekqJ8frTozYw60+ngkQYGFeTFXUZW6w1zEguAauCZ1l40s6tTQ1hLqquro61MpI8oys/jm2dNYOn6PSzKoEuc7j1cz5DCfnGXkdV6Q0h8CbjN2/j1xd1vdPdZ7j6rtLQ04tJE+o7zZ41iUkUx//zI2xypj38VdiLh7D1Ux5AB/eMuJatldEiYWRXwKeC2mEsR6fNyc4zvnT2JzXsP89/Pxr9D7IHaBhKOehKBZXRIAJcBz7t7/P9HigifHF/CvKnl/PuTa1i3sybWWvYdqgdgSKF6EiFFfQpsnpkVALlArpkVmFl7s05fBG6JpDgR6ZSFn59C/9wcvvPb12OdxN5zKLmN+ZAB6kmEFHVP4jrgMHAtcGnq/nVmVpVaD1HVdKCZfQKoRKe+imSUsuICvjVvIi+s3cXiVdtiq2Pv4WRPYuhAhURIUZ8Cu9DdrcVtobtvSK2J2NDs2BfdfaC7axGdSIa56MRRjB9RxM8Wr6ahMRFLDXtTPYnBmrgOKtPnJEQkA+Xl5vDNs45lbXUNDyyLZyvxvak5iaGauA5KISEi3TJ3ShnHjxrCv/7xnVhOiW0KicGakwhKISEi3WJmfPvPjmXrviP8x5NrIv/+vYfrKMrPIy9X/4yFpD9dEem2Tx5dwnkzK/mPp9awaOWWSL+7plZbckRBISEiPfLjc6cya/RQ/v6eFayIcAPAmrpGCvvnRvZ9fZVCQkR6JD8vl19dNpPSQfl88eZXeGFtNFexO1TbwMB89SRCU0iISI8NL8rnrqvmMGJQPl+7cxk79h8J/p3qSURDISEiaTFqWCG/uHQmh+sb+eZ9K0kkwq7GPlTXwMD+6kmEppAQkbQZP6KI686ezDPvVPOTh98K+l2Hahsp1HBTcAoJEUmrS06q4kufGM1/P/c+T7y1Pdj31NQ1MFDDTcEpJEQkrcyM75w9iYnlg/jqncu49YV1QTYCPFTbSKGGm4JTSIhI2uXn5XLnX5/EyUcP5/u/X8VtL65P6+e7e7Inka+eRGgKCREJYnhRPjdffiKfOraUf3rkLTbtOZS2z65tSJBw1JOIgEJCRIIxM3587jQA/vmRt9P2uTW1DQDqSURAISEiQY0cMoCrTzuaRSu38uq63Wn5zEN1yQ0F1ZMITyEhIsF95fRxlBcX8MM/vJmW9RM1damehM5uCk4hISLBFfbP49p5E3l98z5+8+rGHn9eTW2qJ6F1EsEpJEQkEp8//ijmjBvGTx5+i817D/fosw6pJxEZhYSIRCInx/jZeceTcOfa+1f2aO3EBz0JzUkEp5AQkciMGlbIt/9sIs++u5PFq7Z1+3P2H0lelU7XkwhPISEikbrkpCqOLRvE9Q+91e3Lnu48WAtASVF+OkuTVigkRCRSebk5/OM5k9m05zDXP9S9s52qD9QyKD+PAZqTCE4hISKRO3l8CVedOpY7XtrA392znLqGRJfeX32gltJB6kVEQQN6IhKL78yfxJDC/vxs8Wpq6hq58bKZmFmn3lt9oJYShUQk1JMQkViYGV/79Hi+O38Sj7+5nXuXbOr0e6sP1lKq+YhIKCREJFZXnjKWOeOGcd2Db/DCms5dH1vDTdFRSIhIrHJyjF9eOpMxJYVcddsSbn9pfbtrKI7UN3LgSINCIiIKCRGJ3ZDC/tx+5UlMqxzM9x58g7+5cxkHUzu9trR13xEAhUREFBIikhHKigu466o5fGf+RBav2sZlN73calAs37gHgGkjB0ddYp+kkBCRjGFmXH3a0fzi0pms2LiXhb9f9bFjXl23h0H5eUwoGxRDhX1PpCFhZteY2RIzqzWzWzo4dpyZLTKzA2a208xuiKhMEYnZ3Cnl/M2nxnPf0k28/N6uj7y2dN0epo8eSm5O506XlZ6JuiexBbgeuLm9g8ysP/A48CRQDlQCdwSvTkQyxjVnjKesOJ8bFq/+YCJ736F6Vm8/wImjh8ZcXd8RaUi4+wPu/iCwq4NDLwe2uPvP3b3G3Y+4+8rgBYpIxijol8vXzzyGpev38OTbOwBYtiE5HzFzjEIiKpk6JzEHWGdmj6SGmv5kZtPiLkpEonXBrFGMGV7I93+/in2H61myfjd5OcYJo4bEXVqfkanbclQCnwY+DzwBfAP4nZlNdPe65gea2dXA1QBFFUdz4a9ejLpWEQloYH4eq7bsZ/aP/0hdQ4Kigjz+6tevxl1Wn2E9ufBHt7/U7Hqg0t0vb+P13wHF7v7p1GMD9gKnufuKdj73ALC6h+UNBvb18LjWXuvMc80ft3a/BOjcktS2qX0dH9ed9nWmrdnYvubPq30di6p9Xf27N9rdS1v9NneP/EZy8vqWdl7/EfBks8eWasjxHXzukjTUdmNPj2vttc481/xxa/fVvsxtX2famo3ta3GM2pch7evq3732blGfAptnZgVALpBrZgVm1tqQ1x3AHDP7jJnlAv+bZIK/FUGZf0jDca291pnn/tCJ+z2l9nV8XHfa19m29lSmtS+dbevK56l97T+Xtr97kQ43mdlC4Pstnv4ByVNi3wQmu/uG1LFfAG4ARgDLgK+5+8dX1nz085e4+6x0150p1L7eTe3r3bK9fW2JdOLa3RcCC9t4uajFsQ8AD3TxK27selW9itrXu6l9vVu2t69VsUxci4hI75Cp6yRERCQDKCRERKRNfS4kzGyMmVWnVnH/ycxaPze4lzOzi8ysOu460s3MyszsBTN72syeNLOKuGtKJzObbWYvmtkzZnaXmfWLu6Z0MrPBZvaKmR00s6lx15MOZvZTM3vWzG7Ptp8X9MGQSHna3T+VumXjP6S5wPnAxrhrCWAncIq7nw7cBlwZcz3pthE4w91PA9YBfx5vOWl3CDgbuC/uQtLBzI4HRrr7qcDbwHkxl5R2fTUkTk4l/09Sq7mzzUXAvUAi7kLSzd0b3b2pXYOAdk+L7m3cfau7H049rCPLfobuXp9lv5h9Engsdf9R4OQYawkio0OivetPmNkwM/utmdWY2Xozu7iTH7sVGA+cRnINxhfSW3XnhWhfqhdxAXB3gJK7JNDPDzM7wcxeBq4huYYmFqHal3r/aOAs0r+Yqys1BGtfpulBW4cC+1P39wHDIio5Mpm6wV+TputPzAUGtHjtP0n+plUGnAA8ZGYr3H2VmZUDv2nl8/7S3bcBtQBm9gDJHWfvD1N+h9LevtRn3ePuiQzoJAX5+bn7cuAkM7sA+AfgK4Hq70iQ9plZMXA7cLm71wervmOh/v5lom61leSecsWp4wYDu6MoNlI93Yskihst9noCBpL8oU1o9tztwD934rMGNbv/T8AXs6x9PyXZ/X2U5G82/5Zl7evf7P5c4OdZ1r484GHgzLjbFaJ9zY6/BZgad9t62laSoXFb6v53gIvibkO6bxk93NSOCUCDu7/T7LkVwJROvPcUM1tqZs8CI4H/CVFgD3W7fe7+bXc/y93/DHjX3b8eqsge6MnP74TUmT9PkdzT62cB6uupnrTvIuAk4Hups+8uDFFgD/WkfZjZwySH0v7LzC5Pf3lp1W5bPdmr3Z7692QK8Y1KBJPpw01tKeLDccAm+0hOZLbL3R8BHglRVBp1u33NeebuM9OTn98rJOeTMllP2nc7yd9UM1mP/v909/lpryicDtvq7v8n0ooi1lt7Egf5cBywSTFwIIZaQlD7eje1L3v0pba2qreGxDtAnpkd0+y548me0yHVvt5N7csefamtrcrokLA2rj/h7jUkd4j9oZkNNLOTSS46yvRu+keofWpfJsv29jXXl9raZXHPnHdwpsFCwFvcFqZeGwY8CNQAG4CL465X7VP71L7eeetLbe3qTVuFi4hImzJ6uElEROKlkBARkTYpJEREpE0KCRERaZNCQkRE2qSQEBGRNikkRESkTQoJkTQys4Vm9kbcdYikixbTSa+TunJYibt/Lu5aWjKzIiDf3XfFXUtbzMyB8909K64zLWGpJyHSCWbWvzPHufvBOALCzHJSl64VSSuFhGQdM5tsZg+Z2QEz22Fmd6Uuqdn0+olm9piZ7TSz/Wb2nJl9osVnuJl9zcweMLMa4CdNQ0lm9pdmtjb1+Q+aWUmz931kuMnMbjGzRWb2DTPbbGZ7zOzXZlbY7JiBZnabmR00s+1m9g+p99zSThsvTx0/P/V9dcCkjtpmZutSd+9NtXFds9fOSV2Q64iZvW9mP+5sOEr2UkhIVjGzCuAZ4A1gNvAZkheO+Z2ZNf3/PojkLp6npo5ZDjxsZsNbfNz3SV5KdBrJ6xwDjAEuBM4leXW16cCPOyjrVGBqqpam936j2ev/Apyeev4MkltRn9qJ5hYA3wO+DEwG1neibSem/nsVUNH02MzmAncC/0HyCmtXAOcBP+lEHZLN4t5hUDfdunojeX3kRW289kPgiRbPDSW5q+fsNt5jwFbg0mbPOfDvLY5bCBwBBjd77rvAmhbHvNGi1o1AbrPn/gv4Y+p+EclewF82e30gsIdm11pupebLUzXO7ODPqq22ndfiuGeA77V47i9IXnTH4v6Z6xbfTT0JyTYzgdNSQzEHzewgyX+kAY4GMLMRZvYrM3vHzPaRvMrYCKCqxWctaeXz17v7vmaPt6Te25433b2xjfccDfQDXml60ZPXMOjMGVINJHsKH+hC21qaCXy3xZ/b/5AMrPL23yrZrLde41qkLTnAQ8A3W3lte+q/twJlwN8C64Ba4Amg5fh7TSufUd/isdPxsG133tMZtS3CBzrftpZygB8A97byWnXPypTeTCEh2WYZcAHJ3/hb/uPc5BTg6+7+EICZlZEcn4/DWpIhciLwXqqeQpJzGGu78XmdaVs9ySuwNbcMmOjua7rxnZLFFBLSWxWb2QktnttLcoL5KuBuM/spyd+Cx5EMjr939wMkr1t8qZm9THI45QaS8wKRc/eDZnYz8FMz20ly/uA6kr/Zd2cRU2fatg4408yeJtkb2UNyLmeRma0H7iE5lDWV5DzOt7pRh2QJzUlIb3Uq8FqL2/919y3AyUACeJTkBev/k+SwS23qvVeQnDBeCvwGuJnkP5xx+SbwLPB74ClgJcn5kCPd+KzOtO3vgU+TnKt5DcDdFwNnp55/JXW7luTlOqUP04prkQxjZvkkT2f9mbv/S9z1SN+m4SaRmJnZdGASyd/eBwHfTv337jjrEgGFhEim+DvgWD48rfU0d98Ua0UiaLhJRETaoYlrERFpk0JCRETapJAQEZE2KSRERKRNCgkREWmTQkJERNr0/wFkEWqf32ZqNQAAAABJRU5ErkJggg==\n"
     },
     "metadata": {
      "needs_background": "light",
      "image/png": {
       "width": 393,
       "height": 273
      }
     },
     "output_type": "display_data"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00150-4090f3cf-cc4d-4aa3-86ec-8e4e3d074312",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "51219e8f",
    "execution_start": 1659356797682,
    "execution_millis": 1,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 490
   },
   "source": "class OneCycleScheduler(keras.callbacks.Callback):\n    def __init__(self, iterations, max_rate, start_rate=None,\n                 last_iterations=None, last_rate=None):\n        self.iterations = iterations\n        self.max_rate = max_rate\n        self.start_rate = start_rate or max_rate / 10\n        self.last_iterations = last_iterations or iterations // 10 + 1\n        self.half_iteration = (iterations - self.last_iterations) // 2\n        self.last_rate = last_rate or self.start_rate / 1000\n        self.iteration = 0\n    def _interpolate(self, iter1, iter2, rate1, rate2):\n        return ((rate2 - rate1) * (self.iteration - iter1)\n                / (iter2 - iter1) + rate1)\n    def on_batch_begin(self, batch, logs):\n        if self.iteration < self.half_iteration:\n            rate = self._interpolate(0, self.half_iteration, self.start_rate, self.max_rate)\n        elif self.iteration < 2 * self.half_iteration:\n            rate = self._interpolate(self.half_iteration, 2 * self.half_iteration,\n                                     self.max_rate, self.start_rate)\n        else:\n            rate = self._interpolate(2 * self.half_iteration, self.iterations,\n                                     self.start_rate, self.last_rate)\n        self.iteration += 1\n        K.set_value(self.model.optimizer.learning_rate, rate)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00151-b35a486f-8a63-4602-9b68-62f96bf01512",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "29b99e27",
    "execution_start": 1659356797684,
    "execution_millis": 50616,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 768
   },
   "source": "n_epochs = 25\nonecycle = OneCycleScheduler(math.ceil(len(X_train) / batch_size) * n_epochs, max_rate=0.05)\nhistory = model.fit(X_train_scaled, y_train, epochs=n_epochs, batch_size=batch_size,\n                    validation_data=(X_valid_scaled, y_valid),\n                    callbacks=[onecycle])",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.6572 - accuracy: 0.7740 - val_loss: 0.4872 - val_accuracy: 0.8336\nEpoch 2/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.4581 - accuracy: 0.8396 - val_loss: 0.4275 - val_accuracy: 0.8524\nEpoch 3/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.4122 - accuracy: 0.8546 - val_loss: 0.4115 - val_accuracy: 0.8584\nEpoch 4/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.3837 - accuracy: 0.8641 - val_loss: 0.3868 - val_accuracy: 0.8688\nEpoch 5/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.3639 - accuracy: 0.8716 - val_loss: 0.3766 - val_accuracy: 0.8680\nEpoch 6/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.3457 - accuracy: 0.8774 - val_loss: 0.3742 - val_accuracy: 0.8710\nEpoch 7/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.3330 - accuracy: 0.8807 - val_loss: 0.3635 - val_accuracy: 0.8706\nEpoch 8/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.3185 - accuracy: 0.8858 - val_loss: 0.3947 - val_accuracy: 0.8614\nEpoch 9/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.3064 - accuracy: 0.8889 - val_loss: 0.3485 - val_accuracy: 0.8762\nEpoch 10/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.2945 - accuracy: 0.8924 - val_loss: 0.3399 - val_accuracy: 0.8808\nEpoch 11/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.2837 - accuracy: 0.8959 - val_loss: 0.3460 - val_accuracy: 0.8816\nEpoch 12/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.2709 - accuracy: 0.9024 - val_loss: 0.3650 - val_accuracy: 0.8690\nEpoch 13/25\n430/430 [==============================] - 2s 4ms/step - loss: 0.2538 - accuracy: 0.9083 - val_loss: 0.3360 - val_accuracy: 0.8846\nEpoch 14/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.2406 - accuracy: 0.9135 - val_loss: 0.3473 - val_accuracy: 0.8802\nEpoch 15/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.2281 - accuracy: 0.9183 - val_loss: 0.3262 - val_accuracy: 0.8850\nEpoch 16/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.2160 - accuracy: 0.9233 - val_loss: 0.3298 - val_accuracy: 0.8840\nEpoch 17/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.2064 - accuracy: 0.9262 - val_loss: 0.3364 - val_accuracy: 0.8876\nEpoch 18/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.1979 - accuracy: 0.9306 - val_loss: 0.3252 - val_accuracy: 0.8896\nEpoch 19/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.1893 - accuracy: 0.9341 - val_loss: 0.3243 - val_accuracy: 0.8906\nEpoch 20/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.1822 - accuracy: 0.9368 - val_loss: 0.3237 - val_accuracy: 0.8932\nEpoch 21/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.1753 - accuracy: 0.9402 - val_loss: 0.3234 - val_accuracy: 0.8906\nEpoch 22/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.1701 - accuracy: 0.9420 - val_loss: 0.3196 - val_accuracy: 0.8948\nEpoch 23/25\n430/430 [==============================] - 2s 4ms/step - loss: 0.1655 - accuracy: 0.9443 - val_loss: 0.3199 - val_accuracy: 0.8944\nEpoch 24/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.1626 - accuracy: 0.9458 - val_loss: 0.3190 - val_accuracy: 0.8928\nEpoch 25/25\n430/430 [==============================] - 2s 5ms/step - loss: 0.1609 - accuracy: 0.9465 - val_loss: 0.3183 - val_accuracy: 0.8940\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "# Avoiding Overfitting Through Regularization",
   "metadata": {
    "cell_id": "00152-c4da1b57-aacb-4117-a179-ee14451b6b2d",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 82
   }
  },
  {
   "cell_type": "markdown",
   "source": "## $\\ell_1$ and $\\ell_2$ regularization",
   "metadata": {
    "cell_id": "00153-ab62d008-1ed8-4fa6-9630-1ad3722fcdea",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 73.546875
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00154-62129818-e037-4bae-95f1-fd1c49df51a3",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "6380fdfc",
    "execution_start": 1659356848299,
    "execution_millis": 0,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 148
   },
   "source": "layer = keras.layers.Dense(100, activation=\"elu\",\n                           kernel_initializer=\"he_normal\",\n                           kernel_regularizer=keras.regularizers.l2(0.01))\n# or l1(0.1) for ℓ1 regularization with a factor of 0.1\n# or l1_l2(0.1, 0.01) for both ℓ1 and ℓ2 regularization, with factors 0.1 and 0.01 respectively",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00155-54685fbf-5fe1-45e5-899e-d16bc96666ec",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "479d3a8c",
    "execution_start": 1659356848300,
    "execution_millis": 19043,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 419.75
   },
   "source": "model = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    keras.layers.Dense(300, activation=\"elu\",\n                       kernel_initializer=\"he_normal\",\n                       kernel_regularizer=keras.regularizers.l2(0.01)),\n    keras.layers.Dense(100, activation=\"elu\",\n                       kernel_initializer=\"he_normal\",\n                       kernel_regularizer=keras.regularizers.l2(0.01)),\n    keras.layers.Dense(10, activation=\"softmax\",\n                       kernel_regularizer=keras.regularizers.l2(0.01))\n])\nmodel.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\nn_epochs = 2\nhistory = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n                    validation_data=(X_valid_scaled, y_valid))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/2\n1719/1719 [==============================] - 10s 5ms/step - loss: 1.5956 - accuracy: 0.8124 - val_loss: 0.7169 - val_accuracy: 0.8340\nEpoch 2/2\n1719/1719 [==============================] - 9s 5ms/step - loss: 0.7197 - accuracy: 0.8274 - val_loss: 0.6850 - val_accuracy: 0.8376\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00156-8bc54353-9873-4f5e-adc6-ecb29bcaccea",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "63aa3099",
    "execution_start": 1659356867303,
    "execution_millis": 19100,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 455.75
   },
   "source": "from functools import partial\n\nRegularizedDense = partial(keras.layers.Dense,\n                           activation=\"elu\",\n                           kernel_initializer=\"he_normal\",\n                           kernel_regularizer=keras.regularizers.l2(0.01))\n\nmodel = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    RegularizedDense(300),\n    RegularizedDense(100),\n    RegularizedDense(10, activation=\"softmax\")\n])\nmodel.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\nn_epochs = 2\nhistory = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n                    validation_data=(X_valid_scaled, y_valid))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/2\n1719/1719 [==============================] - 10s 5ms/step - loss: 1.6313 - accuracy: 0.8113 - val_loss: 0.7218 - val_accuracy: 0.8310\nEpoch 2/2\n1719/1719 [==============================] - 9s 5ms/step - loss: 0.7187 - accuracy: 0.8273 - val_loss: 0.6826 - val_accuracy: 0.8382\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "## Dropout",
   "metadata": {
    "cell_id": "00157-88106a58-7377-4c3b-b4cb-1f7f7ed037fc",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00158-6a186161-8d75-4450-b1cc-ca924c06d65e",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "dd15e6fd",
    "execution_start": 1659356886392,
    "execution_millis": 41836,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 383.75
   },
   "source": "model = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    keras.layers.Dropout(rate=0.2),\n    keras.layers.Dense(300, activation=\"elu\", kernel_initializer=\"he_normal\"),\n    keras.layers.Dropout(rate=0.2),\n    keras.layers.Dense(100, activation=\"elu\", kernel_initializer=\"he_normal\"),\n    keras.layers.Dropout(rate=0.2),\n    keras.layers.Dense(10, activation=\"softmax\")\n])\nmodel.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\nn_epochs = 2\nhistory = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n                    validation_data=(X_valid_scaled, y_valid))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/2\n1719/1719 [==============================] - 14s 8ms/step - loss: 0.5838 - accuracy: 0.7997 - val_loss: 0.3730 - val_accuracy: 0.8644\nEpoch 2/2\n1719/1719 [==============================] - 14s 8ms/step - loss: 0.4209 - accuracy: 0.8442 - val_loss: 0.3397 - val_accuracy: 0.8716\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "## Alpha Dropout",
   "metadata": {
    "cell_id": "00159-43a7a2dc-6476-474f-bcd9-1f3e47b94390",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00160-37f6a791-465c-4616-8775-6f7a112e4f70",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d690901b",
    "execution_start": 1659356928241,
    "execution_millis": 0,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "source": "tf.random.set_seed(42)\nnp.random.seed(42)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00161-e6036fc9-de95-4658-9b7d-7e0e33d329f8",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "46b19aa3",
    "execution_start": 1659356928247,
    "execution_millis": 142617,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 930
   },
   "source": "model = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    keras.layers.AlphaDropout(rate=0.2),\n    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.AlphaDropout(rate=0.2),\n    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n    keras.layers.AlphaDropout(rate=0.2),\n    keras.layers.Dense(10, activation=\"softmax\")\n])\noptimizer = keras.optimizers.SGD(learning_rate=0.01, momentum=0.9, nesterov=True)\nmodel.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\nn_epochs = 20\nhistory = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n                    validation_data=(X_valid_scaled, y_valid))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/20\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.6617 - accuracy: 0.7611 - val_loss: 0.5777 - val_accuracy: 0.8410\nEpoch 2/20\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.5548 - accuracy: 0.7966 - val_loss: 0.5403 - val_accuracy: 0.8480\nEpoch 3/20\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.5259 - accuracy: 0.8063 - val_loss: 0.4997 - val_accuracy: 0.8552\nEpoch 4/20\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.5050 - accuracy: 0.8126 - val_loss: 0.4763 - val_accuracy: 0.8622\nEpoch 5/20\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.4925 - accuracy: 0.8171 - val_loss: 0.4592 - val_accuracy: 0.8636\nEpoch 6/20\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.4837 - accuracy: 0.8198 - val_loss: 0.4856 - val_accuracy: 0.8566\nEpoch 7/20\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.4709 - accuracy: 0.8250 - val_loss: 0.5037 - val_accuracy: 0.8492\nEpoch 8/20\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.4640 - accuracy: 0.8281 - val_loss: 0.4556 - val_accuracy: 0.8634\nEpoch 9/20\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.4605 - accuracy: 0.8302 - val_loss: 0.4234 - val_accuracy: 0.8750\nEpoch 10/20\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.4524 - accuracy: 0.8317 - val_loss: 0.4398 - val_accuracy: 0.8620\nEpoch 11/20\n1719/1719 [==============================] - 6s 4ms/step - loss: 0.4480 - accuracy: 0.8338 - val_loss: 0.3997 - val_accuracy: 0.8746\nEpoch 12/20\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.4432 - accuracy: 0.8343 - val_loss: 0.5301 - val_accuracy: 0.8542\nEpoch 13/20\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.4425 - accuracy: 0.8361 - val_loss: 0.4214 - val_accuracy: 0.8776\nEpoch 14/20\n1719/1719 [==============================] - 9s 5ms/step - loss: 0.4375 - accuracy: 0.8383 - val_loss: 0.4407 - val_accuracy: 0.8642\nEpoch 15/20\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.4316 - accuracy: 0.8393 - val_loss: 0.4450 - val_accuracy: 0.8660\nEpoch 16/20\n1719/1719 [==============================] - 13s 8ms/step - loss: 0.4301 - accuracy: 0.8409 - val_loss: 0.4142 - val_accuracy: 0.8766\nEpoch 17/20\n1719/1719 [==============================] - 10s 6ms/step - loss: 0.4288 - accuracy: 0.8418 - val_loss: 0.5106 - val_accuracy: 0.8608\nEpoch 18/20\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.4274 - accuracy: 0.8395 - val_loss: 0.4619 - val_accuracy: 0.8760\nEpoch 19/20\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.4240 - accuracy: 0.8435 - val_loss: 0.4752 - val_accuracy: 0.8698\nEpoch 20/20\n1719/1719 [==============================] - 6s 3ms/step - loss: 0.4177 - accuracy: 0.8433 - val_loss: 0.4095 - val_accuracy: 0.8762\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00162-502d1a3c-576c-4a7d-8d63-c8025c049533",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "9afd4cd5",
    "execution_start": 1659357070869,
    "execution_millis": 712,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 159.375,
    "deepnote_output_heights": [
     null,
     21.1875
    ]
   },
   "source": "model.evaluate(X_test_scaled, y_test)",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "313/313 [==============================] - 1s 2ms/step - loss: 0.4627 - accuracy: 0.8604\n",
     "output_type": "stream"
    },
    {
     "output_type": "execute_result",
     "execution_count": 107,
     "data": {
      "text/plain": "[0.4626573622226715, 0.8604000210762024]"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00163-a0cd0869-d245-4af5-afa5-2e057f32cffa",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d676405a",
    "execution_start": 1659357071585,
    "execution_millis": 5374,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 159.375,
    "deepnote_output_heights": [
     null,
     21.1875
    ]
   },
   "source": "model.evaluate(X_train_scaled, y_train)",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "1719/1719 [==============================] - 3s 2ms/step - loss: 0.3372 - accuracy: 0.8837\n",
     "output_type": "stream"
    },
    {
     "output_type": "execute_result",
     "execution_count": 108,
     "data": {
      "text/plain": "[0.3371737599372864, 0.8836727142333984]"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00164-a961cd87-9189-4488-945a-a8ebe91a6794",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "3a185b5b",
    "execution_start": 1659357076961,
    "execution_millis": 10467,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 107.1875
   },
   "source": "history = model.fit(X_train_scaled, y_train)",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "1719/1719 [==============================] - 6s 3ms/step - loss: 0.4201 - accuracy: 0.8433\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "## MC Dropout",
   "metadata": {
    "cell_id": "00165-10eed768-aa9c-467d-a923-936f3679e003",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00166-583c1d5f-e913-49b3-8df8-394933c9c794",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d690901b",
    "execution_start": 1659357087436,
    "execution_millis": 5,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "source": "tf.random.set_seed(42)\nnp.random.seed(42)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00167-d4a36eef-8217-4db3-9a65-5cfe66b78bc8",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "9d2b5e9",
    "execution_start": 1659358010747,
    "execution_millis": 331,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 196.1875
   },
   "source": "y_probas = np.stack([model(X_test_scaled, training=True)\n                     for sample in range(100)])\ny_proba = y_probas.mean(axis=0)\ny_std = y_probas.std(axis=0)",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "error",
     "ename": "NameError",
     "evalue": "name 'np' is not defined",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-2d810fa5cda9>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m y_probas = np.stack([model(X_test_scaled, training=True)\n\u001b[0m\u001b[1;32m      2\u001b[0m                      for sample in range(100)])\n\u001b[1;32m      3\u001b[0m \u001b[0my_proba\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my_probas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0my_std\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my_probas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'np' is not defined"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00168-3cd96ed8-f6b1-4523-9a77-4251b2368c1a",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "1b4c7fe3",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 132.375,
    "deepnote_output_heights": [
     40.375
    ]
   },
   "source": "np.round(model.predict(X_test_scaled[:1]), 2)",
   "execution_count": null,
   "outputs": [
    {
     "data": {
      "text/plain": "array([[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.96]],\n      dtype=float32)"
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00169-bebf9d01-8ff4-4bc4-9a15-2ddc28674c2c",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "6a473bbe",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 699,
    "deepnote_output_heights": [
     607
    ]
   },
   "source": "np.round(y_probas[:, :1], 2)",
   "execution_count": null,
   "outputs": [
    {
     "data": {
      "text/plain": "array([[[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.57, 0.  , 0.42]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.93, 0.  , 0.05]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.  , 0.  , 0.99]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.08, 0.  , 0.92]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.46, 0.  , 0.53]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.75, 0.  , 0.24]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.43, 0.  , 0.51]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.13, 0.  , 0.28, 0.  , 0.59]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.12, 0.  , 0.06, 0.  , 0.81]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.02, 0.  , 0.98]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.07, 0.  , 0.92]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.08, 0.  , 0.28, 0.  , 0.64]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.29, 0.  , 0.7 ]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.26, 0.  , 0.22, 0.  , 0.52]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.25, 0.  , 0.72]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.3 , 0.  , 0.06, 0.  , 0.65]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.  , 0.  , 0.96]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.81, 0.  , 0.17]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.25, 0.  , 0.75]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.03, 0.  , 0.93]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.84, 0.04, 0.01, 0.  , 0.1 ]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.2 , 0.  , 0.1 , 0.  , 0.7 ]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 1.  ]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.28, 0.  , 0.72]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.12, 0.  , 0.12, 0.  , 0.76]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.75, 0.  , 0.21]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.11, 0.  , 0.87]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.28, 0.  , 0.39, 0.  , 0.34]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.04, 0.  , 0.95]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.18, 0.  , 0.51, 0.01, 0.3 ]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.01, 0.  , 0.99]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 1.  ]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.59, 0.  , 0.01, 0.  , 0.39]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.12, 0.  , 0.8 , 0.  , 0.08]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.27, 0.  , 0.67]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.07, 0.  , 0.77, 0.  , 0.16]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.99]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.18, 0.  , 0.81]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.32, 0.  , 0.64]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.75, 0.  , 0.04, 0.  , 0.21]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.24, 0.  , 0.75]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.96]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.44, 0.  , 0.38, 0.  , 0.17]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.98]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.02, 0.  , 0.96]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.99]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.12, 0.  , 0.82]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.97]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.28, 0.  , 0.71]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.17, 0.  , 0.82]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.23, 0.  , 0.27, 0.  , 0.5 ]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.78, 0.  , 0.16]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.99]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.03, 0.  , 0.94]],\n\n       [[0.  , 0.  , 0.01, 0.  , 0.02, 0.21, 0.01, 0.01, 0.  , 0.75]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.06, 0.  , 0.93]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.4 , 0.  , 0.6 ]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.12, 0.  , 0.18, 0.01, 0.69]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.15, 0.  , 0.83]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.24, 0.  , 0.24, 0.  , 0.52]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.11, 0.  , 0.01, 0.  , 0.88]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.2 , 0.  , 0.8 ]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.04, 0.  , 0.91]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.21, 0.  , 0.77]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.26, 0.  , 0.67, 0.  , 0.07]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.27, 0.  , 0.55, 0.  , 0.19]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.09, 0.  , 0.47, 0.  , 0.43]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.99]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.07, 0.  , 0.71, 0.  , 0.22]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.1 , 0.  , 0.63, 0.01, 0.26]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 1.  ]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.16, 0.  , 0.81]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.15, 0.  , 0.8 ]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.24, 0.  , 0.75]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.13, 0.  , 0.81]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.42, 0.  , 0.01, 0.  , 0.58]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.15, 0.  , 0.83]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.79, 0.  , 0.19]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.21, 0.  , 0.79]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.93]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.12, 0.  , 0.87]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.06, 0.  , 0.9 ]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 1.  ]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.01, 0.  , 0.98]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.08, 0.  , 0.87]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.51, 0.  , 0.46]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.96, 0.  , 0.02, 0.  , 0.02]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.38, 0.  , 0.05, 0.  , 0.57]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.07, 0.  , 0.92]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.98]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.06, 0.  , 0.89]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.05, 0.  , 0.94]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.37, 0.  , 0.37, 0.  , 0.26]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.37, 0.  , 0.19, 0.  , 0.44]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.11, 0.  , 0.87]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.01, 0.  , 0.98]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.41, 0.  , 0.59]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.12, 0.  , 0.87]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.08, 0.  , 0.21, 0.  , 0.71]],\n\n       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.12, 0.  , 0.13, 0.  , 0.75]]],\n      dtype=float32)"
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00170-985a535c-36a0-4022-8bfe-efbe1d140b98",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "56e67949",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 132.375,
    "deepnote_output_heights": [
     40.375
    ]
   },
   "source": "np.round(y_proba[:1], 2)",
   "execution_count": null,
   "outputs": [
    {
     "data": {
      "text/plain": "array([[0.  , 0.  , 0.  , 0.  , 0.  , 0.1 , 0.  , 0.22, 0.  , 0.68]],\n      dtype=float32)"
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00171-ee229e78-1664-4a33-80e4-6bb2bf35e713",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "4d303287",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 150.375,
    "deepnote_output_heights": [
     40.375
    ]
   },
   "source": "y_std = y_probas.std(axis=0)\nnp.round(y_std[:1], 2)",
   "execution_count": null,
   "outputs": [
    {
     "data": {
      "text/plain": "array([[0.  , 0.  , 0.  , 0.  , 0.  , 0.18, 0.  , 0.24, 0.  , 0.29]],\n      dtype=float32)"
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00172-1507067d-4851-42d9-8207-a6a660102d18",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "4fdfb0f1",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "y_pred = np.argmax(y_proba, axis=1)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00173-81719507-49c5-47d6-a211-1cb1741ba8f3",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "185a70f3",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 131.1875,
    "deepnote_output_heights": [
     21.1875
    ]
   },
   "source": "accuracy = np.sum(y_pred == y_test) / len(y_test)\naccuracy",
   "execution_count": null,
   "outputs": [
    {
     "data": {
      "text/plain": "0.8661"
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00174-e581c382-52cf-455a-931c-152194380c6d",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "686a1ad7",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 184
   },
   "source": "class MCDropout(keras.layers.Dropout):\n    def call(self, inputs):\n        return super().call(inputs, training=True)\n\nclass MCAlphaDropout(keras.layers.AlphaDropout):\n    def call(self, inputs):\n        return super().call(inputs, training=True)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00175-6a6b3aa9-4606-49fe-a74d-09f474068af5",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "d690901b",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "source": "tf.random.set_seed(42)\nnp.random.seed(42)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00176-c5c05750-81ca-4560-a0fd-3a934b4b5f2f",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "e9f125ad",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 130
   },
   "source": "mc_model = keras.models.Sequential([\n    MCAlphaDropout(layer.rate) if isinstance(layer, keras.layers.AlphaDropout) else layer\n    for layer in model.layers\n])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00177-81d612b2-290f-4c48-af04-c76c08247b45",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "20fccbf1",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 531.125
   },
   "source": "mc_model.summary()",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "Model: \"sequential_20\"\n_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\nflatten_18 (Flatten)         (None, 784)               0         \n_________________________________________________________________\nmc_alpha_dropout (MCAlphaDro (None, 784)               0         \n_________________________________________________________________\ndense_262 (Dense)            (None, 300)               235500    \n_________________________________________________________________\nmc_alpha_dropout_1 (MCAlphaD (None, 300)               0         \n_________________________________________________________________\ndense_263 (Dense)            (None, 100)               30100     \n_________________________________________________________________\nmc_alpha_dropout_2 (MCAlphaD (None, 100)               0         \n_________________________________________________________________\ndense_264 (Dense)            (None, 10)                1010      \n=================================================================\nTotal params: 266,610\nTrainable params: 266,610\nNon-trainable params: 0\n_________________________________________________________________\n"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00178-1973d9cb-dabe-419b-9491-10830d094ae1",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "4cd2bce0",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "source": "optimizer = keras.optimizers.SGD(learning_rate=0.01, momentum=0.9, nesterov=True)\nmc_model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00179-62844632-0491-4dcb-a7dc-d26f634b2991",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "fc49a37a",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "mc_model.set_weights(model.get_weights())",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "Now we can use the model with MC Dropout:",
   "metadata": {
    "cell_id": "00180-8eace055-5445-4990-9a88-15d103459d3b",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00181-9bc33c82-a42d-4a88-bb79-e15f118d93cc",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "d8139e66",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 132.375,
    "deepnote_output_heights": [
     40.375
    ]
   },
   "source": "np.round(np.mean([mc_model.predict(X_test_scaled[:1]) for sample in range(100)], axis=0), 2)",
   "execution_count": null,
   "outputs": [
    {
     "data": {
      "text/plain": "array([[0.  , 0.  , 0.  , 0.  , 0.  , 0.14, 0.  , 0.25, 0.01, 0.61]],\n      dtype=float32)"
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "## Max norm",
   "metadata": {
    "cell_id": "00182-a77434bd-5a1c-44f9-af9d-36f52c4324b5",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00183-d7b3da83-74c8-4224-9e75-afa63cc751dd",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "3d0f9eee",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 94
   },
   "source": "layer = keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n                           kernel_constraint=keras.constraints.max_norm(1.))",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00184-6e9b021d-fafd-4ec5-9ef5-9df9446c75bd",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "e12f41d3",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 401.75
   },
   "source": "MaxNormDense = partial(keras.layers.Dense,\n                       activation=\"selu\", kernel_initializer=\"lecun_normal\",\n                       kernel_constraint=keras.constraints.max_norm(1.))\n\nmodel = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[28, 28]),\n    MaxNormDense(300),\n    MaxNormDense(100),\n    keras.layers.Dense(10, activation=\"softmax\")\n])\nmodel.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\nn_epochs = 2\nhistory = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n                    validation_data=(X_valid_scaled, y_valid))",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "Epoch 1/2\n1719/1719 [==============================] - 5s 3ms/step - loss: 0.5763 - accuracy: 0.8020 - val_loss: 0.3674 - val_accuracy: 0.8674\nEpoch 2/2\n1719/1719 [==============================] - 5s 3ms/step - loss: 0.3545 - accuracy: 0.8709 - val_loss: 0.3714 - val_accuracy: 0.8662\n"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "# Exercises",
   "metadata": {
    "cell_id": "00185-965cded5-3017-4695-bbb4-e52c75c2455e",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 82
   }
  },
  {
   "cell_type": "markdown",
   "source": "## 1. to 7.",
   "metadata": {
    "cell_id": "00186-cd5bae67-3ad7-4d15-85f7-2433b37667e4",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "markdown",
   "source": "See appendix A.",
   "metadata": {
    "cell_id": "00187-7d2b9354-d430-46a3-8051-d47f2ba4f757",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "markdown",
   "source": "1. Is it OK to initialize all the weights to the same value as long as that value is selected randomly using He initialization?\n",
   "metadata": {
    "cell_id": "87ed37a8194e4a4ba32eb052c23420e9",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 69.390625
   }
  },
  {
   "cell_type": "markdown",
   "source": "No, all weights should be sampled independently; they should not all have the same initial value. One important goal of sampling weights randomly is to break symmetry: if all the weights have the same initial value, even if that value is not zero, then symmetry is not broken (i.e., all neurons in a given layer are equivalent), and backpropagation will be unable to break it. Concretely, this means that all the neurons in any given layer will always have the same weights. It’s like having just one neuron per layer, and much slower. It is virtually impossible for such a configuration to converge to a good solution.",
   "metadata": {
    "cell_id": "2e03a500bb63436983e2afcb1c3504e5",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 119.5625
   }
  },
  {
   "cell_type": "markdown",
   "source": "2. Is it OK to initialize the bias terms to 0?",
   "metadata": {
    "cell_id": "ab94fc7469d143bca92188bb93efeb4b",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 69.390625
   }
  },
  {
   "cell_type": "markdown",
   "source": "It is perfectly fine to initialize the bias terms to zero. Some people like to initialize them just like weights, and that’s okay too; it does not make much difference.",
   "metadata": {
    "cell_id": "b1298fec2cd64bd4aae6060a350e566a",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "markdown",
   "source": "3. Name three advantages of the SELU activation function over ReLU.\n",
   "metadata": {
    "cell_id": "3621e6c384ba4bbca9d39b786f357e6f",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 69.390625
   }
  },
  {
   "cell_type": "markdown",
   "source": "A few advantages of the SELU function over the ReLU function are:\n- It can take on negative values, so the average output of the neurons in any given layer is typically closer to zero than when using the ReLU activation function (which never outputs negative values). This helps alleviate the vanishing gradients problem.\n- It always has a nonzero derivative, which avoids the dying units issue that can affect ReLU units. \n- When the conditions are right (i.e., if the model is sequential, and the weights are initialized using LeCun initialization, and the inputs are standardized, and there’s no incompatible layer or regularization, such as dropout or $l_1$ regularization), then the SELU activation function ensures the model is self-normalized, which solves the exploding/vanishing gradients problems.\n",
   "metadata": {
    "cell_id": "4ac94b81da324b7aabb7c65d3981fd92",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 393.734375
   }
  },
  {
   "cell_type": "markdown",
   "source": "4. In which cases would you want to use each of the following activation functions: SELU, leaky ReLU (and its variants), ReLU, tanh, logistic, and softmax?",
   "metadata": {
    "cell_id": "e5beebb6724d4179afc953b99ca0a7d8",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 69.390625
   }
  },
  {
   "cell_type": "markdown",
   "source": "The SELU activation function is a good default. If you need the neural network to be as fast as possible, you can use one of the leaky ReLU variants instead (e.g., a simple leaky ReLU using the default hyperparameter value). The simplicity of the ReLU activation function makes it many people’s preferred option, despite the fact that it is generally outperformed by SELU and leaky ReLU. However, the ReLU activation function’s ability to output precisely zero can be useful in some cases (e.g., see Chapter 17). Moreover, it can sometimes benefit from  optimized implementation as well as from hardware acceleration. The hyperbolic tangent (tanh) can be useful in the output layer if you need to output a number between –1 and 1, but nowadays it is not used much in hidden layers (except in recurrent nets). The logistic activation function is also useful in the output layer when you need to estimate a probability (e.g., for binary classification), but is rarely used in hidden layers (there are exceptions—for example, for the coding layer of variational autoencoders; see Chapter 17). Finally, the softmax activation function is useful in the output layer to output probabilities for mutually exclusive classes, but it is rarely (if ever) used in hidden layers.\n",
   "metadata": {
    "cell_id": "3b551d83455b4e9da60fa43bcec2e8bd",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 209.125
   }
  },
  {
   "cell_type": "markdown",
   "source": "5. What may happen if you set the `momentum` hyperparameter too close to 1 (e.g., 0.99999) when using an SGD optimizer?",
   "metadata": {
    "cell_id": "2f98b33f57394297be5ec6cea0360452",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 69.390625
   }
  },
  {
   "cell_type": "markdown",
   "source": "If you set the momentum hyperparameter too close to 1 (e.g., 0.99999) when using an SGD optimizer, then the algorithm will likely pick up a lot of speed, hopefully moving roughly toward the global minimum, but its momentum will carry it right past the minimum. Then it will slow down and come back, accelerate again, overshoot again, and so on. It may oscillate this way many times before converging, so overall it will take much longer to converge than with a smaller momentum value.",
   "metadata": {
    "cell_id": "08136a575171441cab9da7c1d900d5f6",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 119.5625
   }
  },
  {
   "cell_type": "markdown",
   "source": "6. Name three ways you can produce a sparse model.\n",
   "metadata": {
    "cell_id": "ce06e2f9302a4573895256d6fbf9fa5e",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 69.390625
   }
  },
  {
   "cell_type": "markdown",
   "source": "One way to produce a sparse model (i.e., with most weights equal to zero) is to train the model normally, then zero out tiny weights. For more sparsity, you can apply $l_1$ regularization during training, which pushes the optimizer toward sparsity. A third option is to use the TensorFlow Model Optimization Toolkit.",
   "metadata": {
    "cell_id": "0ecacedab38d486e82658bb38f723bbc",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 74.78125
   }
  },
  {
   "cell_type": "markdown",
   "source": "7. Does dropout slow down training? Does it slow down inference (i.e., making predictions on new instances)? What about MC Dropout?\n",
   "metadata": {
    "cell_id": "5168360ff4fe49448958c29e5afcfbcc",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 69.390625
   }
  },
  {
   "cell_type": "markdown",
   "source": "Yes, dropout does slow down training, in general roughly by a factor of two. However, it has no impact on inference speed since it is only turned on during training. MC Dropout is exactly like dropout during training, but it is still active during inference, so each inference is slowed down slightly. More importantly, when using MC Dropout you generally want to run inference 10 times or more to get better predictions. This means that making predictions is slowed down by a factor of 10 or more.\n",
   "metadata": {
    "cell_id": "823f98dc13444227b3369aed85034032",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 119.5625
   }
  },
  {
   "cell_type": "markdown",
   "source": "8. Practice training a deep neural network on the CIFAR10 image dataset:\n    - Build a DNN with 20 hidden layers of 100 neurons each (that’s too many, but it’s the point of this exercise). Use He initialization and the ELU activation function.\n    - Using Nadam optimization and early stopping, train the network on the CIFAR10 dataset. You can load it with `keras.datasets.cifar10.load_ data()`. The dataset is composed of 60,000 32 × 32–pixel color images (50,000 for training, 10,000 for testing) with 10 classes, so you’ll need a softmax output layer with 10 neurons. Remember to search for the right learning rate each time you change the model’s architecture or hyperparameters.\n    - Now try adding Batch Normalization and compare the learning curves: Is it converging faster than before? Does it produce a better model? How does it affect training speed?\n    - Try replacing Batch Normalization with SELU, and  make the necessary adjustements to ensure the network self normalizes (i.e., standardize the input features, use LeCun normal initialization, make sure the DNN contains only a sequence of dense layers, etc.).\n    - Try regularizing the model with alpha dropout. Then, without retraining your model, see if you can achieve better accuracy using MC Dropout. \n    - Retrain your model using 1cycle scheduling and see if it improves training speed and model accuracy.",
   "metadata": {
    "cell_id": "b810e630b10d470e9942d9c0ab015868",
    "tags": [],
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 333.6875
   }
  },
  {
   "cell_type": "markdown",
   "source": "## 8. Deep Learning on CIFAR10",
   "metadata": {
    "cell_id": "00188-9fd6280c-5624-4885-83a5-57078672d100",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 70
   }
  },
  {
   "cell_type": "markdown",
   "source": "### a.\n*Exercise: Build a DNN with 20 hidden layers of 100 neurons each (that's too many, but it's the point of this exercise). Use He initialization and the ELU activation function.*",
   "metadata": {
    "cell_id": "00189-9c10ac35-8e49-4923-b7e9-760bfe9e39da",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 122.78125
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00190-a008560d-6baf-43ae-ada9-12178adf9071",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "99b489b",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 238
   },
   "source": "keras.backend.clear_session()\ntf.random.set_seed(42)\nnp.random.seed(42)\n\nmodel = keras.models.Sequential()\nmodel.add(keras.layers.Flatten(input_shape=[32, 32, 3]))\nfor _ in range(20):\n    model.add(keras.layers.Dense(100,\n                                 activation=\"elu\",\n                                 kernel_initializer=\"he_normal\"))",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "### b.\n*Exercise: Using Nadam optimization and early stopping, train the network on the CIFAR10 dataset. You can load it with `keras.datasets.cifar10.load_data()`. The dataset is composed of 60,000 32 × 32–pixel color images (50,000 for training, 10,000 for testing) with 10 classes, so you'll need a softmax output layer with 10 neurons. Remember to search for the right learning rate each time you change the model's architecture or hyperparameters.*",
   "metadata": {
    "cell_id": "00191-e034f403-bb27-42b4-a01c-694638ec47ab",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 145.171875
   }
  },
  {
   "cell_type": "markdown",
   "source": "Let's add the output layer to the model:",
   "metadata": {
    "cell_id": "00192-560f387e-4c92-46c3-a7e5-c9e1044783ee",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00193-7db63013-3244-48e2-ba1c-2d1e97dd451b",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "4eae7f1e",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "model.add(keras.layers.Dense(10, activation=\"softmax\"))",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "Let's use a Nadam optimizer with a learning rate of 5e-5. I tried learning rates 1e-5, 3e-5, 1e-4, 3e-4, 1e-3, 3e-3 and 1e-2, and I compared their learning curves for 10 epochs each (using the TensorBoard callback, below). The learning rates 3e-5 and 1e-4 were pretty good, so I tried 5e-5, which turned out to be slightly better.",
   "metadata": {
    "cell_id": "00194-6cded11a-10d1-47f2-ac60-3d76a9fc3597",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 97.171875
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00195-c8054eb9-e638-4455-9ff8-8f0fea293b15",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "1cb6db96",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 130
   },
   "source": "optimizer = keras.optimizers.Nadam(learning_rate=5e-5)\nmodel.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=optimizer,\n              metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "Let's load the CIFAR10 dataset. We also want to use early stopping, so we need a validation set. Let's use the first 5,000 images of the original training set as the validation set:",
   "metadata": {
    "cell_id": "00196-87502ab8-8673-474d-99f1-e5ab172af78c",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 74.78125
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00197-6397dddb-d96d-47ed-8b86-7339bb8ad28c",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "54203219",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 166
   },
   "source": "(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.cifar10.load_data()\n\nX_train = X_train_full[5000:]\ny_train = y_train_full[5000:]\nX_valid = X_train_full[:5000]\ny_valid = y_train_full[:5000]",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "Now we can create the callbacks we need and train the model:",
   "metadata": {
    "cell_id": "00198-87eb6964-75b4-42e7-a0cc-48e390df533b",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00199-dfb109c2-3822-4b9a-8af7-0172cc6aa6ac",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "5799edf",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 166
   },
   "source": "early_stopping_cb = keras.callbacks.EarlyStopping(patience=20)\nmodel_checkpoint_cb = keras.callbacks.ModelCheckpoint(\"my_cifar10_model.h5\", save_best_only=True)\nrun_index = 1 # increment every time you train the model\nrun_logdir = os.path.join(os.curdir, \"my_cifar10_logs\", \"run_{:03d}\".format(run_index))\ntensorboard_cb = keras.callbacks.TensorBoard(run_logdir)\ncallbacks = [early_stopping_cb, model_checkpoint_cb, tensorboard_cb]",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00200-57917064-6d92-4338-9027-3c533f9e5143",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "ae0162b6",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 170.75
   },
   "source": "%tensorboard --logdir=./my_cifar10_logs --port=6006",
   "execution_count": null,
   "outputs": [
    {
     "data": {
      "text/plain": "ERROR: Failed to launch TensorBoard (exited with 255).\nContents of stderr:\nE0213 19:12:55.493896 4621630912 program.py:311] TensorBoard could not bind to port 6006, it was already in use\nERROR: TensorBoard could not bind to port 6006, it was already in use"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00201-684ec2bc-947b-4d63-b8d1-df82bfc8842d",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "9de92f89",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 784.1875,
    "deepnote_output_heights": [
     null,
     21.1875
    ]
   },
   "source": "model.fit(X_train, y_train, epochs=100,\n          validation_data=(X_valid, y_valid),\n          callbacks=callbacks)",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "Epoch 1/100\n1407/1407 [==============================] - 9s 5ms/step - loss: 9.4191 - accuracy: 0.1388 - val_loss: 2.2328 - val_accuracy: 0.2040\nEpoch 2/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 2.1097 - accuracy: 0.2317 - val_loss: 2.0485 - val_accuracy: 0.2402\nEpoch 3/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.9667 - accuracy: 0.2844 - val_loss: 1.9681 - val_accuracy: 0.2964\nEpoch 4/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.8740 - accuracy: 0.3149 - val_loss: 1.9178 - val_accuracy: 0.3254\nEpoch 5/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.8064 - accuracy: 0.3423 - val_loss: 1.8256 - val_accuracy: 0.3384\nEpoch 6/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.7525 - accuracy: 0.3595 - val_loss: 1.7430 - val_accuracy: 0.3692\nEpoch 7/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.7116 - accuracy: 0.3819 - val_loss: 1.7199 - val_accuracy: 0.3824\nEpoch 8/100\n1407/1407 [==============================] - 8s 5ms/step - loss: 1.6782 - accuracy: 0.3935 - val_loss: 1.6746 - val_accuracy: 0.3972\nEpoch 9/100\n1407/1407 [==============================] - 8s 5ms/step - loss: 1.6517 - accuracy: 0.4025 - val_loss: 1.6622 - val_accuracy: 0.4004\nEpoch 10/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.6140 - accuracy: 0.4194 - val_loss: 1.7065 - val_accuracy: 0.3840\nEpoch 11/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.5884 - accuracy: 0.4301 - val_loss: 1.6736 - val_accuracy: 0.3914\nEpoch 12/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.5640 - accuracy: 0.4378 - val_loss: 1.6220 - val_accuracy: 0.4224\nEpoch 13/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.5437 - accuracy: 0.4448 - val_loss: 1.6332 - val_accuracy: 0.4144\nEpoch 14/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.5214 - accuracy: 0.4555 - val_loss: 1.5785 - val_accuracy: 0.4326\nEpoch 15/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.5117 - accuracy: 0.4564 - val_loss: 1.6267 - val_accuracy: 0.4164\nEpoch 16/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.4972 - accuracy: 0.4622 - val_loss: 1.5846 - val_accuracy: 0.4316\nEpoch 17/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.4888 - accuracy: 0.4661 - val_loss: 1.5549 - val_accuracy: 0.4420\nEpoch 18/100\n<<24 more lines>>\n1407/1407 [==============================] - 8s 5ms/step - loss: 1.3362 - accuracy: 0.5212 - val_loss: 1.6025 - val_accuracy: 0.4500\nEpoch 31/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.3360 - accuracy: 0.5207 - val_loss: 1.5175 - val_accuracy: 0.4602\nEpoch 32/100\n1407/1407 [==============================] - 8s 5ms/step - loss: 1.3031 - accuracy: 0.5302 - val_loss: 1.5397 - val_accuracy: 0.4572\nEpoch 33/100\n1407/1407 [==============================] - 8s 5ms/step - loss: 1.3082 - accuracy: 0.5308 - val_loss: 1.4997 - val_accuracy: 0.4776\nEpoch 34/100\n1407/1407 [==============================] - 8s 5ms/step - loss: 1.2882 - accuracy: 0.5338 - val_loss: 1.5482 - val_accuracy: 0.4620\nEpoch 35/100\n1407/1407 [==============================] - 8s 5ms/step - loss: 1.2889 - accuracy: 0.5355 - val_loss: 1.5474 - val_accuracy: 0.4604\nEpoch 36/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.2761 - accuracy: 0.5410 - val_loss: 1.5434 - val_accuracy: 0.4658\nEpoch 37/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.2658 - accuracy: 0.5481 - val_loss: 1.5502 - val_accuracy: 0.4706\nEpoch 38/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.2554 - accuracy: 0.5489 - val_loss: 1.5527 - val_accuracy: 0.4624\nEpoch 39/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.2504 - accuracy: 0.5471 - val_loss: 1.5482 - val_accuracy: 0.4602\nEpoch 40/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.2516 - accuracy: 0.5545 - val_loss: 1.5881 - val_accuracy: 0.4574\nEpoch 41/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.2401 - accuracy: 0.5566 - val_loss: 1.5403 - val_accuracy: 0.4670\nEpoch 42/100\n1407/1407 [==============================] - 8s 5ms/step - loss: 1.2305 - accuracy: 0.5570 - val_loss: 1.5343 - val_accuracy: 0.4790\nEpoch 43/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.2228 - accuracy: 0.5615 - val_loss: 1.5344 - val_accuracy: 0.4708\nEpoch 44/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.2161 - accuracy: 0.5619 - val_loss: 1.5782 - val_accuracy: 0.4526\nEpoch 45/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.2124 - accuracy: 0.5641 - val_loss: 1.5182 - val_accuracy: 0.4794\nEpoch 46/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.1870 - accuracy: 0.5766 - val_loss: 1.5435 - val_accuracy: 0.4650\nEpoch 47/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.1925 - accuracy: 0.5701 - val_loss: 1.5532 - val_accuracy: 0.4686\n"
    },
    {
     "data": {
      "text/plain": "<tensorflow.python.keras.callbacks.History at 0x7fb1d8ef8790>"
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00202-5704e032-355c-4e6c-b82e-d75eada679c0",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "4fcc449",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 177.375,
    "deepnote_output_heights": [
     null,
     21.1875
    ]
   },
   "source": "model = keras.models.load_model(\"my_cifar10_model.h5\")\nmodel.evaluate(X_valid, y_valid)",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "157/157 [==============================] - 0s 1ms/step - loss: 1.4960 - accuracy: 0.4762\n"
    },
    {
     "data": {
      "text/plain": "[1.4960416555404663, 0.47620001435279846]"
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "The model with the lowest validation loss gets about 47.6% accuracy on the validation set. It took 27 epochs to reach the lowest validation loss, with roughly 8 seconds per epoch on my laptop (without a GPU). Let's see if we can improve performance using Batch Normalization.",
   "metadata": {
    "cell_id": "00203-5359c5dd-0a05-4eb1-a824-33529f09e2df",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 74.78125
   }
  },
  {
   "cell_type": "markdown",
   "source": "### c.\n*Exercise: Now try adding Batch Normalization and compare the learning curves: Is it converging faster than before? Does it produce a better model? How does it affect training speed?*",
   "metadata": {
    "cell_id": "00204-9ef821b9-e2b4-4509-957d-c0428e45b865",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 122.78125
   }
  },
  {
   "cell_type": "markdown",
   "source": "The code below is very similar to the code above, with a few changes:\n\n* I added a BN layer after every Dense layer (before the activation function), except for the output layer. I also added a BN layer before the first hidden layer.\n* I changed the learning rate to 5e-4. I experimented with 1e-5, 3e-5, 5e-5, 1e-4, 3e-4, 5e-4, 1e-3 and 3e-3, and I chose the one with the best validation performance after 20 epochs.\n* I renamed the run directories to run_bn_* and the model file name to my_cifar10_bn_model.h5.",
   "metadata": {
    "cell_id": "00205-456f687f-c46c-46bc-b08a-720c94066c70",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 175.953125
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00206-9f3619b4-d4d0-4a65-9464-56dea6a145be",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "181b1300",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 1288.1875,
    "deepnote_output_heights": [
     null,
     21.1875
    ]
   },
   "source": "keras.backend.clear_session()\ntf.random.set_seed(42)\nnp.random.seed(42)\n\nmodel = keras.models.Sequential()\nmodel.add(keras.layers.Flatten(input_shape=[32, 32, 3]))\nmodel.add(keras.layers.BatchNormalization())\nfor _ in range(20):\n    model.add(keras.layers.Dense(100, kernel_initializer=\"he_normal\"))\n    model.add(keras.layers.BatchNormalization())\n    model.add(keras.layers.Activation(\"elu\"))\nmodel.add(keras.layers.Dense(10, activation=\"softmax\"))\n\noptimizer = keras.optimizers.Nadam(learning_rate=5e-4)\nmodel.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=optimizer,\n              metrics=[\"accuracy\"])\n\nearly_stopping_cb = keras.callbacks.EarlyStopping(patience=20)\nmodel_checkpoint_cb = keras.callbacks.ModelCheckpoint(\"my_cifar10_bn_model.h5\", save_best_only=True)\nrun_index = 1 # increment every time you train the model\nrun_logdir = os.path.join(os.curdir, \"my_cifar10_logs\", \"run_bn_{:03d}\".format(run_index))\ntensorboard_cb = keras.callbacks.TensorBoard(run_logdir)\ncallbacks = [early_stopping_cb, model_checkpoint_cb, tensorboard_cb]\n\nmodel.fit(X_train, y_train, epochs=100,\n          validation_data=(X_valid, y_valid),\n          callbacks=callbacks)\n\nmodel = keras.models.load_model(\"my_cifar10_bn_model.h5\")\nmodel.evaluate(X_valid, y_valid)",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "Epoch 1/100\n1407/1407 [==============================] - 19s 9ms/step - loss: 1.9765 - accuracy: 0.2968 - val_loss: 1.6602 - val_accuracy: 0.4042\nEpoch 2/100\n1407/1407 [==============================] - 11s 8ms/step - loss: 1.6787 - accuracy: 0.4056 - val_loss: 1.5887 - val_accuracy: 0.4304\nEpoch 3/100\n1407/1407 [==============================] - 11s 8ms/step - loss: 1.6097 - accuracy: 0.4274 - val_loss: 1.5781 - val_accuracy: 0.4326\nEpoch 4/100\n1407/1407 [==============================] - 11s 8ms/step - loss: 1.5574 - accuracy: 0.4486 - val_loss: 1.5064 - val_accuracy: 0.4676\nEpoch 5/100\n1407/1407 [==============================] - 11s 8ms/step - loss: 1.5075 - accuracy: 0.4642 - val_loss: 1.4412 - val_accuracy: 0.4844\nEpoch 6/100\n1407/1407 [==============================] - 11s 8ms/step - loss: 1.4664 - accuracy: 0.4787 - val_loss: 1.4179 - val_accuracy: 0.4984\nEpoch 7/100\n1407/1407 [==============================] - 11s 8ms/step - loss: 1.4334 - accuracy: 0.4932 - val_loss: 1.4277 - val_accuracy: 0.4906\nEpoch 8/100\n1407/1407 [==============================] - 12s 8ms/step - loss: 1.4054 - accuracy: 0.5038 - val_loss: 1.3843 - val_accuracy: 0.5130\nEpoch 9/100\n1407/1407 [==============================] - 12s 8ms/step - loss: 1.3816 - accuracy: 0.5106 - val_loss: 1.3691 - val_accuracy: 0.5108\nEpoch 10/100\n1407/1407 [==============================] - 12s 8ms/step - loss: 1.3547 - accuracy: 0.5206 - val_loss: 1.3552 - val_accuracy: 0.5226\nEpoch 11/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 1.3244 - accuracy: 0.5371 - val_loss: 1.3678 - val_accuracy: 0.5142\nEpoch 12/100\n1407/1407 [==============================] - 12s 8ms/step - loss: 1.3078 - accuracy: 0.5393 - val_loss: 1.3844 - val_accuracy: 0.5080\nEpoch 13/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 1.2889 - accuracy: 0.5431 - val_loss: 1.3566 - val_accuracy: 0.5164\nEpoch 14/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 1.2607 - accuracy: 0.5559 - val_loss: 1.3626 - val_accuracy: 0.5248\nEpoch 15/100\n1407/1407 [==============================] - 12s 8ms/step - loss: 1.2580 - accuracy: 0.5587 - val_loss: 1.3616 - val_accuracy: 0.5276\nEpoch 16/100\n1407/1407 [==============================] - 12s 8ms/step - loss: 1.2441 - accuracy: 0.5586 - val_loss: 1.3350 - val_accuracy: 0.5286\nEpoch 17/100\n1407/1407 [==============================] - 12s 8ms/step - loss: 1.2241 - accuracy: 0.5676 - val_loss: 1.3370 - val_accuracy: 0.5408\nEpoch 18/100\n<<29 more lines>>\nEpoch 33/100\n1407/1407 [==============================] - 12s 8ms/step - loss: 1.0336 - accuracy: 0.6369 - val_loss: 1.3682 - val_accuracy: 0.5450\nEpoch 34/100\n1407/1407 [==============================] - 11s 8ms/step - loss: 1.0228 - accuracy: 0.6388 - val_loss: 1.3348 - val_accuracy: 0.5458\nEpoch 35/100\n1407/1407 [==============================] - 12s 8ms/step - loss: 1.0205 - accuracy: 0.6407 - val_loss: 1.3490 - val_accuracy: 0.5440\nEpoch 36/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 1.0008 - accuracy: 0.6489 - val_loss: 1.3568 - val_accuracy: 0.5408\nEpoch 37/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 0.9785 - accuracy: 0.6543 - val_loss: 1.3628 - val_accuracy: 0.5396\nEpoch 38/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 0.9832 - accuracy: 0.6592 - val_loss: 1.3617 - val_accuracy: 0.5482\nEpoch 39/100\n1407/1407 [==============================] - 12s 8ms/step - loss: 0.9707 - accuracy: 0.6581 - val_loss: 1.3767 - val_accuracy: 0.5446\nEpoch 40/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 0.9590 - accuracy: 0.6651 - val_loss: 1.4200 - val_accuracy: 0.5314\nEpoch 41/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 0.9548 - accuracy: 0.6668 - val_loss: 1.3692 - val_accuracy: 0.5450\nEpoch 42/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 0.9480 - accuracy: 0.6667 - val_loss: 1.3841 - val_accuracy: 0.5310\nEpoch 43/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 0.9411 - accuracy: 0.6716 - val_loss: 1.4036 - val_accuracy: 0.5382\nEpoch 44/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 0.9383 - accuracy: 0.6708 - val_loss: 1.4114 - val_accuracy: 0.5236\nEpoch 45/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 0.9258 - accuracy: 0.6769 - val_loss: 1.4224 - val_accuracy: 0.5324\nEpoch 46/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 0.9072 - accuracy: 0.6836 - val_loss: 1.3875 - val_accuracy: 0.5442\nEpoch 47/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 0.8996 - accuracy: 0.6850 - val_loss: 1.4449 - val_accuracy: 0.5280\nEpoch 48/100\n1407/1407 [==============================] - 13s 9ms/step - loss: 0.9050 - accuracy: 0.6835 - val_loss: 1.4167 - val_accuracy: 0.5338\nEpoch 49/100\n1407/1407 [==============================] - 12s 9ms/step - loss: 0.8934 - accuracy: 0.6880 - val_loss: 1.4260 - val_accuracy: 0.5294\n157/157 [==============================] - 1s 2ms/step - loss: 1.3344 - accuracy: 0.5398\n"
    },
    {
     "data": {
      "text/plain": "[1.3343921899795532, 0.5397999882698059]"
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "* *Is the model converging faster than before?* Much faster! The previous model took 27 epochs to reach the lowest validation loss, while the new model achieved that same loss in just 5 epochs and continued to make progress until the 16th epoch. The BN layers stabilized training and allowed us to use a much larger learning rate, so convergence was faster.\n* *Does BN produce a better model?* Yes! The final model is also much better, with 54.0% accuracy instead of 47.6%. It's still not a very good model, but at least it's much better than before (a Convolutional Neural Network would do much better, but that's a different topic, see chapter 14).\n* *How does BN affect training speed?* Although the model converged much faster, each epoch took about 12s instead of 8s, because of the extra computations required by the BN layers. But overall the training time (wall time) was shortened significantly!",
   "metadata": {
    "cell_id": "00207-fd15760a-37b7-4796-833c-7e88d2936735",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 209.734375
   }
  },
  {
   "cell_type": "markdown",
   "source": "### d.\n*Exercise: Try replacing Batch Normalization with SELU, and make the necessary adjustements to ensure the network self-normalizes (i.e., standardize the input features, use LeCun normal initialization, make sure the DNN contains only a sequence of dense layers, etc.).*",
   "metadata": {
    "cell_id": "00208-afc2ecac-a083-4deb-b623-4403b19cd589",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 122.78125
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "scrolled": true,
    "cell_id": "00209-7418b013-2552-49dc-9dd1-0848b3e65eae",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "65264ce9",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 1378.1875,
    "deepnote_output_heights": [
     null,
     21.1875
    ]
   },
   "source": "keras.backend.clear_session()\ntf.random.set_seed(42)\nnp.random.seed(42)\n\nmodel = keras.models.Sequential()\nmodel.add(keras.layers.Flatten(input_shape=[32, 32, 3]))\nfor _ in range(20):\n    model.add(keras.layers.Dense(100,\n                                 kernel_initializer=\"lecun_normal\",\n                                 activation=\"selu\"))\nmodel.add(keras.layers.Dense(10, activation=\"softmax\"))\n\noptimizer = keras.optimizers.Nadam(learning_rate=7e-4)\nmodel.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=optimizer,\n              metrics=[\"accuracy\"])\n\nearly_stopping_cb = keras.callbacks.EarlyStopping(patience=20)\nmodel_checkpoint_cb = keras.callbacks.ModelCheckpoint(\"my_cifar10_selu_model.h5\", save_best_only=True)\nrun_index = 1 # increment every time you train the model\nrun_logdir = os.path.join(os.curdir, \"my_cifar10_logs\", \"run_selu_{:03d}\".format(run_index))\ntensorboard_cb = keras.callbacks.TensorBoard(run_logdir)\ncallbacks = [early_stopping_cb, model_checkpoint_cb, tensorboard_cb]\n\nX_means = X_train.mean(axis=0)\nX_stds = X_train.std(axis=0)\nX_train_scaled = (X_train - X_means) / X_stds\nX_valid_scaled = (X_valid - X_means) / X_stds\nX_test_scaled = (X_test - X_means) / X_stds\n\nmodel.fit(X_train_scaled, y_train, epochs=100,\n          validation_data=(X_valid_scaled, y_valid),\n          callbacks=callbacks)\n\nmodel = keras.models.load_model(\"my_cifar10_selu_model.h5\")\nmodel.evaluate(X_valid_scaled, y_valid)",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "Epoch 1/100\n1407/1407 [==============================] - 10s 5ms/step - loss: 2.0622 - accuracy: 0.2631 - val_loss: 1.7878 - val_accuracy: 0.3552\nEpoch 2/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.7328 - accuracy: 0.3830 - val_loss: 1.7028 - val_accuracy: 0.3828\nEpoch 3/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.6342 - accuracy: 0.4279 - val_loss: 1.6692 - val_accuracy: 0.4022\nEpoch 4/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.5524 - accuracy: 0.4538 - val_loss: 1.6350 - val_accuracy: 0.4300\nEpoch 5/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.4979 - accuracy: 0.4756 - val_loss: 1.5773 - val_accuracy: 0.4356\nEpoch 6/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.4428 - accuracy: 0.4902 - val_loss: 1.5529 - val_accuracy: 0.4630\nEpoch 7/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.3966 - accuracy: 0.5126 - val_loss: 1.5290 - val_accuracy: 0.4682\nEpoch 8/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.3549 - accuracy: 0.5232 - val_loss: 1.4633 - val_accuracy: 0.4792\nEpoch 9/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.3162 - accuracy: 0.5444 - val_loss: 1.4787 - val_accuracy: 0.4776\nEpoch 10/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.2825 - accuracy: 0.5534 - val_loss: 1.4794 - val_accuracy: 0.4934\nEpoch 11/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.2529 - accuracy: 0.5682 - val_loss: 1.5529 - val_accuracy: 0.4982\nEpoch 12/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.2256 - accuracy: 0.5784 - val_loss: 1.4942 - val_accuracy: 0.4902\nEpoch 13/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.2049 - accuracy: 0.5823 - val_loss: 1.4868 - val_accuracy: 0.5024\nEpoch 14/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.1627 - accuracy: 0.6012 - val_loss: 1.4839 - val_accuracy: 0.5082\nEpoch 15/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.1543 - accuracy: 0.6034 - val_loss: 1.5097 - val_accuracy: 0.4968\nEpoch 16/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.1200 - accuracy: 0.6135 - val_loss: 1.5001 - val_accuracy: 0.5120\nEpoch 17/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.1028 - accuracy: 0.6199 - val_loss: 1.4856 - val_accuracy: 0.5056\nEpoch 18/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.0863 - accuracy: 0.6265 - val_loss: 1.5116 - val_accuracy: 0.4966\nEpoch 19/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.0715 - accuracy: 0.6345 - val_loss: 1.5787 - val_accuracy: 0.5070\nEpoch 20/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.0342 - accuracy: 0.6453 - val_loss: 1.4987 - val_accuracy: 0.5144\nEpoch 21/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.0169 - accuracy: 0.6531 - val_loss: 1.6292 - val_accuracy: 0.4462\nEpoch 22/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.1346 - accuracy: 0.6074 - val_loss: 1.5280 - val_accuracy: 0.5136\nEpoch 23/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 0.9820 - accuracy: 0.6678 - val_loss: 1.5392 - val_accuracy: 0.5040\nEpoch 24/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 0.9701 - accuracy: 0.6679 - val_loss: 1.5505 - val_accuracy: 0.5170\nEpoch 25/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.3604 - accuracy: 0.6753 - val_loss: 1.5468 - val_accuracy: 0.4992\nEpoch 26/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.0177 - accuracy: 0.6510 - val_loss: 1.5474 - val_accuracy: 0.5020\nEpoch 27/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 0.9425 - accuracy: 0.6798 - val_loss: 1.5545 - val_accuracy: 0.5076\nEpoch 28/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 0.9005 - accuracy: 0.6902 - val_loss: 1.5659 - val_accuracy: 0.5138\n157/157 [==============================] - 0s 1ms/step - loss: 1.4633 - accuracy: 0.4792\n"
    },
    {
     "data": {
      "text/plain": "[1.4633383750915527, 0.47920000553131104]"
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00210-d633a453-cff0-4de8-956a-bc8ac070bb39",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "703a20a3",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 177.375,
    "deepnote_output_heights": [
     null,
     21.1875
    ]
   },
   "source": "model = keras.models.load_model(\"my_cifar10_selu_model.h5\")\nmodel.evaluate(X_valid_scaled, y_valid)",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "157/157 [==============================] - 0s 1ms/step - loss: 1.4633 - accuracy: 0.4792\n"
    },
    {
     "data": {
      "text/plain": "[1.4633383750915527, 0.47920000553131104]"
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "We get 47.9% accuracy, which is not much better than the original model (47.6%), and not as good as the model using batch normalization (54.0%). However, convergence was almost as fast as with the BN model, plus each epoch took only 7 seconds. So it's by far the fastest model to train so far.",
   "metadata": {
    "cell_id": "00211-710afb36-26b7-4705-bd3e-a8b45e9190d6",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 74.78125
   }
  },
  {
   "cell_type": "markdown",
   "source": "### e.\n*Exercise: Try regularizing the model with alpha dropout. Then, without retraining your model, see if you can achieve better accuracy using MC Dropout.*",
   "metadata": {
    "cell_id": "00212-08c67e73-53a1-44fe-91ff-9b71464f7bdc",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 100.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00213-d8193a8f-67d7-424b-82b7-6f7784773a11",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "58ff3dea",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 1414.1875,
    "deepnote_output_heights": [
     null,
     21.1875
    ]
   },
   "source": "keras.backend.clear_session()\ntf.random.set_seed(42)\nnp.random.seed(42)\n\nmodel = keras.models.Sequential()\nmodel.add(keras.layers.Flatten(input_shape=[32, 32, 3]))\nfor _ in range(20):\n    model.add(keras.layers.Dense(100,\n                                 kernel_initializer=\"lecun_normal\",\n                                 activation=\"selu\"))\n\nmodel.add(keras.layers.AlphaDropout(rate=0.1))\nmodel.add(keras.layers.Dense(10, activation=\"softmax\"))\n\noptimizer = keras.optimizers.Nadam(learning_rate=5e-4)\nmodel.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=optimizer,\n              metrics=[\"accuracy\"])\n\nearly_stopping_cb = keras.callbacks.EarlyStopping(patience=20)\nmodel_checkpoint_cb = keras.callbacks.ModelCheckpoint(\"my_cifar10_alpha_dropout_model.h5\", save_best_only=True)\nrun_index = 1 # increment every time you train the model\nrun_logdir = os.path.join(os.curdir, \"my_cifar10_logs\", \"run_alpha_dropout_{:03d}\".format(run_index))\ntensorboard_cb = keras.callbacks.TensorBoard(run_logdir)\ncallbacks = [early_stopping_cb, model_checkpoint_cb, tensorboard_cb]\n\nX_means = X_train.mean(axis=0)\nX_stds = X_train.std(axis=0)\nX_train_scaled = (X_train - X_means) / X_stds\nX_valid_scaled = (X_valid - X_means) / X_stds\nX_test_scaled = (X_test - X_means) / X_stds\n\nmodel.fit(X_train_scaled, y_train, epochs=100,\n          validation_data=(X_valid_scaled, y_valid),\n          callbacks=callbacks)\n\nmodel = keras.models.load_model(\"my_cifar10_alpha_dropout_model.h5\")\nmodel.evaluate(X_valid_scaled, y_valid)",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "Epoch 1/100\n1407/1407 [==============================] - 9s 5ms/step - loss: 2.0583 - accuracy: 0.2742 - val_loss: 1.7429 - val_accuracy: 0.3858\nEpoch 2/100\n1407/1407 [==============================] - 6s 5ms/step - loss: 1.6852 - accuracy: 0.4008 - val_loss: 1.7055 - val_accuracy: 0.3792\nEpoch 3/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.5963 - accuracy: 0.4413 - val_loss: 1.7401 - val_accuracy: 0.4072\nEpoch 4/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.5231 - accuracy: 0.4634 - val_loss: 1.5728 - val_accuracy: 0.4584\nEpoch 5/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.4619 - accuracy: 0.4887 - val_loss: 1.5448 - val_accuracy: 0.4702\nEpoch 6/100\n1407/1407 [==============================] - 6s 5ms/step - loss: 1.4074 - accuracy: 0.5061 - val_loss: 1.5678 - val_accuracy: 0.4664\nEpoch 7/100\n1407/1407 [==============================] - 6s 5ms/step - loss: 1.3718 - accuracy: 0.5222 - val_loss: 1.5764 - val_accuracy: 0.4824\nEpoch 8/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.3220 - accuracy: 0.5387 - val_loss: 1.4805 - val_accuracy: 0.4890\nEpoch 9/100\n1407/1407 [==============================] - 6s 5ms/step - loss: 1.2908 - accuracy: 0.5487 - val_loss: 1.5521 - val_accuracy: 0.4638\nEpoch 10/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.2537 - accuracy: 0.5607 - val_loss: 1.5281 - val_accuracy: 0.4924\nEpoch 11/100\n1407/1407 [==============================] - 6s 5ms/step - loss: 1.2215 - accuracy: 0.5782 - val_loss: 1.5147 - val_accuracy: 0.5046\nEpoch 12/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.1910 - accuracy: 0.5831 - val_loss: 1.5248 - val_accuracy: 0.5002\nEpoch 13/100\n1407/1407 [==============================] - 6s 5ms/step - loss: 1.1659 - accuracy: 0.5982 - val_loss: 1.5620 - val_accuracy: 0.5066\nEpoch 14/100\n1407/1407 [==============================] - 6s 5ms/step - loss: 1.1282 - accuracy: 0.6120 - val_loss: 1.5440 - val_accuracy: 0.5180\nEpoch 15/100\n1407/1407 [==============================] - 6s 5ms/step - loss: 1.1127 - accuracy: 0.6133 - val_loss: 1.5782 - val_accuracy: 0.5146\nEpoch 16/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.0917 - accuracy: 0.6266 - val_loss: 1.6182 - val_accuracy: 0.5182\nEpoch 17/100\n1407/1407 [==============================] - 6s 5ms/step - loss: 1.0620 - accuracy: 0.6331 - val_loss: 1.6285 - val_accuracy: 0.5126\nEpoch 18/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.0433 - accuracy: 0.6413 - val_loss: 1.6299 - val_accuracy: 0.5158\nEpoch 19/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 1.0087 - accuracy: 0.6549 - val_loss: 1.7172 - val_accuracy: 0.5062\nEpoch 20/100\n1407/1407 [==============================] - 6s 5ms/step - loss: 0.9950 - accuracy: 0.6571 - val_loss: 1.6524 - val_accuracy: 0.5098\nEpoch 21/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 0.9848 - accuracy: 0.6652 - val_loss: 1.7686 - val_accuracy: 0.5038\nEpoch 22/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 0.9597 - accuracy: 0.6744 - val_loss: 1.6177 - val_accuracy: 0.5084\nEpoch 23/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 0.9399 - accuracy: 0.6790 - val_loss: 1.7095 - val_accuracy: 0.5082\nEpoch 24/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 0.9148 - accuracy: 0.6884 - val_loss: 1.7160 - val_accuracy: 0.5150\nEpoch 25/100\n1407/1407 [==============================] - 6s 5ms/step - loss: 0.9023 - accuracy: 0.6949 - val_loss: 1.7017 - val_accuracy: 0.5152\nEpoch 26/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 0.8732 - accuracy: 0.7031 - val_loss: 1.7274 - val_accuracy: 0.5088\nEpoch 27/100\n1407/1407 [==============================] - 6s 5ms/step - loss: 0.8542 - accuracy: 0.7091 - val_loss: 1.7648 - val_accuracy: 0.5166\nEpoch 28/100\n1407/1407 [==============================] - 7s 5ms/step - loss: 0.8499 - accuracy: 0.7118 - val_loss: 1.7973 - val_accuracy: 0.5000\n157/157 [==============================] - 0s 1ms/step - loss: 1.4805 - accuracy: 0.4890\n"
    },
    {
     "data": {
      "text/plain": "[1.4804893732070923, 0.48899999260902405]"
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "The model reaches 48.9% accuracy on the validation set. That's very slightly better than without dropout (47.6%). With an extensive hyperparameter search, it might be possible to do better (I tried dropout rates of 5%, 10%, 20% and 40%, and learning rates 1e-4, 3e-4, 5e-4, and 1e-3), but probably not much better in this case.",
   "metadata": {
    "cell_id": "00214-c62e6fff-bdbc-4098-9de6-1f9d7c409c47",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 97.171875
   }
  },
  {
   "cell_type": "markdown",
   "source": "Let's use MC Dropout now. We will need the `MCAlphaDropout` class we used earlier, so let's just copy it here for convenience:",
   "metadata": {
    "cell_id": "00215-592170c5-c8c2-4b0c-be5c-20b9abcfdb79",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00216-0849e103-b6f4-49dc-a98e-df01c8ad2b83",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "8c2a7c93",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 112
   },
   "source": "class MCAlphaDropout(keras.layers.AlphaDropout):\n    def call(self, inputs):\n        return super().call(inputs, training=True)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "Now let's create a new model, identical to the one we just trained (with the same weights), but with `MCAlphaDropout` dropout layers instead of `AlphaDropout` layers:",
   "metadata": {
    "cell_id": "00217-8116f61d-f1d4-4875-881f-358640d46497",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 74.78125
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00218-24eb8a4e-53e9-4160-83de-22a79cf16696",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "e9f125ad",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 130
   },
   "source": "mc_model = keras.models.Sequential([\n    MCAlphaDropout(layer.rate) if isinstance(layer, keras.layers.AlphaDropout) else layer\n    for layer in model.layers\n])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "Then let's add a couple utility functions. The first will run the model many times (10 by default) and it will return the mean predicted class probabilities. The second will use these mean probabilities to predict the most likely class for each instance:",
   "metadata": {
    "cell_id": "00219-5ad038cf-2330-480f-ab28-2f78a4a9da55",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 74.78125
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00220-d8d70f5b-8db5-44e2-a5ea-5fc514dd0f46",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "a824ab09",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 184
   },
   "source": "def mc_dropout_predict_probas(mc_model, X, n_samples=10):\n    Y_probas = [mc_model.predict(X) for sample in range(n_samples)]\n    return np.mean(Y_probas, axis=0)\n\ndef mc_dropout_predict_classes(mc_model, X, n_samples=10):\n    Y_probas = mc_dropout_predict_probas(mc_model, X, n_samples)\n    return np.argmax(Y_probas, axis=1)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "Now let's make predictions for all the instances in the validation set, and compute the accuracy:",
   "metadata": {
    "cell_id": "00221-0a3eb718-c76c-4fd9-b58b-b433c7a7c28d",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 52.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00222-bf895036-122e-4451-8d01-32370f38e5a0",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "2dd51ae7",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 221.1875,
    "deepnote_output_heights": [
     21.1875
    ]
   },
   "source": "keras.backend.clear_session()\ntf.random.set_seed(42)\nnp.random.seed(42)\n\ny_pred = mc_dropout_predict_classes(mc_model, X_valid_scaled)\naccuracy = np.mean(y_pred == y_valid[:, 0])\naccuracy",
   "execution_count": null,
   "outputs": [
    {
     "data": {
      "text/plain": "0.4892"
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "We get no accuracy improvement in this case (we're still at 48.9% accuracy).\n\nSo the best model we got in this exercise is the Batch Normalization model.",
   "metadata": {
    "cell_id": "00223-32f295da-7e30-4ed6-bb34-b69da59fede2",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 88.78125
   }
  },
  {
   "cell_type": "markdown",
   "source": "### f.\n*Exercise: Retrain your model using 1cycle scheduling and see if it improves training speed and model accuracy.*",
   "metadata": {
    "cell_id": "00224-722b2bdb-0a64-4082-93d5-369652b66dd6",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 100.390625
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00225-9be05859-de2b-45c2-89ce-bbd9a19fae81",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "ebcff951",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 382
   },
   "source": "keras.backend.clear_session()\ntf.random.set_seed(42)\nnp.random.seed(42)\n\nmodel = keras.models.Sequential()\nmodel.add(keras.layers.Flatten(input_shape=[32, 32, 3]))\nfor _ in range(20):\n    model.add(keras.layers.Dense(100,\n                                 kernel_initializer=\"lecun_normal\",\n                                 activation=\"selu\"))\n\nmodel.add(keras.layers.AlphaDropout(rate=0.1))\nmodel.add(keras.layers.Dense(10, activation=\"softmax\"))\n\noptimizer = keras.optimizers.SGD(learning_rate=1e-3)\nmodel.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=optimizer,\n              metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00226-6f4c8e7f-4028-48cc-9c38-d56f4bb3eea6",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "519388f3",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 576.9375,
    "deepnote_output_heights": [
     null,
     78.75,
     275
    ]
   },
   "source": "batch_size = 128\nrates, losses = find_learning_rate(model, X_train_scaled, y_train, epochs=1, batch_size=batch_size)\nplot_lr_vs_loss(rates, losses)\nplt.axis([min(rates), max(rates), min(losses), (losses[0] + min(losses)) / 1.4])",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "352/352 [==============================] - 2s 6ms/step - loss: nan - accuracy: 0.1255\n"
    },
    {
     "data": {
      "text/plain": "(9.999999747378752e-06,\n 9.999868392944336,\n 2.6167492866516113,\n 3.9354368618556435)"
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAERCAYAAACO6FuTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAng0lEQVR4nO3deXxV1bn/8c+TgYRMQEgCyBTCPAlIHJFRLc5jtdSrV+tVW/VqtXq1s8O1tWr1d1tbtVhaqqhVW1vnqcUJQSSoKCCiSJihYZ4DSZ7fH/sEY5oDCUnOPjn5vl+v/crZe69zzrM44TxZa+29lrk7IiIidUkKOwAREYlfShIiIhKVkoSIiESlJCEiIlEpSYiISFRKEiIiElVK2AE0pby8PC8sLAw7DBFpAht27GH15l0M7JJDSpKFHU5Cmzt37np3z6/rXEIlicLCQkpKSsIOQ0SawCPvLuMnf5/Paz86nvzstLDDSWhmtizaOXU3iYhIVEoSIiISlZKEiIhEpSQhIiJRKUmIiEhUShIiIhKVkoSIiESlJCEiIlEpSYiISFRKEiIiEpWShIiIRKUkISIiUSlJiIhIVEoSIiISlZKEiIhEpSQhIiJRKUmIiEhUMU0SZjbNzNaY2VYzW2xml0YpZ2Z2u5mtMrMtZvaGmQ2OZawiIhL7lsQdQKG75wCnA7eb2cg6yp0LXAKMBnKBWcAjMYtSRESAGCcJd1/g7uXVu5Gtdx1FewEz3P0Ld68EpgGDYhSmiIhExHxMwszuN7OdwCJgDfBiHcX+DPQxs35mlgpcBLwc5fUuN7MSMyspKytrtrhFRFqjmCcJd78SyCboSnoaKK+j2BrgbeBTYBdB99N1UV5vsrsXu3txfn5+8wQtItJKhXJ1k7tXuvsMoBtwRR1FbgYOB7oD6cCtwHQzy4hdlCIiEvYlsCnUPSYxDHjC3Ve6e4W7TwU6oHEJEZGYilmSMLMCM5tkZllmlmxmE4FvAtPrKD4HONfMOplZkpldCKQCn8cqXhERCf6SjxUn6Fp6kCA5LQOudfdnzKwHsBAY5O7LgTuBAuBDIJMgOZzj7ptjGK+ISKsXsyTh7mXA2CjnlgNZNfZ3A1dFNhERCUnYYxIiIhLHlCRERCQqJQkREYlKSUJERKJSkhARkaiUJEREJColCRERiUpJQkREolKSEBGRqJQkREQkKiUJERGJSklCRESiUpIQEZGolCRERCQqJQkREYlKSUJERKJSkhARkaiUJEREJColCRERiUpJQkREolKSEBGRqJQkREQkKiUJERGJKqZJwsymmdkaM9tqZovN7NL9lC0ys+fNbJuZrTezu2IZq4iIxL4lcQdQ6O45wOnA7WY2snYhM2sDvAZMBzoD3YBpsQxURERinCTcfYG7l1fvRrbedRS9GFjt7ve6+w533+3uH8UqThERCcR8TMLM7jezncAiYA3wYh3FjgJKzeylSFfTG2Y2NMrrXW5mJWZWUlZW1oyRi4i0PjFPEu5+JZANjAaeBsrrKNYNmAT8GjgEeAF4JtINVfv1Jrt7sbsX5+fnN1/gIiKtUChXN7l7pbvPIEgGV9RRZBcww91fcvc9wC+BjsDAGIYpItLqhX0JbAp1j0l8RDBeISIiIYpZkjCzAjObZGZZZpZsZhOBbxJcwVTbNOAoMzvezJKBa4H1wCexildERGLbknCCrqWVwCaCLqRr3f0ZM+thZtvNrAeAu38KXAA8GCl7BnB6pOtJRERiJCVWb+TuZcDYKOeWA1m1jj1NMLAtIiIhCXtMQkRE4piShIiIRKUkISIiUSlJiIhIVEoSIiISlZKEiIhEpSQhIiJRKUmIiEhUShIiIhKVkoSIiESlJCEiIlEpSYiISFRKEiIiEpWShIiIRKUkISIiUSlJiIhIVEoSIiISlZKEiIhEpSQhIiJRKUmIiEhUShIiIhKVkoSIiEQV0yRhZtPMbI2ZbTWzxWZ2aT2eM93M3MxSYhGjiIh8KdYtiTuAQnfPAU4HbjezkdEKm9l/AEoOIiIhiWmScPcF7l5evRvZetdV1szaATcDN8YoPBERqSXmYxJmdr+Z7QQWAWuAF6MU/TnwALA2VrGJiMhXxTxJuPuVQDYwGngaKK9dxsyKgVHAfQd6PTO73MxKzKykrKysqcMVEWnVQrm6yd0r3X0G0A24ouY5M0sC7ge+6+4V9Xitye5e7O7F+fn5zROwiEgrFfYlsCn8+5hEDlAMPGFma4E5keMrzWx0LIMTEWntYnblkJkVABOA54FdwPHAN4HzaxXdAhxSY7878B4wElB/kohIDMXy8lIn6Fp6kKAFswy41t2fMbMewEJgkLsvp8ZgtZmlRx6uq0/3k4iINJ2YJQl3LwPGRjm3HMiKcq4UsOaLTEREogl7TEJEROKYkoSIiETV6CRhZqlNEYiIiMSfBiUJM7vGzM6psT8F2GVmn5pZ/yaPTkREQtXQlsQ1RC5DNbMxwHkEl7B+CNzTpJGJiEjoGnp1U1egNPL4NOApd3/SzD4G3m7KwEREJHwNbUlsBarnvjgB+Gfk8V4gvc5niIhIi9XQlsSrwENm9gHQB3gpcnwwsLQpAxMRkfA1tCVxFfAOkAd83d03Ro4fBjzelIGJiEj4GtSScPetwNV1HL+5ySISEZG40dBLYAfVvNTVzE6IrFv9AzNLbvrwREQkTA3tbpoCjAAws27AM0AuQTfU7U0bmoiIhK2hSWIg8H7k8bnAbHc/GbiQYNpvERFJIA1NEsnAnsjj4/hyfeolQKemCkpEROJDQ5PEfOCKyApxxwEvR453BdY3ZWAiIhK+hiaJm4DLgDeAx93948jx0wlWjxMRkQTS0Etg3zKzfCDH3TfVOPU7YGeTRiYiIqFr8Mp07l5pZrvMbAjBkqRLIqvHiYhIgmnofRIpZnY3sAmYB3wMbDKzu7SuhIhI4mloS+IugktdvwPMiBwbDdxBkHBuaLrQREQkbA1NEucDl7j7izWOLTGzMuD3KEmIiCSUhl7d1I7gnojalgDtGx2NiIjElYYmiXkEq9PV9t3IORERSSANTRI3AheZ2WIz+5OZTTWzT4ELqEdXU2QywDVmtjXyGpdGKXeRmc2NlFsZGRhv8JVYIiLSOA1KEu7+FtAPeArIAnIijydSdwujtjuAQnfPIbgB73YzG1lHuQzgWoJ1K44kuLtb4x0iIjF2MPdJrAZ+VPOYmQ0DzqnHcxfU3I1svYG5tco9UGN3lZk9CoxvaKwiItI4De1uajQzu9/MdgKLgDV8OUng/owBFhywlIiINKmYJwl3vxLIJri/4mmgfH/lzexbQDHwyyjnLzezEjMrKSsra+pwRURatZgnCQim9nD3GUA34Ipo5czsTOAXwEnuXucss+4+2d2L3b04Pz+/WeIVEWmt6jUmYWbPHqBITiPev3eU9zwReAg4pcZssyIiEkP1HbjeUI/zS/dXwMwKgAnA88Au4HiCKT7Or6PsBOBR4Cx31xTkIiIhqVeScPdvNcF7OUHX0oME3VzLgGvd/Rkz6wEsBAa5+3LgJwR3d79oZtXPf9vdT2qCOEREpJ5idoOau5cBY6OcW05w30X1vi53FWnt3MOOQAhp4FpE5EAqq4IkkZxkBygpzUlJQkTiUmWkIZFsShJhUpIQkbhUFWlJJOlbKlT65xeRuFTp6m6KB0oSIhKXqiJJIkndTaFSkhCRuFSlgeu4oCQhInGpsir4qZZEuJQkRCQuVe7rbgo5kFZOSUJE4lJVlZNkYGpJhEpJQkTiUqW7xiPigJKEiMSloCWhJBE2JQkRiUuVVWpJxAMlCRGJS5XumpIjDihJiEhcqqpyktSSCJ2ShIjEJQ1cxwcliVqq7/IUkXBVVulGunigJFHDo7OXMeJ/X+OtxWVhhyLS6lVVOcn6hgpdwn8Ev/rHZzwxZ/lXjm3dvZen31/J5p172F5ewS9f+ZSfvbCQW59dyI7yCi57uITlG3aGFLGIgAau40XMli8Nw56KKh5483M6ZqZxXnF33OHCP8xm9hcbqahyDi/swOBD2jF1ZilJBqP65PHDkwdy9v0zueGpeXTLbUt5RRV3nnMotz+/kFWbd3HbGUPolZcZdtVEEl6Va+A6HiR0kpi/egu791axavMuFqzeSlpKEu98voFTD+3C0K7tuOOlRcwp3cQ5h3XjznOGkhJp2157fF/ueGkRn6xNYUd5BTM+W8+WXXtpm5rMhVNm89p1Y2nbJjnk2okktirdJxEXEjpJvLd0IwBm8OrCdXRtnw7A907oR1F+Fkf0yuWl+Wu5fEzRvgQB8O2xvfnPowtJS0ni1YXreG7eao7olUv/ztlMmvwud7z0CbeePnjfnDKVVc7spRuY/sm/aNsmmdTkJIZ2a8dn67axeN12ivIzuWRUL9JTlVhE6qvStXRpPEjoJFFSupGivEyy0lOYs3Qj63IzaJ+Ruq+7aESPDozo0aHO51a3FE4c0pkTh3Ted/xbowr54zulLNuwk6Fd25GRlszj7y1nxcZdtElJYm9lFV7jAqn87DT+Mnclr8xfy40nDmBUn7zmq7BIAtF9EvEhoZPE4nXbGda9Pe3bpvK3D1bxr227GdG9faNmlfzpqYPomZvBPa8u5u3Pyqhy6J2fyX3fHMHxAzvRJiWJHXsq+GD5ZgZ0zqZTTjovfbyGm59dwAVTZnP314fRrUNbjijM1X8Akf2ojMwCK+GKaZIws2nAcUAmsBa4y91/H6XsdcBNQFvgr8AV7l5e3/eqqKxi9eZdnD7sEArzMnnk3WVsL6vgvOLuja0DF4/qxaQjemAG67fvIT8rjTYpX3ZX5aSnMrZf/r79k4Z2YUy/fE69bwY3PDUPgEFdcvjdhSPpnpvRqHhEElWla4K/eBDrS2DvAArdPQc4HbjdzEbWLmRmE4HvEySUQqAIuLUhb7Rmy24qqpweuRkM795u3/GzDut68NHXkJ6aTFpKMl3bt/1KgogmMy2FKRcV879nDuGX5w5jxaadjLn7dc57cBYvz19LpW7iE/kKDVzHh5i2JNx9Qc3dyNYbmFur6EXAlOryZva/wKMEiaNeVmwM7nPoltuWorwsAHp2zKAgO/1gw2+0ovwsivKDWI4ozOVvH6ziqbkr+M60ubTPSOXIXrkcVdSRY/vksaRsO30KsuhTkB1avCJh0rQc8SHmYxJmdj9wMUE30gfAi3UUGww8U2N/HtDJzDq6+4b6vM/ySJLokZtBUpIx/fqxdMxKa0zoTapHxwy+e3xfrhrfm9cWrmP6on/x7tINvLJg3VfKnTK0Cz84eQDdOqhbSlqXSq0nERdiniTc/Uozuxo4GhgH1DXOkAVsqbFf/Tgb+EqSMLPLgcsBevTowfbyCm5/fiF/nrOC5CSjS7u2APv+go83KclJnDS0CycN7QIELaA3F5fRIzeDkmWbmPzWEl5ZsJY2KUnkZ6dRlJfJ2Yd1wwlaIx+v2kLJso1s311BZloKh3Zrx7Bu7enWoa2WfZQWrUotibgQytVN7l4JzDCzC4ArgF/XKrIdyKmxX/14Wx2vNRmYDFBcXOx3v7yIP89ZAbTMRUu652ZwwVE9ARjTL59vHN6dae8uo3xvFf/atpuS0k1c/fgHX3lOm+QkstJT2F5ewZ6KKgC6dWjLqN55nDqsC6P75v/b+4jEu8oqTcsRD8K+BDaFYEyitgXAMODJyP4wYF19upqWbtjJsG7tGNa9PdnpYVev8bq2b8tNJw7Yt79rTyXzVm4mJcn4eNUWeuVlMqpPHqnJSeypqOLTtdv4cMUm3vi0jFcWruWJkhUM7dqOY/vmcWyfPEb27KCb+qRFqKqCpISfXS7+xexb1MwKgAnA88Au4Hjgm8D5dRR/GJhqZo8Ca4AfA1Pr8z6bduyhY1YbbjtjSFOEHXfatknmqKKOABQX5n7lXJuU4E7vod3aceHRhZRXVPLIrGW8umAdD731BQ+8sYS0lCTOGH4Il40uoig/q8W1tKT1qHQnVVkidLH8U9sJupYeJLj0dhlwrbs/Y2Y9gIXAIHdf7u4vm9ldwOt8eZ/EzfV5k00799C3ID7HH2ItLSWZS0cXcenoInaUV/De0o289sk6/jp3JU+WrCQlyThhUCf+Z2L/uB2zkdZLA9fxIWZJwt3LgLFRzi0nGKyueexe4N6Gvs+mHXvokNnmoGJMZJlpKYwfUMD4AQVcf0I/nv9oDUvX7+Cv76/kH5+sY8KAAo7s1ZFh3dsz+JAcdUlJ6FwD13Gh5Xfa1+AOO/ZU0iEjNexQ4lrHrDQuOqYQgCvH9+bBN77gpflr9l1+m5JkjOjRnpOGdKFfp2yOKsr9ygSIIrGg9STiQ0IliYrIXctqSdRfQXY6Pz1tED89bRDrtu7mwxWb+XDFZl6Zv5bbnl8IBHNT3X3uMIZ3a6/5piRmKqvQ71scSKgkUVkVXP6Zm6EkcTA65aQzcXBnJg7uzI0T+7N++x5mL93Azc8s4Oz7Z9I9ty1Xj+/LucXddA+GNLsqXQIbFxIqSagl0XTMjPzsNE499BBG9c7jH5+s47H3lnPjXz9i2uxlXHBUT8b2y6cgO00JQ5qFpuWIDwmVJKonyeuglkST6pDZhnOLu3POYd14smQFv5+xlBv/8hEAmW2SKcrPYlz/fE4bdgj9OmmuKWkaWk8iPiRUkviyJaGB6+aQlGRMOqIH3zi8O+8v38TC1VtZUraDT9Zs5bevf8590z/n5KGduXJcH4Z0bXfgFxTZj2Cq8LCjkIRKEpWVThJqSTQ3M2Nkz1xG9vzyZr6ybeU8Nns5D765hBc/Xsvw7u254KienHpoF11OKwdF03LEh4S6rrGiqorstBRSdblmzOVnp/Hd4/vy7g+P4+bTBrF1915ueGoeR93xT+56eRFrt+wOO0RpYdTdFB8SqiWxa28lRbrbOlTt2qbyrVG9uPiYQmYt2cDUmaU88OYSJr/1Bacc2oXLRhepK0rqRfdJxIfEShJ7Kjmmd8ewwxCCLqlj+uRxTJ88lm/YyR9nLuXJOSt45sPVjO+fz4QBBZx3eHfSUtQVJXXTfRLxIaH6ZRw4ukhJIt706JjBzacNZuYPjuM7Y3tTumEnP3lmAcfcMZ0H3liipVulTsF6EmFHIQnVkjCguLBD2GFIFO3apvL9kwZw04n9mbVkAw+9/QV3vryIl+avYdLhPThuYAGdcsJbXlbiiwau40NCJYlD2rclo01CVSkhVXdFHd27I898uJq7X/mUH/7tY9o8m8S5xd24dHQRvfIyww5TQqaB6/iQUN+oubrTukUxM84c0ZUzhh/CkrLt/OGdUp4qWcmjs5dz7shuXDm+j5JFK1algeu4kFBJQlomM6NPQTY/P2so1x7fl9+/vZQpM5by1NyVDO3ajivH9ebEIZ01/Ucro2k54oOGhSSuFGSn88OTBzLjpvH8+JSB7NxTwRWPvs+Z989k5pL1YYcnMVSlq5vigpKExKUu7dpy6egiXr1uLHd9/VDKtu7m/Idmc/Ef32PNll1hhycxoPsk4oOShMS15CTjvOLuTL9hHD86eSBzlm7khHvf4mcvLGTr7r1hhyfNqFID13FBSUJahPTUZC4bU8QL14zmuIEFTJmxlOPueZNn563GXfdZJJqqyL0zakmET0lCWpTCvEx+NWkEz1x1LJ1z0rnm8Q+YNPldpi9at++LRVq+ykji18104dNHIC3S0G7t+PtVo7j19MGUbtjBJVNLOO7eN3n8veXsrawKOzxppOq78NXdFD4lCWmxkpOMi44pZMZNE/jVpOHkpKfwg6c/5oR7g24otSxaripXd1O8UJKQFi81OYkzhnfl71eNYspFxaSnJnPN4x9w6n0zeHn+Ws0N1QLta0koSYQuZknCzNLMbIqZLTOzbWb2gZmdFKWsmdntZrbKzLaY2RtmNjhWsUrLZGYcN7ATL14zmv/7xnB27KngO9PmMuGeN5j6zlK2l1eEHaLUU1Wkx1DdTeGLZUsiBVgBjAXaAT8BnjSzwjrKngtcAowGcoFZwCOxCVNauqSkYLqP6deP4/7/OIyOmW245bmFHP3zf3LbcwspXb8j7BDlAPYNXCtHhC5m03K4+w7glhqHnjezpcBIoLRW8V7ADHf/AsDMpgHXxSBMSSDJScbJQ7tw8tAufLhiM398ZykPzyrlD+8sZVz/fG74Wn8tgBSnqrubNC1H+EIbkzCzTkA/YEEdp/8M9DGzfmaWClwEvBzL+CSxDO/enl9NGsE735/Adcf346OVWzj1vhn8x+/fZfG6bWGHJ7VUD1yruyl8oSSJyBf/o8Cf3H1RHUXWAG8DnwK7CLqf6mxJmNnlZlZiZiVlZWXNFbIkiE456Xz3+L68fv04rj+hH5+s2cYpv36bH//9Y9Zt1Trc8UJXN8WPmCcJM0siGF/YA/x3lGI3A4cD3YF04FZgupll1C7o7pPdvdjdi/Pz85spakk07TJSufq4vrxy7RjOLe7On99bwZi7XudnLyxkw/bysMNr9XSfRPyIaZKwYK7nKUAn4Bx3jzb5zjDgCXdf6e4V7j4V6AAMik2k0lrkZ6fx87OGMv36cZxyaBemzFjKmLte555XP2XLLs0NFZbqq5vUkghfrFsSDwADgdPcfX9Tec4BzjWzTmaWZGYXAqnA57EIUlqfHh0zuPe84bx63RjGDSjgvumfM+oX07ntuYWs2Lgz7PBanS+n5VCSCFvMrm4ys57At4FyYG2NBWS+TTD+sBAY5O7LgTuBAuBDIJMgOZzj7ptjFa+0Tn0Ksvnt+Ydx1bitPPT2Fzw8q5SpM5dy3MBOXDmuNyN6aA31WFB3U/yI5SWwy4D9feJZNcruBq6KbCIxN+iQHP7fN4Zz04kDeHhWKU/MWcHZD8zkrBFduXpCXy2r2sw0cB0/NC2HyH50bpfOjScO4I3/Gcclo3rx8vy1TPy/t/j1Pz+jvKIy7PAS1pf3SYQciChJiNRHdnoqPzl1EG/cMI6vDerEva8t5uRfvc2z81Zr1tlmoLmb4oeShEgDFOSk85vzD2Pqtw7HHa55/ANG/WI6v/7nZ5Rt06WzTaVKA9dxI2ZjEiKJZFz/Asb0zefNxWVMnVnKva8t5jfTP+fUQ7tw9XEas2gsDVzHDyUJkYOUlGSMH1DA+AEFLCnbziOzlvFkyQqenbeaSUd055oJfSnISQ87zBZJA9fxQ0lCpAn0zs/iltMHc9X4Ptw3/TMem72cae8upyg/k7OGd+Xskd3o2r5t2GG2GNXDPBqTCJ/GJESaUH52GredMYR/fG8s/zOxPwXZadzz2mKOvXM6l0ydw0crN4cdYovwZXdTyIGIWhIizaEwL5OrxvfhqvF9WLFxJ0/NXcnDs0o5/TfvMGFAAZePKeLIXrmY/lKuk7qb4oeShEgz656bwfdO6Mdlo3sx9Z1S/jizlEmT36V/p2zOPqwrXx/ZjY5ZaWGHGVd0dVP8UGNOJEay04OZZ2d+fwK/OHsomWnJ3PHSIkbf9Tq3PreAL8q2hx1i3NDVTfFDLQmRGEtPTWbSET2YdEQPPlu3jd++/jnT3l3GH98p5ZShXbhhYv9Wfwmtupvih5KESIj6dsrm/yaN4EenDOKRWaX8fsZSXl6wlpE9O3Dm8K6cNqwL2empYYcZc9VXN6m7KXxKEiJxID87je99rT8XHl3In2aW8sqCtfzwbx9z63MLKC7sQN+CbE4a0pnDC3NbRReMpuWIH0oSInEkPzuNGyb25/qv9ePDFZt55sPVzF22iSfmrGDqzFK6tm/LqYd24ciiXEb2zKVd28RsZWjgOn4oSYjEITNjRI8O+9av2LmnglcXrOPpD1YxZcZSfvfWF5hB/07ZHF6YS3FhB8b2y6d9RpuQI28amgU2fihJiLQAGW1SOHNEV84c0ZVdeyr5cMVm5pRuZE7pRp5+fyWPvLuMjDbJnH1YV84Y3pWRPTq06G6p6paEupvCpyQh0sK0bZPM0b07cnTvjgBUVFbx8aotPDJrGX+Zu5Jp7y6nb0EWl40u4owRh5CWkhxyxA23pGwHoO6meGAeydiJoLi42EtKSsIOQyQ028sreHn+WqbMWMona7aSn53GxccUMnFwZ3rnZ7aIO7zfXFzGRX94j2N6d+SR/zpSiSIGzGyuuxfXeU5JQiTxuDszPl/P5Le+4O3P1gPQMbMNw7q3Z+LgThxdlEf33LZxmTR+9LeP+fsHq3j/pye0yFZQS7S/JKHuJpEEZGaM7pvP6L75LNuwg1lLNjCndBNzSjcyfdG/gOBKqqOKOvKN4u4c07tjXIxhuDtvfVbG0b3zlCDihJKESILr2TGTnh0zmXRED9ydRWu3UbJsE3NLN/Lm4jKem7eaXnmZnD2iKyN6dGBot3ahXVo764sNrNi4i8tHF4Xy/vLvlCREWhEzY2CXHAZ2yeHCo3qye28lL81fw7R3l3PPa4v3leuVl8mwbu0Y2bMDRxV1pE9BVrN2Tc1ftYXH3lvO4+8tp3NOOhOHdG6295KG0ZiEiACwZedePlq1mY9WbmHeis3MW7mZdVuDdbs7ZrYhIy2ZrLRUurRLp2fHDHrnZwVbQSb5WWkNSiJl28qZMmMpyzfuYNWmXcxbuYXkJOPCo3py/df6tcqpSMLUagauc3sO9BN++IewwxBJCO5OeUUVW3dXsG33XtyDm9z2VFaxe28lVTW+OpIsuKchKclom5pMkgWtFgPMiPwMksjOPZVsL6/AgLTUJFKSksjNbENeVhtSdfdcKJ78zjGtI0mY2Tbg00a+TDtgSyPL1XXuQMdqn6/er3k8D1hfj9j2J1b1299+tMexql9D61bX8TDq11yfXV3HG1q/lvS7WdexRK5ffb5berp7fp3v6O4JswElTfAakxtbrq5zBzpW+3z1fq0yLaZ++9vfz+OY1K+hdYuX+jXXZ9cU9WtJv5utrX71+W7Z36a23b97rgnK1XXuQMdqn38uyvHGilX99re/v3o3Vn1er6F1q+t4GPVrrs+uruOJVL+G/r4mWv0a9d2SaN1NJR6lXy0RqH4tWyLXL5HrBolfv/1JtJbE5LADaGaqX8uWyPVL5LpB4tcvqoRqSYiISNNKtJaEiIg0ISUJERGJqtUlCTMrNLMyM3sjstV9bXALZmbfNLOysONoambWycxmmtmbZjbdzLqEHVNTMrOjzWxWpH6Pm1lC3XZsZu3M7D0z225mQ8KOpymY2c/M7G0z+4uZZYQdT3NodUki4k13HxfZEurL1MySgK8DK8KOpRmsB45197HAw8B/hRxPU1sGTIjU7wvgjJDjaWo7gVOAv4QdSFOIJLre7j4a+AdwScghNYvWmiRGRbL/zy0eJ9RvnPMJ/hNWhR1IU3P3Snevrlc2sCDMeJqau692912R3QoS7DN0970J9kfZaOClyOOXgGNDjKXZxHWSMLP/NrMSMys3s6m1zuWa2d/MbIeZLTOz8+v5smuAPsAYoAA4u2mjrp/mqJuZJQPnAU80Q8gN0kyfHWY23MxmA/8NvN/EYddbc9Uv8vxewEnA800YcoM0Z/3iTSPq2oEvp7XYAuTGKOSYivepwlcDtwMTgba1zv0W2AN0AoYDL5jZPHdfYGadqbtJ+3V3XwuUA5jZ08BRwF+bJ/z9avK6RV7rSXevioMGUrN8du7+IXCkmZ0H/AD4TjPFfyDNUj8zywH+BFzo7nuaLfoDa67/e/HooOoKbCKY/4jIz40xiTbWGjsfSSw2gg9wao39TIIPrl+NY48Av6jHa+XUeHwH8J8JVLc7gVeBlwn+svl1gn12aTUeTwTuTbD6pQAvEIxLhFqv5qhfjfJTgSFh162xdQWGAo9FHl8OXB12HZpji+vupv3oB1S6++Iax+YBg+vx3LFmNtfM3ga6Ao81R4CNcNB1c/eb3P1r7n4i8Jm7X9NcQTZCYz67w8zsLTN7HbgWuLsZ4musxtTvm8CRwE8jV959ozkCbKTG1A8zexH4GvCQmV3c9OE1qf3W1d0/BpZFvksmAgm5TkG8dzdFk8W/T427hWAwc7/c/TmaflK5pnTQdavJ43eemcZ8drMIxpLiWWPq9wjBX6rxrFG/n+5+cpNH1HwOWFd3/0FMIwpBS21JbAdyah3LAbaFEEtTS+S6gerX0iV6/WpqTXWNqqUmicVAipn1rXFsGIlxSWQi1w1Uv5Yu0etXU2uqa1RxnSTMLMXM0oFkINnM0s0sxd13AE8Dt5lZppmNIrjxKN6b6vskct1A9UP1azFaU10PStgj5we42uAWwGttt0TO5QJ/B3YAy4Hzw45XdVP9VL+Wt7Wmuh7MpqnCRUQkqrjubhIRkXApSYiISFRKEiIiEpWShIiIRKUkISIiUSlJiIhIVEoSIiISlZKESBMys1vMbH7YcYg0Fd1MJy1OZPWwPHc/NexYajOzLIJ1LzaEHUs0ZubAue6eEGtNS/NSS0KkHsysTX3Kufv2MBKEmSVFlq8VaVJKEpJwzGyQmb1gZtvM7F9m9nhkWc3q84eb2atmtt7MtprZDDM7utZruJldZWZPm9kO4OfVXUlmNsnMlkRe/+9mllfjeV/pbjKzqWb2vJl918xWmdkmM/ujmWXUKJNpZg+b2XYzW2dmP4g8Z+p+6nhxpPzJkffbAww8UN3MrDTy8KlIHUtrnDstsiDXbjNbamY/q29ylMSlJCEJxcy6AG8B84EjgOMJFo951syqf9+zCWbyHB0p8yHwYs0v+4ibgRcJlqn8beRYIfAN4CyCFdZGAD87QFijgSGRWKqf+90a5+8BxkaOTyCYjnp0PaqbDvwY+DYwCFhWj7odHvl5GdClet/MJgKPAr8hWHntEoJ1039ejzgkkYU9w6A2bQ3dCNZIfj7KuduAf9Y61oFgZs8jojzHgDXABTWOOXBfrXK3ALuBdjWO/Qj4vFaZ+bViXQGk1Dj2EPCPyOMsglbApBrnM4FN1FhvuY6YL47EOPIA/1bR6vb1WuXeAn5S69iZBAvvWNifubbwNrUkJNGMBMZEumK2m9l2gi9pgN4AZlZgZr8zs8VmtoVgpbECoEet1yqp4/WXuXvNJS1XR567PwvdvSLKc3oDqcB71Sc9WMegPldIVRC0FPZpQN1qGwn8qNa/22MECavz/p8qiaylrnEtEk0S8AJwQx3n1kV+/gnoBFwHlALlwD+B2v3vO+p4jb219p0Dd9vu7zlW41hDlbt7Za1j9a1bbUnArcBTdZwrO4jYJEEoSUiieR84j+Av/tpfztWOBa5x9xcAzKwTQf98GD4nSCJHAEsj8WQQjGEsOYjXq0/d9hKswlbT+8AAd//8IN5TEpiShLRUOWY2vNaxzQQDzJcBT5jZnQR/BRcRJI7r3X0bwdrFF5jZbILulLsIxgVizt23m9kfgDvNbD3B+MGPCf6yP5jWRX3qVgocZ2ZvErRGNhGM5TxvZsuAJwm6soYQjOPceBBxSILQmIS0VKOBD2ptv3T31cAooAp4mWDR+t8SdLuUR557CcGA8Vzgz8AfCL44w3ID8DbwLPA68BHBeMjug3it+tTtemA8wVjNBwDu/gpwSuT4e5Ht+wRLdkorpjuuReKMmaURXM56t7vfE3Y80rqpu0kkZGY2AhhI8Nd7NnBT5OcTYcYlAkoSIvHie0B/vrysdYy7rww1IhHU3SQiIvuhgWsREYlKSUJERKJSkhARkaiUJEREJColCRERiUpJQkREovr/bDaS6ZZQmN4AAAAASUVORK5CYII=\n",
      "text/plain": "<Figure size 432x288 with 1 Axes>"
     },
     "metadata": {
      "needs_background": "light",
      "image/png": {
       "width": 393,
       "height": 273
      }
     },
     "output_type": "display_data"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00227-4033b784-7996-4b1a-9fb8-8c56eca7e425",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "5bb1a84c",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 382
   },
   "source": "keras.backend.clear_session()\ntf.random.set_seed(42)\nnp.random.seed(42)\n\nmodel = keras.models.Sequential()\nmodel.add(keras.layers.Flatten(input_shape=[32, 32, 3]))\nfor _ in range(20):\n    model.add(keras.layers.Dense(100,\n                                 kernel_initializer=\"lecun_normal\",\n                                 activation=\"selu\"))\n\nmodel.add(keras.layers.AlphaDropout(rate=0.1))\nmodel.add(keras.layers.Dense(10, activation=\"softmax\"))\n\noptimizer = keras.optimizers.SGD(learning_rate=1e-2)\nmodel.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=optimizer,\n              metrics=[\"accuracy\"])",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00228-f9f6dda1-34a8-4729-8d0f-3dcfc07ae901",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "d5a79ec",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 764.625
   },
   "source": "n_epochs = 15\nonecycle = OneCycleScheduler(math.ceil(len(X_train_scaled) / batch_size) * n_epochs, max_rate=0.05)\nhistory = model.fit(X_train_scaled, y_train, epochs=n_epochs, batch_size=batch_size,\n                    validation_data=(X_valid_scaled, y_valid),\n                    callbacks=[onecycle])",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "Epoch 1/15\n352/352 [==============================] - 3s 6ms/step - loss: 2.2298 - accuracy: 0.2349 - val_loss: 1.7841 - val_accuracy: 0.3834\nEpoch 2/15\n352/352 [==============================] - 2s 6ms/step - loss: 1.7928 - accuracy: 0.3689 - val_loss: 1.6806 - val_accuracy: 0.4086\nEpoch 3/15\n352/352 [==============================] - 2s 6ms/step - loss: 1.6475 - accuracy: 0.4190 - val_loss: 1.6378 - val_accuracy: 0.4350\nEpoch 4/15\n352/352 [==============================] - 2s 6ms/step - loss: 1.5428 - accuracy: 0.4543 - val_loss: 1.6266 - val_accuracy: 0.4390\nEpoch 5/15\n352/352 [==============================] - 2s 6ms/step - loss: 1.4865 - accuracy: 0.4769 - val_loss: 1.6158 - val_accuracy: 0.4384\nEpoch 6/15\n352/352 [==============================] - 2s 6ms/step - loss: 1.4339 - accuracy: 0.4866 - val_loss: 1.5850 - val_accuracy: 0.4412\nEpoch 7/15\n352/352 [==============================] - 2s 6ms/step - loss: 1.4042 - accuracy: 0.5056 - val_loss: 1.6146 - val_accuracy: 0.4384\nEpoch 8/15\n352/352 [==============================] - 2s 6ms/step - loss: 1.3437 - accuracy: 0.5229 - val_loss: 1.5299 - val_accuracy: 0.4846\nEpoch 9/15\n352/352 [==============================] - 2s 5ms/step - loss: 1.2721 - accuracy: 0.5459 - val_loss: 1.5145 - val_accuracy: 0.4874\nEpoch 10/15\n352/352 [==============================] - 2s 6ms/step - loss: 1.1942 - accuracy: 0.5698 - val_loss: 1.4958 - val_accuracy: 0.5040\nEpoch 11/15\n352/352 [==============================] - 2s 6ms/step - loss: 1.1211 - accuracy: 0.6033 - val_loss: 1.5406 - val_accuracy: 0.4984\nEpoch 12/15\n352/352 [==============================] - 2s 6ms/step - loss: 1.0673 - accuracy: 0.6161 - val_loss: 1.5284 - val_accuracy: 0.5144\nEpoch 13/15\n352/352 [==============================] - 2s 6ms/step - loss: 0.9927 - accuracy: 0.6435 - val_loss: 1.5449 - val_accuracy: 0.5140\nEpoch 14/15\n352/352 [==============================] - 2s 6ms/step - loss: 0.9205 - accuracy: 0.6703 - val_loss: 1.5652 - val_accuracy: 0.5224\nEpoch 15/15\n352/352 [==============================] - 2s 6ms/step - loss: 0.8936 - accuracy: 0.6801 - val_loss: 1.5912 - val_accuracy: 0.5198\n"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "One cycle allowed us to train the model in just 15 epochs, each taking only 2 seconds (thanks to the larger batch size). This is several times faster than the fastest model we trained so far. Moreover, we improved the model's performance (from 47.6% to 52.0%). The batch normalized model reaches a slightly better performance (54%), but it's much slower to train.",
   "metadata": {
    "cell_id": "00229-4217ae33-7964-4100-86b7-2ae4aacb43de",
    "deepnote_cell_type": "markdown",
    "deepnote_cell_height": 97.171875
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "00230-afcea27c-6ae7-48cf-bf6a-d8d500a2fba7",
    "deepnote_to_be_reexecuted": true,
    "source_hash": "b623e53d",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 76
   },
   "source": "",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "<a style='text-decoration:none;line-height:16px;display:flex;color:#5B5B62;padding:10px;justify-content:end;' href='https://deepnote.com?utm_source=created-in-deepnote-cell&projectId=8b8747e0-d5e2-4759-af6a-42b7fe569db7' target=\"_blank\">\n<img alt='Created in deepnote.com' style='display:inline;max-height:16px;margin:0px;margin-right:7.5px;' src='data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyB3aWR0aD0iODBweCIgaGVpZ2h0PSI4MHB4IiB2aWV3Qm94PSIwIDAgODAgODAiIHZlcnNpb249IjEuMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayI+CiAgICA8IS0tIEdlbmVyYXRvcjogU2tldGNoIDU0LjEgKDc2NDkwKSAtIGh0dHBzOi8vc2tldGNoYXBwLmNvbSAtLT4KICAgIDx0aXRsZT5Hcm91cCAzPC90aXRsZT4KICAgIDxkZXNjPkNyZWF0ZWQgd2l0aCBTa2V0Y2guPC9kZXNjPgogICAgPGcgaWQ9IkxhbmRpbmciIHN0cm9rZT0ibm9uZSIgc3Ryb2tlLXdpZHRoPSIxIiBmaWxsPSJub25lIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiPgogICAgICAgIDxnIGlkPSJBcnRib2FyZCIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTEyMzUuMDAwMDAwLCAtNzkuMDAwMDAwKSI+CiAgICAgICAgICAgIDxnIGlkPSJHcm91cC0zIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgxMjM1LjAwMDAwMCwgNzkuMDAwMDAwKSI+CiAgICAgICAgICAgICAgICA8cG9seWdvbiBpZD0iUGF0aC0yMCIgZmlsbD0iIzAyNjVCNCIgcG9pbnRzPSIyLjM3NjIzNzYyIDgwIDM4LjA0NzY2NjcgODAgNTcuODIxNzgyMiA3My44MDU3NTkyIDU3LjgyMTc4MjIgMzIuNzU5MjczOSAzOS4xNDAyMjc4IDMxLjY4MzE2ODMiPjwvcG9seWdvbj4KICAgICAgICAgICAgICAgIDxwYXRoIGQ9Ik0zNS4wMDc3MTgsODAgQzQyLjkwNjIwMDcsNzYuNDU0OTM1OCA0Ny41NjQ5MTY3LDcxLjU0MjI2NzEgNDguOTgzODY2LDY1LjI2MTk5MzkgQzUxLjExMjI4OTksNTUuODQxNTg0MiA0MS42NzcxNzk1LDQ5LjIxMjIyODQgMjUuNjIzOTg0Niw0OS4yMTIyMjg0IEMyNS40ODQ5Mjg5LDQ5LjEyNjg0NDggMjkuODI2MTI5Niw0My4yODM4MjQ4IDM4LjY0NzU4NjksMzEuNjgzMTY4MyBMNzIuODcxMjg3MSwzMi41NTQ0MjUgTDY1LjI4MDk3Myw2Ny42NzYzNDIxIEw1MS4xMTIyODk5LDc3LjM3NjE0NCBMMzUuMDA3NzE4LDgwIFoiIGlkPSJQYXRoLTIyIiBmaWxsPSIjMDAyODY4Ij48L3BhdGg+CiAgICAgICAgICAgICAgICA8cGF0aCBkPSJNMCwzNy43MzA0NDA1IEwyNy4xMTQ1MzcsMC4yNTcxMTE0MzYgQzYyLjM3MTUxMjMsLTEuOTkwNzE3MDEgODAsMTAuNTAwMzkyNyA4MCwzNy43MzA0NDA1IEM4MCw2NC45NjA0ODgyIDY0Ljc3NjUwMzgsNzkuMDUwMzQxNCAzNC4zMjk1MTEzLDgwIEM0Ny4wNTUzNDg5LDc3LjU2NzA4MDggNTMuNDE4MjY3Nyw3MC4zMTM2MTAzIDUzLjQxODI2NzcsNTguMjM5NTg4NSBDNTMuNDE4MjY3Nyw0MC4xMjg1NTU3IDM2LjMwMzk1NDQsMzcuNzMwNDQwNSAyNS4yMjc0MTcsMzcuNzMwNDQwNSBDMTcuODQzMDU4NiwzNy43MzA0NDA1IDkuNDMzOTE5NjYsMzcuNzMwNDQwNSAwLDM3LjczMDQ0MDUgWiIgaWQ9IlBhdGgtMTkiIGZpbGw9IiMzNzkzRUYiPjwvcGF0aD4KICAgICAgICAgICAgPC9nPgogICAgICAgIDwvZz4KICAgIDwvZz4KPC9zdmc+' > </img>\nCreated in <span style='font-weight:600;margin-left:4px;'>Deepnote</span></a>",
   "metadata": {
    "tags": [],
    "created_in_deepnote_cell": true,
    "deepnote_cell_type": "markdown"
   }
  }
 ],
 "nbformat": 4,
 "nbformat_minor": 4,
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "nav_menu": {
   "height": "360px",
   "width": "416px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "deepnote_notebook_id": "057c6ec3-a4d3-4c6f-93ab-0d4d1d0a5984",
  "deepnote": {},
  "deepnote_execution_queue": []
 }
}