gradient_descent_neural_network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import necessary modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import pandas as pd # for loading the Excel data set file\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "\n",
    "np.random.seed(1)\n",
    "tf.random.set_seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementation of a callback function that allows to evaluate the functional $J(\\omega)$ after each update step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BatchHistory(keras.callbacks.Callback):\n",
    "    loss_per_batch = []\n",
    "    def __init__(self):\n",
    "        self.loss_per_batch = []\n",
    "    def on_train_batch_end(self, batch, logs=None):\n",
    "        keys = list(logs.keys())\n",
    "        self.loss_per_batch.append(logs[\"loss\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementation of function that carries out stochastich gradient descent such that it returns the evolution of the functional $J({\\omega})$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stochastic_gradient(X,y,lr):\n",
    "    model = keras.Sequential(\n",
    "        [\n",
    "            layers.Dense(20, activation=\"relu\", name=\"layer1\"),\n",
    "            layers.Dense(10, activation=\"relu\", name=\"layer2\"),\n",
    "            layers.Dense(1, name=\"layer3\"),\n",
    "        ]\n",
    "    )\n",
    "    opt = tf.keras.optimizers.SGD(learning_rate=lr, momentum=0.0)\n",
    "    model.compile(optimizer=opt, loss='mse')\n",
    "    cb = BatchHistory()\n",
    "    model.fit(X, y, batch_size=1, epochs=5, verbose=0,callbacks=[cb])\n",
    "\n",
    "    J_omegas = np.asarray(cb.loss_per_batch)\n",
    "    ns = np.linspace(1,len(cb.loss_per_batch),len(cb.loss_per_batch))\n",
    "    \n",
    "    return J_omegas, ns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementation of function that carries out mini-batch gradient descent such that it returns the evolution of the functional $J({\\omega})$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def minibatch_gradient(X,y,lr):\n",
    "\n",
    "    N_b = 10  # needs to divide the training set size without remainder\n",
    "\n",
    "    model = keras.Sequential(\n",
    "        [\n",
    "            layers.Dense(20, activation=\"relu\", name=\"layer1\"),\n",
    "            layers.Dense(10, activation=\"relu\", name=\"layer2\"),\n",
    "            layers.Dense(1, name=\"layer3\"),\n",
    "        ]\n",
    "    )\n",
    "    opt = tf.keras.optimizers.SGD(learning_rate=lr, momentum=0.0)\n",
    "    model.compile(optimizer=opt, loss='mse')\n",
    "    cb = BatchHistory()\n",
    "    history = model.fit(X, y, batch_size=N_b, epochs=5, verbose=0,callbacks=[cb])\n",
    "\n",
    "    J_omegas = np.asarray(cb.loss_per_batch)\n",
    "    ns = np.linspace(1,len(cb.loss_per_batch),len(cb.loss_per_batch))*N_b\n",
    "    \n",
    "    return J_omegas, ns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementation of function that carries out batch gradient descent such that it returns the evolution of the functional $J({\\omega})$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_gradient(X,y,lr):\n",
    "\n",
    "    N_b = X.shape[0]\n",
    "\n",
    "    model = keras.Sequential(\n",
    "        [\n",
    "            layers.Dense(20, activation=\"relu\", name=\"layer1\"),\n",
    "            layers.Dense(10, activation=\"relu\", name=\"layer2\"),\n",
    "            layers.Dense(1, name=\"layer3\"),\n",
    "        ]\n",
    "    )\n",
    "    opt = tf.keras.optimizers.SGD(learning_rate=lr, momentum=0.0)\n",
    "    model.compile(optimizer=opt, loss='mse')\n",
    "    cb = BatchHistory()\n",
    "    history = model.fit(X, y, batch_size=N_b, epochs=5, verbose=0,callbacks=[cb])\n",
    "\n",
    "    J_omegas = np.asarray(cb.loss_per_batch)\n",
    "    ns = np.linspace(1,len(cb.loss_per_batch),len(cb.loss_per_batch))*N_b\n",
    "    \n",
    "    return J_omegas, ns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download and load the energy efficiency data set, furthermore scale the inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2023-03-10 16:23:10--  https://archive.ics.uci.edu/ml/machine-learning-databases/00242/ENB2012_data.xlsx\n",
      "Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n",
      "Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 76189 (74K) [application/x-httpd-php]\n",
      "Saving to: ‘ENB2012_data.xlsx.8’\n",
      "\n",
      "ENB2012_data.xlsx.8 100%[===================>]  74.40K   216KB/s    in 0.3s    \n",
      "\n",
      "2023-03-10 16:23:11 (216 KB/s) - ‘ENB2012_data.xlsx.8’ saved [76189/76189]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://archive.ics.uci.edu/ml/machine-learning-databases/00242/ENB2012_data.xlsx\n",
    "data_set = pd.read_excel('ENB2012_data.xlsx')\n",
    "data_set_np = np.array(data_set)\n",
    "\n",
    "N= 200\n",
    "\n",
    "X = data_set_np[:N,:8]\n",
    "y = data_set_np[:N,8]\n",
    "\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X)\n",
    "X = scaler.transform(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot magnitude of $J(\\omega)$ during stochastic gradient descent for different learning rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD5CAYAAADSiMnIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd5xU1d348c+Zvr03dhdYeheQIiBobChqbNHYEiwJmsd0UzTN5HkkP0000cSeGCVGoybGEkWs2EUE6b3Dsiy7sL1MP78/7mxvM7tTdtnv+/XiNXfunHvvuQPc75yutNYIIYQQnTHFOgNCCCH6LwkSQgghuiRBQgghRJckSAghhOiSBAkhhBBdkiAhhBCiS1ENEkqp/UqpTUqp9UqpNYF96Uqpt5RSuwKvaa3S366U2q2U2qGUWhjNvAohhIhNSeJLWuupWusZgfe3Ae9orUcD7wTeo5SaAFwJTATOBR5SSpljkF8hhBi0+kN100XAssD2MuDiVvuf1Vq7tNb7gN3ArBjkTwghBi1LlK+ngTeVUhp4VGv9GJCjtT4CoLU+opTKDqTNB1a1OrY4sK9LmZmZevjw4eHPtRBCnMDWrl17TGud1dln0Q4S87TWJYFA8JZSans3aVUn+zrMIaKUWgIsARg6dChr1qwJT06FEGKQUEod6OqzqFY3aa1LAq9lwIsY1UdHlVJ5AIHXskDyYqCw1eEFQEkn53xMaz1Daz0jK6vTQCiEEKKXohYklFIJSqmkpm3gHGAz8AqwOJBsMfByYPsV4EqllF0pVQSMBlZHK79CCCGiW92UA7yolGq67jNa6xVKqc+B55VSNwIHgcsBtNZblFLPA1sBL3CL1toXxfwKIcSgF7UgobXeC5zUyf7jwJldHLMUWBrhrAkhYszj8VBcXIzT6Yx1Vk5oDoeDgoICrFZr0MdEu+FaCCE6KC4uJikpieHDhxOobRBhprXm+PHjFBcXU1RUFPRx/WGchBBikHM6nWRkZEiAiCClFBkZGSGX1iRICCH6BQkQkdeb71iCRB+8tO4wNU5PrLMhhBARI0GiFzw+P+sPVfH959bzs/9sinV2hBAiYqThuheuemwVaw5UAlBc2Rjj3AghRORISaIXmgIEgFSjCiHaW7FiBWPHjmXUqFHcddddIacLdT/ADTfcQHZ2NpMmTQrrvUiQ6KN1B6vYXVYb62wIIfoJn8/HLbfcwuuvv87WrVv55z//ydatW4NOF+r+Jtdddx0rVqwI+/1IkAiR1h3mGOTl9R2mlBJCDEB33303S5Ys4eKLL6aoqIh77rkn5HOsXr2aUaNGMWLECGw2G1deeSUvv/xy0OlC3d9kwYIFpKen9+n+OyNtEiGqcXo77Et2BD96UQjRvd/8dwtbS2rCes4JQ5K548KJPabbtGkTZrOZF154gb1793LZZZfxox/9qPnz+fPnU1vbsebgnnvu4ayzzgLg8OHDFBa2zE1aUFDAZ5991uGYrtKFuj/SJEiEqKrB3WHf0uXb+OaCETHIjRAinDZu3MiLL76I2WzGbDZ3+GX+4Ycf9niOzmobOhuf0FW6UPdHmgSJEFU2dD4uotHtI84mq6sK0VfB/OKPBI/Hw7Fjxxg5ciRgBIzJkye3SRNMSaKgoIBDhw41f1ZcXMyQIUM6HNNVulD3R5oEiRBsO1LDTU+1XdToxwvH8vs3dnCwooGxuUkxypkQoq927NjB+PHjm9+vX7+ek05qOydpMCWJmTNnsmvXLvbt20d+fj7PPvsszzzzTNDpxo4dG9L+SJOG6xB8Y9kajta42uw7dVQmAAcrGmKRJSFEmGzatIkpU6Y0v+8sSATDYrHwwAMPsHDhQsaPH88VV1zBxIlG6WjRokWUlJR0my7U/U2uuuoq5syZw44dOygoKODxxx/v4zdiUJ3Vcw1UM2bM0JFcvvRL97zHvmP1ANhx83b670m55F6mPF7BL84fz5icJNITbEzKT4lYHoQ4EW3btq3Nr/j+YNSoUWzatIm4uLhYZyWsOvuulVJrtdYzOksvJYkQxAfaHO63PsAOx3UUNmwh6b2fk2Az8+GuY3z9b6u54M8fxTiXQoi+qqqqwmaznXABojckSIQgwW404Vxk/qR5n1Im6t0+3t9Z3rzv7a1HO+2JIIQYGFJTUzsdADcYSZAIQUJnvZdUx6/wG39fwxcHq7jnjR2c/H9vUV7r6nicEEIMABIkQhBvt2Cm3TLbhz7jga+M6ZD24ff28MDK3Ryvd/PJnmNRyqEQQoSXBIkQ2C0m0unYR/o89WmHfW9vO9q8vaesjq0lNRySHlBCiAFGgkQI4lzlfMPyWof9Zl9LddL7Pz69w+dfHKxi0Z8+ZP7vVnKkWqYWF0IMHBIkQvCN4l9wUydBgl1vMSIrgdHZiQzLSOA7Z4xq8/FHu1uqm578eH+EcymEEOEjI65DkOCr7vyDXW/w9i+fbV5c4odnj2HR5DzWHKjkUEUDj32wtzlpo8fX+TmEEKIfkpJECJyqkz7ThacAYGo8jslkBAmlFOPzkvnaKcOY3Gpg3YS8ZHYe7XntiQ2Hqlj2yf6w5FkIIfpCShIhaMDR8mbMuTB9MZgs8MzlULEHErM6HNN6PqeTClP55+qDOD0+HNauJwO84tFPcXn91Lu9XDGjkMxEe1jvQwghgiUliRA0tg4SVz8H4xZBhjFjJBV7Oz2mKDOhebsw3SiJ/OzFTd1ex+X1A/C7FTu45ekv+pBjIUQsRHv50kOHDvGlL32J8ePHM3HiRO6///6w3YsEiRA00Mkv+tShoMxdBgmrueUrvnCKMa3v1pIaVm4vw+Pzd0jv92ssppY54jcd7qIdRAjRL8Vi+VKLxcK9997Ltm3bWLVqFQ8++GDYRoxLkAhBvd/WcafZCnFpsPoxWP8M7FnZIclzS07huSWnUJgezzWzh7K9tJbrn/ycP7+7uznNvW/uYPKv32DvsTq8fs0dF07g5tNG0uD2UVHfcaEjIUT4DdTlS/Py8pg+fToASUlJjB8/nsOHD/ftywiQNokglVQ14nS7obOmhIZAF9eXvmW8/rrtr//ZIzKat6cNTePpzw4C8PSqA9w4r4iUeGtzwLj3zZ0A5CQ7mFKQyiPv7+GTPce4YErkFxcRol94/TYo7b5KNmS5k+G8rqt9mpwIy5fu37+fdevWMXv27B7vNxgSJIJ06/MbWExgfWuLo+2HmWPg2M6gzjNtaGrz9vF6Nxc+8BFP3Tired/rm0sBGJYRz9icJJIcFt7fUS5BQogoGOjLl9bV1XHZZZdx3333kZyc3GNegyFBIkjH613Y8FCvEkj4/vq2Hy7+L9w7tuX9+n/CSVc2j5torSgjoc37gxUNfOURY1qPgrQ4iiuNEdnDMxKwmE0sGJPFezvL8ft1cxfb43UuEh0W6pxeqho9jMxK7HCdpvYOj8+P2+sn0W7hz+/u5rSxWUwfmtbr70GIiAviF38kDPTlSz0eD5dddhnXXHMNl156aai33yUJEkGqd/mw4uWguZDxidltP0zMafv+pZuN0kXByR3OYzIpfnDWGJ5atZ9jdUZbQ9Mssb84fwK1TmMN7aZpyb80NpvXNh5h65EaJuWnsL20hnPvM37NmBT4Nbz+vfmU1bo4bUxLF9xz7/uAPeX1jM1JYkersRn3v7OLZ5ecwimtqsCEEAN7+VKtNTfeeCPjx4/nhz/8YR+/ibak4TpITo8Pm/Li0p3E1U5KDN1VP33vrNF8ctuZLFkwos3+hRNzuHxGIZfPaKl3PH2s8eBfub0MoDlAgBEgAM67/0MW/201D723m0a3j3+tOcSecmMFvR2dDN676/XtXeZNiMFqIC9f+vHHH/PUU0/x7rvvMnXqVKZOncry5cvD8K3EYPlSpZQZWAMc1lpfoJRKB54DhgP7gSu01pWBtLcDNwI+4Lta6ze6O3ckly89/fcrub/2h+Tl5ZP9rVc7JvjgHiNYzP0u/HYIFC2Aq54Dc9eFNa01Rbcbf5GzitJ5/qY5naa76IGPMJsUf7tuJlP/960uz5cab+WqWUN5+L09gLH+Rb3bmAbk0mn5nDY2i7IaF0uXb+NfN89h5vD0Ls8lRDTJ8qXRE+rypbGobvoesA1oalW5DXhHa32XUuq2wPufKqUmAFcCE4EhwNtKqTFa65hMfuTy+smJ85Od3kV9/oKWHhDEpcHut+HBmXD5Msib0ukhSin+76KJ1Lt93HhqUZfXPm1sNn96ZxfXPm70ZHhuySkcqmzE4/PT4Pbxf69u5eeLxrN0+bbmAAHwl8UzOKUoA5/WzeM1qhs8LF2+jcsf+ZRHrp3OuZPyQvwmhDjxyfKlLaJa3aSUKgDOB/7aavdFwLLA9jLg4lb7n9Vau7TW+4DdwCxipKrBTbKvEqzxPSduarOo2AuPzoeGii6Tfm3OcG4+bWSbQXftXX5yAQCbD9eQFm9l9ogMvnJyAVfNGsoN84bz1g8W8M0FI0hPaDuOY3R2EiaTanPulHgrSy+ZBMCvX5HlGYXojCxf2iLabRL3AT8BWg81ztFaHwEIvDa1CucDh1qlKw7sa0MptUQptUYptaa8vLz9x33m92uWfbKfr/hXEO+thsauH/jNLv1r2/f7PuhTHgrT47kp0H6R0W4eJ6UUo3OM+aHuvswosXxzfhFv//A0spI6n/PpmtnDuG7ucBrc3j7lSwhx4otakFBKXQCUaa3XBntIJ/s6NKBorR/TWs/QWs/Iyuo4wV5frdhSyh2vbOEK83s4lWJXY1nPB2WPgzN+0fK+fEef89HUc6m7EsfZE3LYvfQ8frZoPKOyO3aLbc1sUvj80W2PEkIMPNFsk5gHfFkptQhwAMlKqX8AR5VSeVrrI0qpPKDpKVwMFLY6vgAoiWJ+AWgMNPzG4+JXmem8bq3gY3cNybYeBqrM/R5Y4uCjP0J533sTzSxK56szCrlu3vBu01m6CSJt0pkUvih3WhBCDDxRK0lorW/XWhdorYdjNEi/q7W+FngFWBxIthhomuTkFeBKpZRdKVUEjAZWRyu/TexW4ytKUfVssBvVN9WuICbds9hg7rehYAaUbYW37oDtnaxqFySr2cTdX5nC+LzwjKI0SUlCCBGE/jCY7i7geaXUjcBB4HIArfUWpdTzwFbAC9wSi55NFpMRJOx4iNNGU0qNqwaSujuqlayxsHNFS2nil8e77RYbLWYlQUII0bOYPK201u8B7wW2jwNndpFuKbA0ahnrhNvnBzQ2vDhsyYCT487jwZ8ga1zb91v+A1OuCGcWe8VsUvi1MVajs3llhBACZMR1s5U7ythTXtdhv8vjw4Ebu/IQZzcm57vlnVtw+4Kcvju37dwvwU4EGGnmwDxQUpoQQnRHgkTA9U98zpn3vt9hv8vt5peWfwCQ0GqMxKMbHw3uxK1LElnjoHRzn/IZLs1BQhqvhRDdkCDRg/zDr3ON5R0AbK2mCH/34LvBncBsbdnOnQxH+1mQkJKEEGEX7eVLAYYPH87kyZOZOnUqM2Z0OsNGr0iQCPiO+T+cblrfYb/P62ne9rSquy+tLw3+5N9eC99eYwSJ6kPdjsCOFosECSEiIhbLlzZZuXIl69evJ5xz2EmQCLjV+m+etP2uw/7Ws746VcsDtc5TF3y7ROYoyBwNw+YZ7zf9q095DQeTkiAhRHsDdfnSSIp9X8x+4unkRMa6PbQvpLn8LXHUpf3Myp3FeUXn8ZtPf0OFs4LchNzgL1IwA9KKjGk6Zt8Unoz3klQ3if7q7tV3s70ivNPZj0sfx09n/bTHdAN5+VKlFOeccw5KKW666SaWLFnS4/0GQ4JEwF0ZxrTZ7VfW9XuMBYFKzWbWVW7n9MLTSXcYaUMOEgAFM40goXXn61BEiQQJIToayMuXfvzxxwwZMoSysjLOPvtsxo0bx4IFC3rMb08kSPTE6wTginwjGFQ5q9oEiZAVzIRNz0PNYUgpCFs2QyW9m0R/Fcwv/kgY6MuXNm1nZ2dzySWXsHr1agkS0eD21vLd7EwqzWYAXD5XH4NEoELrjxNhyfswZGq4shoSKUkI0dZAXr60vr4ev99PUlIS9fX1vPnmm/zqV7/q4zdikCDRg81qHyvjW8ZHePye5iBR6awM/YQ5k1q21z0VuyAhDddCtNHZ8qXnn39+yOdpvcyoz+fjhhtuaLN86V//+leGDBnSbbpQ9x89epRLLrkEAK/Xy9VXX825557bp++j+X7CcpYTmMXvb/Pe6/eSYE3AZrKFNj1H8wltMHw+7P8QqovDlMteZMMsQUKI1q666iquuuqq5vebN29m0qRJ3RzRtUWLFrFo0aIO+9uvO91VulD3jxgxgg0bNvQqrz2RLrA9sPraLsxz7fhrUUqRHpdORTALEHVm8X9hypXGxH9HNoYhl6GTLrBCdE2WL20hQaIHSrcMpnvuguf46rivApBmT+Now9FenlRB6lBj+9H5fc1ir1ik4VqILsnypS0kSHRBa83dK7ZT62xo3pdmT2veLkgqYHXpamrcNb27wLRrW7a9QQ7KCyNTIEh4fRIkhBBdkyDRhQa3j4ff24Pf1FLdlOpIbd4+Y+gZ+LWfQ7WHOju8Z2nD4IqnjO0jHacDibSmhmu/lCSEEN2QINEFlzfQYK1aqpviLC31k0UpRQCU1oUwh1N7w+YarzuWw4OzYeVve3+uEJml4Vr0M50NFhPh1ZvvWIJEF1xeYxG8ppLEteOvbfN5XkIeAKUNfQgSCZmQnN+yDvb7d0NjVe/PFwKTlCREP+JwODh+/LgEigjSWnP8+HEcDkfPiVuRLrBdcHmMkoRf+bDrjqNA0+xp2M12jtQd6duFzv1/8PzXW95vf7Vte0WEBJokkP+Toj8oKCiguLiY8vLyWGflhOZwOCgoCG2mBwkSXWiqbvKZfNjpOO+KUoq8hDyO1PcxSEy4CNKGQ9VBSMiG7cujEiQUTSWJiF9KiB5ZrVaKiopinQ3RCQkSXXB5fVxm+gBMjcQR32ma3ITc0NaV6Mr3AoNgXvsRrPsHeBrBGtn+2aq5JCFRQgjRNWmT6IKnsY57bY+A8uBQ5k7ThKUk0dq4ReBthNV/gbqy8J23E01lIwkRQojuSJDogsfVCIDTZMJusnWaJi8hj/LG8uAXH+rJsFPBlgRv/RL+ND085+yKtEkIIYIgQaILXpcxiM6pFAldVP00rSXR65HX7VlsMGyOse2uhdowVGV1oalNQktZQgjRDQkSXXC5aplcNJTVcQ7ieggSYWmXaHLGL6HoNGN707/Dd952lNQ3CSGCIEGiCw3OlvEKCdakTtNEJEjkTYHFr8CQ6bDx2fCdtx2JEUKIYEiQ6ILLVde8nRGf1WmasFc3tTb5cijdBMd2hf/ctMzdJG0SQojuSJDogtPdskRhV0EizhJHij0lvCWJJhMvBhRs/k/4z01LSUJGXAshuiNBogsuT0tJIjN1RJfpcuJzOFofgZJE8hBjbqfNL0Tk537zOImwn1kIcSKRINEFt7e+eTszKb/LdLkJuZGpbgKYeAkc22HM6xR2TdVNEiaEEF2TINEFt7dlHYkMR0aX6XLjcympL4lMJsZfCCjY9mrYTx1KSeI/XxTz03/HZgU9IURsSZDogtfXEiQSbAldpitKKaLaVc3xxl6sd92TpFwomAnbXgn7qZtnowoiSvzw+Q08t6aX62YIIQY0CRJdcHuNEdfnFpxOUXLXE4+NCLRX7KnaE5mMjL8QSjdC5f6wnlap0AfT1bm8PScSQpxQohYklFIOpdRqpdQGpdQWpdRvAvvTlVJvKaV2BV7TWh1zu1Jqt1Jqh1JqYbTyCuD1OwG4ZfI3mx+onRmZMhKAPdWRChIXGK/b/hvW0zaPkwihSaK02hnWPAgh+r9oliRcwBla65OAqcC5SqlTgNuAd7TWo4F3Au9RSk0ArgQmAucCDynVxUx7EeAJBAmHPaXbdNnx2SRZkyJXkkgfAblTYMuLYT1t06JDwQQJm9n4Z7LvWH0PKYUQJ5qoBQltaOpXag380cBFwLLA/mXAxYHti4BntdYurfU+YDcwK1r59fpdADhsyd2mU0pRlFLE/ur9kcvMpMvg8Fqo2Bu2UzYVjoIZJ1GYbkxLsqO0JmzXF0IMDFFtk1BKmZVS64Ey4C2t9WdAjtb6CEDgNTuQPB9o3VpaHNgXFR5tzOzqCGJdh6KUIvZWh+8B3sGky4zXzS+E/dTB1DY1lTpKpLpJiEEnqkFCa+3TWk8FCoBZSqlJ3STvrCGgwzNNKbVEKbVGKbUmnEsf+jCChN1s7zFtUUoR5Y3l1LYapR1WqYVQeEpYR1+rEKYKb/QY630flSAhxKATk95NWusq4D2MtoajSqk8gMBr02o7xUBhq8MKgA4DErTWj2mtZ2itZ2RldT59Rij8gfU8G31ubH7dbaN1kxEpRg+nfdX7+nz9Lk3+CpRtNeZzCgMV5BR/WmtqnUavpne2l7HuYGVYri+EGBii2bspSymVGtiOA84CtgOvAIsDyRYDLwe2XwGuVErZlVJFwGhgdaTzuaWkhvd3luP2e7AH2fOnKMXoIhvRKqcJgaaaR06Fxr4/qIMtSWwvraW60dP8/pKHPunztYUQA0c0SxJ5wEql1Ebgc4w2iVeBu4CzlVK7gLMD79FabwGeB7YCK4BbtNa+SGfS5fGy7UgNfpMPa5BBoiCpAKvJGtmSRGIWTA/E0i/+3ufTBTvievmmIygF50zIad7n8fn7fH0hxMBgidaFtNYbgWmd7D8OnNnFMUuBpRHOWht+TyNxVjN+5cPm77mqCcBisjAseRh7qyJYkgD48p+MHk6fPQan3ALm3v/1Na9M10OUqHV6SbRbOGdiLm9uNeaoOljRwMisxF5fWwgxcMiI63bcjZXE2cz4lB+LDi5IQBR6ODU55X+gphi2921wnam5JNF9lHB5/dgtZi6bns8/bpwNwJ6yum6PEUKcOCRItFPfWIYdD/HmWmwhBIkRKSMorivG5XNFMHfAmIWQVgSrHu7TaVrGSXSfzuXx4bCaUEpxUqExsPB/X93K1hIZMyHEYCBBop3GxmMM2fMMn8bFkUJDzwcEjEwdiV/7OVBzIIK5A0xmmH0THPoMitf24UTBTRXu9PpwWI2B7kkOKxkJNoorG1n0pw/7cG0hxEAhQaKdRk8173p2ALDO4Qj6uKZusFGpcpp6DdiS4LPelyaC6NkLgMvjx25p+WeS6IhaM5YQoh+QINGOx+vC2YuvZVjyMBQq8o3XAI5kmP41Yz6n6sO9OkVXE/zVubw0uFtme21dkgCYMSy95TNPxDubCSFiTIIEoP0tXTo9Phcm1fMo6/YcFgf5ifnRKUmAUeWk/b0uTXQ1VfikO95g5p1vN7/fVFzdPMEfwNJLJnHVLGOM4xMf7+/VtYUQA4cEiXZ8Pg9mf+++lpGpI6MXJNKGG8ubrnkSnNUhH97dVOH1bqOEsL20hhqnl0/3tiyo5LCaufPiyTisJu5esZ2fvxieEeBCiP5JggTgb1WS8Ppc+P3GvE3jkoaFdJ4RKSM4UH0Anz9K1TBzvwvuWljzt5APDWaq8NX7KgAYl5vUZr/ZpJg/2pgC5enPDoZ8bSHEwCFBgrajjr1+N95AkPjz6X8M6TxFKUW4/W4O1/WunSBkQ6bCiNPh04fAFdrkgsFMFV5WY3TnffU7p3b4bERWy5Kue8pl3IQQJyoJEoDWrUoSfjeewFoSiY60rg7p1MhUY5W6XVW7wpe5niz4CdSXwYrbenV4VyHC79ccr3eTmWjDYu74z+Sb80dw3qRcAJ76NMLdfoUQMSNBgrbVTT6ft7kkYe9hwaH2RqWOQqHYXrE9rPnr1vB5MPMbsO4fsP/joA9TPUwCW+P0UFnvJj3B1unnmYl2Hr72ZE4dlclngWopIcSJJ+QgoZRKiOYyotHR8qT0+t14tQelNRazNaSzxFvjmZw1mZUHV4Y7g907505IyIIP7wn6kK56NzWpavBQUe8mLb7zINHk5GFp7Citodbp6TadEGJg6jFIKKVMSqmrlVKvKaXKMKb3PqKU2qKU+r1SanTksxlZ/lbVTX6/B5/2YNMEtZZEe6fmn8rOyp00eIIfrd1n1jiYcwvseddY5jQI3fVuAqhscFPR4CYjsfsgMX1YGn4NG4tD72ElhOj/gilJrARGArcDuVrrQq11NjAfWAXcpZS6NoJ5jCqf9uLVHuzBLNnWiQnpE9BodlTuCHPOejDjRnCkwvu/Dyp5T1OFVzUGV5I4qcCYz+nl9YfZd6yeOpe32/RCiIElmDkWztJad6hL0FpXAC8ALyilQquX6Wda9/DZVVaFsruwWkMvRQCMzxgPwNbjW5mW3WFm9MhxJMPcb8O7d0LxGiiY0W3ynqYKr6hzU9nQdZtEk9R4G8My4nl+TTHPrykGYO9vF2Ey9e77E0L0Lz2WJJoChFLqfqXUm0qpN5RSv1NKTW2fZsBq9aRUeNEq+AWH2suKyyLDkcG249vClLkQzP4WxGfCE4vg6NZuk/Y0Vfit/9qA1pAS13P8n5yf0ub99tIIrfUthIi6UBqutwG/B+7HWIf6H0qpb0ckV1HWugus3+zBZfZhDWGa8NaUUozPGM+2ihgECXsinH8v+Fzw8Bz4oJuG7C6mCje3KwHYrT33UZg4pG2QeGl9lMaJCCEiLuggobV+RGv9ltZ6udb6HmAGcFPkshY9rRuuN2QcZluCu099g8enj2dP1Z7Iry3RmYkXw7l3Gdvv3gmlmztN1lTd1L6+qf3U4a1ngO3K1bOG8p0zRpGb7CDRbuHDXcdCz7cQol/qTRfYm5VS92CUKE7YlWc8qpf1TcCkzEn4tI8tx7aEMUchOOVb8JN9EJcGr90K/o5rUnfWcO336w4li2CCREq8lVvPGcuqn53JTQtGsO1IDZX17j7cgBCiv+jND+blGFVPBcD/C292YqN1dVOTvjSyTM+eDsAXZV/04Sx9FJ9ujJ84tApe+laHEkNnXWB9gTdTC1Ob99ktoQ2JmTsqA4BVrSYFFEIMXEEHCaXU80qp8Vrrg1rrx4ELgaWRy1r0dNZ46zX1viSR6khlZMpI1h7ty8pxYTD1ajj5etj4LGz6V5uPmgfTtYoSvkAxIj81rnmf3Rra74gpBanE2xE+8KcAACAASURBVMx8skeChBAnglCeAP8AnlNKbVJKPQk8D3T8CT4AddYN1NuLgXStTc+Zzvqy9dGbEbYzShkN2QUzjbmdGlqmz+hsVg6Pz/jrHJoR37wvmOqm1qxmE0WZCTy16gAfSduEEANeKA3Xr2itpwDXAO8CrwILI5WxqOokSmSbg1+6tDPTc6ZT56ljZ+XOPp2nz0xmuOA+aKyC134IgaClOmm3bipJZCW2LLoUanUTwIIxxjTi97wZ5QGFQoiwC2ZajjY/qbXWG7XWf9daP6m1LusszUDTWZvEIxf9p0/nnJFjDGb7vPTzPp0nLHInwbzvGsudfmRMf94yd1MLj894ZzW3/HWGWpIA+OHZYzipMJX9x+vx+k6IwqYQg1ZQ03Iopb6jlBraeqdSyqaUOkMptQxYHJnsRYduV2s215VMbsrQLlIHJzchl+HJw/ms9LM+nSdszrwDxn8Z3r8byra3Kkm0hIl1BysBGJqRwEmBxuvezE5iNZtYMn8EVQ0evjhY1eesCyFiJ5ggcS7gA/6plCpRSm1VSu0DdgFXAX/UWj8ZwTxGXPsHoSlMBaPZebNZU7oGj78fDEhXCs7/A9gS4T/fRPmMPLW+99X7KrBbTMwdmcGj157MDfOKGJ+X1MUJu7dgTCZWs+KtraXhyL0QIkaCmZbDqbV+SGs9DxgGnAlM01oP01p/U2u9PuK5jDB/u+omU5iW2Tgl7xQavA1sPtb5gLaoS8yCRb+H0o3EPX8FCn+bnl17j9UzIisRq9lEboqDX104odMFh4KR5LAyZ2Qm/15bzIrNEiiEGKiCaZP4g1LqOqXUdMCktT6itT6h6hA6lCTCFCRm5s5EoVhVsios5wuLSZfBaT/FcuBDFpvfbHPvbq8fR4hdXrtz04IRVDZ4uPkfa/l8vyxMJMRAFMwTYTdwCvBnjHUktiqlnlVK/UwpdbZSyt7D8f1e+6kowhUkUuwpTMiYwKoj/ShIKAWn345v2HxusrzaoQus1RS+IDF3ZEbz9qPv7+nwPQsh+r9gqpse0lrfrLWep7VOB84Hngkc+y1gm1JqgHeFbRckVPgelLPzZrOxfGN0FyHqiVLozLE4cLcpSXj9Gos5fB3VlFI89rWTAXh7WxlFty/n2dUHw3Z+IUTkhfw01FrvC4yZuFNrfSkwD/ht+LMWPR1LEuF7UM4ZMgev9vaPrrCtKYVCt2mT8Pr8vW6D6Mo5E3NZ/fMzm9/f9p9N+NtPECWE6Lf6/ETQWh/BKFkMWO27wJrCuIT39OzpxFniWL5vedjOGRZNQaLV89rj01gjsFhQdpKDv3y9ZRGkT2VeJyEGjLD8bNRa39tTGqVUoVJqpVJqW2B97O8F9qcrpd5SSu0KvKa1OuZ2pdRupdSOSFZptf9hq8LUJgFgM9s4vfB0lu9bzqHaQ2E7b1+1TPDXqiTh94e1uqm1syfksPYXZ5ESZ+UZqXISYsAIb91C97zArVrr8RgN4bcopSYAtwHvaK1HA+8E3hP47EpgIsZYjYeUCuNP/DbaRglzGNskAL4z7TsAvHvw3bCet0+UsaJExzaJyP2TyEi0c9n0At7cUsqxuhistSGECFnUgkSg6+wXge1ajOnG84GLgGWBZMuAiwPbFwHPaq1dWut9GL2sZkUkb+2KEuGORYVJhYxNG9uvgkQgRLQJj94IVTe1dvXsQjw+zb8C62ELIfq3aJYkmimlhgPTgM+AnEC7RlP7RnYgWT7Qun6mOLAv/NrP3eQP/4PyzKFnsq5sHcca+8nMqJ2VJCLQcN3eqOwk5ozI4MlP9uH0xHCGXCFEUKIeJJRSicALwPe11t2tbNfZk7pDtxil1BKl1Bql1Jry8vJe5qptkPDr8H8tZww9A43uN6UJRcfeTR6/bjO5X6R8+4xRHK1x8cr6kohfSwjRN1ENEkopK0aAeFpr3TTN6lGlVF7g8zygLLC/GChsdXgB0OGporV+TGs9Q2s9Iysrq1f5aqptsgc23L7wd9EckzaG4cnDeXP/m2E/d6900rvJ6/NjCeNguq7MHZnB2Jwk/r5qvwywE6Kfi1qQCEwn/jiwTWv9h1YfvULLLLKLgZdb7b9SKWVXShUBo4HVEclc4EEVF3h1+8O/PrNSioXDF/L50c853hj7LqCqqbqp1b7KBk/Eeje1v/bX5gxj8+Ea1h06oWZ4EeKEE82SxDzga8AZSqn1gT+LgLuAs5VSu4CzA+/RWm/BWP1uK7ACuEVrHZFKbH/gUZkQeGImp3WTuA8WDl+IX/t55+A7kblALzT9kn9lg1FI21LSXQ1g+FwyLZ8kh4XH3t8blesJIXonmr2bPtJaK631FK311MCf5Vrr41rrM7XWowOvFa2OWaq1Hqm1Hqu1fj2CuQNgvtsKwCkRaJMAGJU6iqKUIl7fF8FbCVa76qYvDhhrSayP0voPCXYL188rYsWWUraUVEflmkKI0MWkd1N/o/1Gw7Utfixf7DvIlydfH5HrKKVYVLSINUfXUFIX60bbtg3XafE2ANxRXEnuxlOLSHJYuP/tXVG7phAiNBIkaKly8Vrisd5RBZMujdi1Lhx5IQB/2/y3iF0jKO26wMbbjLEhf/zqSVHLQkqclRtPLeLNrUfZfFhKE0L0RxIkaBlSplAQ4eW68xPzmTtkLs/teI791fsjeq3uNZUkDI2BMQsXTBkS1VzccGoRyQ4L9765I6rXFUIER4IE4G/uhhn5nj0Ad867E7My85/d/+k5caQoo29T0703uH3YzCasER5M116yw8q3zxjFyh3lfLirt+NchBCRIkECaO4IGp0YQVZ8FqcVnMbLu1/G44vV+tdGdZPfr6msd3OszkWcLUJTY/Vg8dzhFKbH8bXHV7OnvC4meRBCdE6CBNAyWiBKUQK4bMxlVDgreK/4vahdszM+P0z7v7f499pi4qyxCRJ2i5lfXTARgB8+v0HWmxCiH5EgAdEuSAAwb8g8hiQM4e9b/h6bUcdKoZTG52/pzWQP4/rWoTp7Qg73Xn4SGw5V8eQn+2OWDyFEWxIkAN08wV/0woTZZOa6Sdexvnw9a46uidp1WxjVTb5WASra7RHtXTo9n+lDU/nfV7eyTAKFEP2CBAlausCqqJYl4JJRl5DhyOCxjY9F9bpA82A6X6uqHVuMg4RSin98YzZTC1O5e8V2Dlc1xjQ/QggJEkCrVRWiGyNwWBwsnriYVUdW8Z9d0e7ppDC1CxJWS+z/OcTbLPz5qmloDb94cZNMAChEjMX+qdAPtDyHohwlgCvGXgHAHZ/cwRv734jehQPjQbytgoQ9xiWJJoXp8fxo4VhW7ijnz+/ujnV2hBjU+sdTIcZaqpuiL8GawGNnG9VNv1v9O5xeZ5SubNytv/U0HLH4Arpww7zhXDAlj/vf2cU7247GOjtCDFoSJADdtOhQhEdbd2XOkDn8beHfKGss445P7ohOFUvgXlv3bnL1o5XilFL89tLJjM9L4vvPrWfn0dpYZ0mIQUmCBK2n5YidmbkzmZU7i+X7lkd1KnF/q4Dk9ERvcr9gJDusPHzNyVhMiise/ZTiyoZYZ0mIQUeCBDQvTRft3k3tPXTWQxQmFfKD934QhYbsQJtEq+oml7f/lCSaFKbH8/xNc/D6NP/z9BccqpBAIUQ0SZCgVe+mGAcJu9nOHXPuAIyG7Ff2vBK5iwWqm1pXbTX2o+qm1kbnJPG7r0xhY3E183+3ko92HYt1loQYNCRIADEZct2F2Xmz+feF/ybZlsxvPvkN2yu2R+hKgYbrVm0SR2tcEbpW3y2anMePzhkDwLf+sVbaKISIEgkS9J+SRJOx6WP57yX/JdWeyu0f3o7LF4GHd+BWvX5/81oSafHW8F8njL59xmg+vu0M4mxmrn/ic8pqo9UTTIjBS4IENE8o1z9ChCHdkc5v5v2G3VW7eXDdgxG4QlN1kx8FjMpO5MX/mReB64RXfmocjy+eSUl1I7OWvsPfP90f6ywJcUKTIAHEYhbYYJyafyqXj7mcJ7c8yRdHvwjvyZu6wPo0fg1njMtmeGZCeK8RIZMLUnh88QwAfvXyFl5YWxzjHAlx4pIgAWh//wwSALfOuJUhiUP4+Uc/p8ET/p49fq3xaR2rISK9dsa4HLb/37mcOiqTH/97A69ujPWa4UKcmCRItNIfn5MJ1gSWnrqUw3WH+d3nvwvjmVsG02mtMQ+0KAE4rGYe+/rJzBiWzvefXc+bW0pjnSUhTjgSJIjdBH/BOjnnZK6fdD0v7HqBZVuW4ddhGPSmWno3+TWYBmCQAGNCwMevm8Gk/BRueeYLnl9zKNZZEuKEIkGClvUkYj2YrjvfmfYd5ufP554193DxyxdTWt/XX81NJQljnWtT/731HiU5rCy7fhazitL5yb838pcP9sY6S0KcMCRItNF/n5QWk4UHznyA703/Hvuq93H9ius51tiHQWWtShJaG3MlDWQp8VaeuG4W50/JY+nybdz1+nZZBlWIMJAgQeuSRP9mUia+Mfkb3DX/Lo41HuOWd27pQ2O2cbceny9w7v5+9z2zWUz86cppXDN7KI+8v4cLH/iID3eVxzpbQgxoEiSApi6wA+XX9Pkjzuee0+5he8V2vr/y+70bbBe410aXF4B+spREn5lNijsvnsQdF05gS0kNX3t8NRc9+DF/fGsn6w5Wxjp7Qgw4J8ijoW9aZkIdGEEC4LTC0/jN3N/w6ZFP+d6736PS2bsHYFWjBxg4ATIYSimun1fEi/8zl9PHZrHhUBX3v7OLSx76hK89/lmssyfEgCJBAlrG0g0wF4+6mDvm3MGnRz7lqteuYk/VnhCODlQ3eY2qthOhuqm9aUPTeOK6mTx14yyevH4mAB/K5IBChESCBODv511gu/OVMV/hqfOewul1ctVrV/Hm/jeDOzAQFFTg3gdy76buKKWYPzqL08dmc+m0fArT42KdJSEGFAkSgKL/d4HtzpSsKTx3wXOMThvNre/fyvSnpvODlT/g48Mfd3NU0702BYmBee+hUErh71/rKgnR70mQAFqWVBi4X0dOQg5PLHyCq8ddjcfv4e2Db3Pz2zezdNVSatw1HQ9oLkkYTCdqUaIVpYjO0rBCnEAssc5Af+Bv7t0U44z0kc1s4/bZt3P77NupdlXz2MbH+PvWv/PsjmcZmjSUmbkzuWXqLWTFZ9EUHk706qbWTGrANj8JETNR++mslPqbUqpMKbW51b50pdRbSqldgde0Vp/drpTarZTaoZRaGNHM6f6xfGk4pdhT+PHMH/PEwieYkTODg7UHeWHXC1z2ymX8d89/adCBXk2DqLrJpFSbNb2FED2LZv3Kk8C57fbdBryjtR4NvBN4j1JqAnAlMDFwzENKKXOkMtZUBXEiPj5m5M7giXOfYMPXN/Ds+c+Sm5DLzz76GWfufpIXE1umBh8MJQmlFDIIW4jQRC1IaK0/ACra7b4IWBbYXgZc3Gr/s1prl9Z6H7AbmBXBvAEn1liB9kzKxMTMiTy96Glun3U7OZZE7ktPbS47ncj33sQkbRJChCzWLbU5WusjAIHX7MD+fKD1dJ7FgX0RcuJVN3XFarZy9firmZ1QiJeW6ibzIChKmKQkIUTIYh0kutLZE6vT/95KqSVKqTVKqTXl5b2dp2fwPTlMKDSKJIdRizcIYgQmhbRJCBGiWAeJo0qpPIDAa1lgfzFQ2CpdAdDp0mNa68e01jO01jOysrJ6lYnBUN3UnlIKv4L8lLjm9yc6Y5yEBAkhQhHrIPEKsDiwvRh4udX+K5VSdqVUETAaWB2pTOgBOHdTX5kCQwiHpNqBwXHnxjiJWOdCiIElml1g/wl8CoxVShUrpW4E7gLOVkrtAs4OvEdrvQV4HtgKrABu0Vr7IpU33dwmMXg0lRzGZicBUFrtjGV2osKk1CCsWBSib6I2mE5rfVUXH53ZRfqlwNLI5aj1tZq2Bk+YUIGSxDWnFHLIm8xlJxfEOksRJ20SQoRORlwDNM3dNHhihNHTRynirGZ+c9GkWGcnKmQwnRChi3WbRL/Q8tgYPFHCpEzGfQ+ih6YMphMidBIkoNW0HIOHap77dvA8NWUwnRChkyDBIO3dFKhuGkwlCRlMJ0ToJEjQ6rf0IGqUaBpdrvXgWWBBScO1ECGTIAGgB/aiQ72hlPFXP7iChBpMBSchwkKCRCuDYdRxE1MgIPoHUZBomnpE2iWECJ4ECcDP4HlQNjE1lSQG0b03rZkh7RJCBE+CBKCbnhqDqCTRVLXmH0SLPjeVJKRdQojgSZBoZTC1SbT8qh48QUI137MECSGCJUGCliqXwRMiWlU3RW5KrH6nKTBqDYerGmOcGyEGBgkSbQyer6O5umkQ/apuqm5ae6CSeXe9y9LXtnL9E6upc3ljmzEh+rHB81TsxiB6TjZTanCOk4CWUsRfPtzHyh3lvLz+cAxzJUT/JkGilUHUbo0p8Fc/mHp2NVU3pcZZ2+xfe6AyFtkRYkCQWWCBzPwz8bxZS+bUs2OdlahpKUkMnmJU0z372vWB/UKChBBdkpIEYLfE43QNI96WHOusRE1z7yb/YGq4Nl7dvralp/3HG9hwqCoGORKi/5MgweBsk2gecT0Iq5vc3rb37LCa+Pfa4lhkSYh+T4IEYLOYGJoeT5zNHOusRE3z3E2DcDCdx9f2V8EZ47J5ffORDtVQYmD6185/MXnZZLx+6bUWDhIkgPF5yXzwky8xd2RmrLMSNc2zwA6ikoRqLkm0rWK7YMoQjtW5eXd7WSyyJcLsD2v+AECDtyHGOTkxSJAYpJoG0w2uEdfGa/uSxFnjcyjKTODnL26i1umJQc5EOJlNRo2A2+eOcU5ODBIkBilTqOMkjm4BV20EcxR5zW0SgYbr+746lWU3zMJmMXHfV6dSXufij2/timUWRRiYlREkXD5XjHNyYpAgMUipUEoSr/8UHp4Lf54BZdsjnLPIaWmTMO55xvA0ThuTBcBJhalcPWsoyz7dz9aSmhjlUIRDU5Bwep0xzsmJQYLEIBX0ehKrHobPHjG260rhoVNg+U+g6lCEcxh+TW0STUHCam77z/8nC8eRGmfl+8+to7pBqp0GqqbqJqdPgkQ4SJAYpBRBrExXfdgIEgDzvg+37oSZN8LqR+FP0+DNX4Jz4Pzqbt0Fdorag61yN7zwDXDXA5ASb+XPV01j/7EGblz2OY3uwTOG5ERiUcYYYZdXqpvCQUZcD1IqmKnCnzgXqg7C0Llw5q/AZIbz74UZN8KnD8Anf4KNz8H0r4M9CUrWw/HdcNKVcNJVEJ/e9nxbXoTyHZAzCYrmgyMlgnfYUVN1U271Rl6x/xKe+KWxI2ssLPgxAHNHZXLflVO55ZkvuPzRT/jTldMYkZVIg9vL+zvKmTsyk5R4axdXCJ/Saidur5/UBCu2QInnWJ2LynoPqfFW7BYTWUl2lFJUN3pYe6ACvx/qXF6cHh8en5/C9HicHh85yQ6mDU2LeJ77C7PH6NXkDAT/7vj9GpNpEM3H0wsSJAap5obr9l1gP3kAlMl40FcdxA0cnHMTO/av4JS8U8iIy4CcCXDxQ0awePsO+OD3bc9RuhHe/jWkjzQCQfk2sCZAbUnbdClDIX8aDJkOQ6bBkKlG+qqD4HFCxigwha+w23zPnnbThK9dBnO+DdY4ABZNzuPRa0/mJy9s5Ix732fa0FQK0uL574YSEmxmvrlgBNfNHY5Sih2ltUwbmsqGQ1U88cl+zhibzTkTc6iod3P9E58DkJVkx+vXFKbFMSk/hZOHpTFxSAo2i3Fvmw9XU+v04rCaGJGVyHs7yrj1+Q14W43bUKrjoM84qxmzSfU4i21BWhwf/fQMfH7Nd/75BdfPK2Lm8PRujxnIzHXlYDXjdB7vNt1rG4/w0xc2cvXsoXxjfhHZSY4o5XBgkSAxSJm6Gkz35s+N1zdup9Jk4g+zv8pLq41f3Em2JJZMXsL0nOn4tZ/ROeNIuO5VqNgHh9fC2EVgi4fSzbD+aShZB5X7wWSFghlgscOpP4D6cjj8hRFMStbB1pdbru9IBWdgigxbolHqsCeC12WUVuqPQWI2ZAQCUFw6pBRCSgGk5BtpWmt6sjZUkFh/gAQa8bcfZFV9CN69ExYubd51zsRcphSk8ouXNvH2tjLWHTTydOroTO57excPvbenw8htMB48/Kvl/Wljsqhu9HC83sXhykZeWm8ESpvFRG6ygwa3l2N1HbtqFmUmcP284dQ6vZRUNZISZ6W02smY3CQsJoXVbOJgRQOHKxvJSrIze0Q6OckOvD5NdrKdeJuZ4spGrvnrZ1gCv5SP1jhZvqmU5ZtK2XHnudgtJ+bgUXugva2u4VjzPq01T3y8n5xkB3vK61iyYAT/WnuIOpeXxz7Yy5Of7Gd2UTr7j9ezYHQWp4/N5rWNJXxpXDYLJ+bisJ6Y31UwJEgMUqqrWWDjM6DB+AX2g+FjWVv6MQBTs6bi8rm4d+29zUntZjunF57OBSMuYHTRHDYd/oC91XuZlTuLk875X6ymlmoZrTVHG46yt3ovmWlDGDH8VCymwD+/hgoo+cKorqo9AhaHUYo4usX4U7nf+JVfsQ/iUuHYTtj1JnTWD96WBMl5RrBw1hgBwGQFdy1nAU/bRrLKfV3bY6ZcaVSfHfwUrnjKCDZAboqDvy6eSY3TwzOfHWR8XjKnjclie2kNz64+xOubj5Bgs3Dq6EzqXF5OHZXJsIwE3t1+lIp6N6eNyebcSbltLlVW42TtgUrWHqhk77F63F4/dS4vV84sJDXeysGKBqxmE1fPHtrnh3heShwXTM5j9f4KwKiuavLdf65jbG4y184eSnZyeH9BN7i9rN5XQZLDitPjIz3BRkFaHEmOln8PHp8fi0k1V3t2pqzGCQqO1bqpdXoormwkN8VBSpyVnUdr2V5aS0lVI5mJdhLsZuwWM3UuLwk+DVY4XncEgOc+P8hPX9jU5txvbi1l8+Earps7nJHZiWw5XM1rG49Q6/Ly9GcHefqzgwC8tL6EJIeFhRNzaXT7cHn9DEl1sHJHGcPSEzhlRDrTh6YxMT+FlLjIV0PGggSJQap5MF39cdjwHEy5wqjT8Hlg9s3sn3MTa1++CICXLnqJkakjAVhXto6Xd7/M1OypbD2+lRX7VvDG/jfanPvhDQ+TZE1iavZU/Pg5XHuYkroS3P6Wh7rD7GBs+lgmZExgZMpIsuKzyJ24iOz4bNaUrsHpczK08GqGJg8lzZ5GeWM5cZY44q3xmJUZk98P7jrjT3Wx0duq5rARZGpKjMZoRyoMnW3cU9pw9Ae/Z6QqYUWNMd5DX/wIjYlZxA9fAI5kWP0XeHAWnH4bzL4ZzMZ/+mSHlZtPG9mc93G5yfz6yxP59Zcndvrdnjys6/r/7GQH503O47zJeaH+lfWK3WrG6TF+CLwSKMWMy03ijS1HeWPLUZ5edYBHv3YyM9pVP20vrWFHaS2nj81ufviVVDXyyoYSDlc2svdYHcfr3EzOT2H9oSoa3D4K0uIorXFy4HjnI50L0+Pw+40fDKU1TixmEznJdgrT4hmWEc/Q9AQa3V5W769ge2ktVUH0MEu0W1BAg8eHz68xmxQLCo3PjtUbI+hfWtdSzTkiK4HMBHtz4Fw0OY9ZRca933HhRDx+Pw0uH//dUMKQ1DjS4q38a20xb209SnWjh2SHBafXj9fnR6H4aHdLaWVYRjzZSXbibBayEu1sPVLD8ToXHp+frCQ7afE2clMc1Dq9lNU6yUq047CayUl2kJfiIC81znhNcZCT7GjufVdZ78ZmMRFvM6OUwuX1saO0ltQ4Gz6tcVhN1Dm9mE2KEVmJPX5noZIgMUg1189//hco2WxU4RSdxm8TzPyzbDm8vByLycKrl7xKfmJ+83HTsqcxLXsaABePupgfz/wxnxz+hJ2VOxmaPJQJGRPYUbGDjw5/xMZjG6l0VpIdn835I84nMy6TopQiALYe38rW41t5effLvZo+IcmWRLojnRR7Cim2FOPVnkJK3nCSh08h2ZaMw+Kg3lOP2+emxl3D/jGzmVi8jcO1ZSwcOoRrvEd54LM/ctaRs7jzvLsxnfIteP02ePMXsOoRI8AUngJH1sOwecZ3lH9yxwb5aPE0gqsOrA7w+4zSkqldacPnBVeNUfIy28n1H8Hqqcfn1/z1o30ALLthFp/vr8BiMrF0+Va+8sinjMlJ5IIpQ3hn21EK0+N5daPxK9xqVpw6KpPclDj+ufpg82Wa2kiO1jipbPA0t7tYTIrZRemMyEpg7shMEh0WGlw+9h2rY9uRWo7Xu8hJduCwmEmNt3Kk2snBigbe2GKUvprMH53J+LxkspPs5KY4aHT7GJ2TRKPbx9EaJ0NS4xiXl0SS3dJcGnF5fZiU4sa//hSAI/UlOD0+4m1m8lPj+MX541k4MReTSbHhUBXFlY3NAQIgzmYmDjPJDivfXDDCuEGlmDsqE59fU+/2kuyworVGazCZFNUNHtYXV7H5cDWbD1dTXNmIy+tmZ2ktLq+PaUPTyEq0U9XopqLezZr9lVQ2uJlSkEJZrYtGj48PdpZT30lPukS7BatZURkIljazibQEK7VOLw2dpD99bBZPXj+rN/+yuqVOpPUEZsyYodesWRPrbAwI7356D9/buYzn/TmMP/A5pBXBnFuYvO1PzWnumn8X5484P6L58Pl9VDgrONpwlKP1RznacJREWyInZZ3EodpDHKw5yOG6w2TFZWE2mWnwNODVXqpd1VQ4K6hx1VDtrqbaVU2Nq4ZaT/ejwvO8XoaXzuDTgvUdPlsyZQmXjLqE/MMbUasfg9JN0Kpeu5nFAbmTjcCRNwVShxltJnvehcQcSB5iPKQ//yuYLJCQaexPzDZeE7KM7bg046G/Y7lRdabM4PfC8V2w7b9G431cmtGRwNPQeV6s8Ubbjc9lnKuTNcs/8U+k6NZ3mPP/3uXcibk88rWTmz+rcXp4bvUhXt10pMN06ZdNTRzldgAAC3JJREFULyA9wcryTaXNq/ldOi2fu78yJVBdZMJqVhypdpKT7MDcx15CtU4PhyoaGZGV0HMbgNbQWGlUQWq/0W5lSwSfm6teuJTNdj/Zfjvu8t9RXVHOd5Pe4xtLvm90pqgvM/6Oju2CL/4OjRUQnwnZE3CXbaGksZzX6/byUd1+JpoSKEoaxvCM8RRlTSIneSjKnoiuK0eZTMbfZ1yqUU0b6PjQE611h2q2GqeH0monJVWNlFY7Ka1xUtPopdbpIclhJSfZTkWDm8p6N3UuL/NHZ2ExKfxa49fQ6PaxaHIeuSm9qzpUSq3VWs/o9DMJEoPTylV/4Ls7nuBZTzoTi1semPOG5jM5fRx3n/cEKfbodlENB6/fS627lhp3DU6vkwRrAiZlIsGawIPLb+S1yq1MLZ3IB/nGyPE4Sxw3TLqBB9c/iEmZ8Gs/49PHc+bQM7EpM5luJ1mONLJsqWRiJr5kHdbqYijfabSjdDHTqAbW2e1YknJJc9WTVF9BvN+PFWj/KNVAtcmEH2Pg0i5HPJXZY6hPyafBXYvL68RptlJtsVBlUli0H7vJTpzWxPn9xPm9mE1WXCYLldpDjQIHCrvW7CvbwDabgwsn/JgHX7WzYM4nnD/6VC4fc3mHB9WWkmoOVTRy5vhstpTUMC43CYfVjNaaBrfxK91qVliaBiG668FsM0o1VQfx+N1QW4pqqMBddQCLyYrV70VZ44xOC2a7EWDNVqNKEIxSkSXOeLUlgauGij1vsadiB35XLTaTmXiTA7vfQ13DMfzKRKNS7PfWUGFS7LNasWhwaD9xWlNrMvF+fBwVZiPI1O/5AbfqtziU+xnZXh/zGhvZY7WyMTUbp89NjrOeLXZbc8vcIauFKnNLgIpD0UjLMzLB7yfR76fKZMICZPp85Hp9ZPl8xJusZFoSybQmkmFLxuKux+l1kmiNJ8mSSKI9mSR7MpWeesrdNdhsiTjsydgdKdhtyTgcqTgcqcTFZWB1pIItAawOqo+sx+NzYTZZMZkteD1ODpV8Rr324ddQ52ukzFNHXtpIzjn9f0P9LwMM8CChlDoXuB8wA3/VWt/VVVoJEsF7/7P7+Pb2x3m00c563cjIk7/Jlt2v84SnhP+ZcjPfmnZLrLMYdn/672L+dnwtP42fym8bN/D4mQ+Tk1zIsORhAByoOcAb+9/gjf1vsLNyZ5fnSbImUZBUQG5cFjnmODL94G2sYJty47c4cPj9HGsoY13dgQ7HWpSZOJONeJOFOEw4zDb+f3t3H1rVeQdw/Pu77zf3JjeJuXpNfI31Lb7Fl7p1tnuhE7Gw+kJdpa3dHNTBVlj7RxmDwso22BgbjLHCOtaBo8IGG92klK60BcEyW2fnKrMtE60sNtEkauLNPffl3PPbHznGWHMTddF71d8HDsl98txzfueG5HfO8zzneU4WB3C8iSejiwVjtMRb8NQjX87juA6Oe/lw3ngoTlO0iXw5T97NV2zKW9Uwj60dj9LsljlPmYMfv05L/Qwyje20Ns7j/Y/f4FjvETKJ6WSS05kWb6G+4HD4+GsU8LigLgcLvYTUI6zQHQpe9s91tIinRFAiqoT97UwwhCcQ1uHymCpxT3EFusJX3wHcFm0CBKdcwPGKRCWEU/RYfiHMP1MX6MiFOBkrMRAceyh1NBCm4JVojKaYn2jDQ1jVeg9z6qZx94z7yCRb6XP6OHH6MMd7DnFi8CTZUpZSuUBzYjq9udP05M/SX7rAULnAeZ2cJ/VD/ucRRkcS3kQ2htP89JG3rut44yWJmu6TEJEg8DywHugCDorIXlU9Wt3Ibn0XryK/GS8AAfjgxZGfZZKtVYrqxkqEE5RFyH2yH5rquat5Ec11l6aHn90wm13Ld/HEsic4VzhHJBChz+mj1+mlz+njTO4MuVKOs/mzdGW76Mp184+hHi4Uh5u4mmPNZBIZCm6BPC6rpq5i59KdDBQGyJayOK5DrpQj5+ZGvjquw8xglM50J6FACEVJRVMsaFpAIpwgHooTD8WJBqMjgw1GU1Xy5TyeekSCEUISuuwO4Tu7H+AtLk2hEvU8vnV+gD3uRzx74NJVZ8zzKPYewvvU3UVs8Bj5Tz2rElYoCcyPJJkZa2HAHWJVrJlMbArxUJxgOE6kLo0EIxQFiiWHUjlP0fW3coFEpJ5UvIWi61Ao5ci7OZziEHk3x5fTy1k9Yx2JcIJSuTTyOdVH6glIgJCEyCQytNW3EQ1Gr/hMNv1qPwe6T/O0PMMvUwkgwFSCvLj5L7z9ydu4nsuaaWvomNKBpx6nsqeYWT+z4kirdF2a9Nz1rJ078fLGpXKJ/nw//fl+yl6ZWChGtpglW8oyWBwkW8ySjCRpTbRS9IrkSw7F0hCF/ABOYXjLF7M4xSxOKUvezZGKNtHW2I7ruXhaJgDMyKwmVZdGgGQ0RSrSQEti6oTxXY+aThLAWuCYqh4HEJE/AJsASxL/p4FRV6BxhSfvfoZFzYsIBUIsbVlaxchunGRseNTRL5qGn6VIRsderlZEaI4Nd2gmI0nmpOaMu9+LHeNN0aaReYNuFhEhHqrcFt5U1wi5/9JRKPClnMOue39IYMXDfP3DV/jPR3txkmku4LFy3gPE84P0nj1Gt3Oa09kePrvsMVKJDINDPfQ4ffQXztHevoFMejGFcoFIIDLuENZq+c3ja3jnxFkenLufL5x6h/3df+erHTtIpuZc8bsMSpBZDbMm7djhYJhMIkMmkZm48i2i1pNEGzB6Jrku4DNViuW2siQ+DYAdA4Osn7ORlUser3JEN168dTWc3AvAuoZ5RIKRSdlvJBihJV6bC1a1zO6ED46wzp3F1kQjgSWbIBwnsGwbC5dtu6J+xt9GS/nbaGNdwdeKaQ0xHlwxfDe8oGELCxZvqXJEt7ZaTxJjXaZc1okiIruAXQCzZk3eFcHtrn3RFo6cOQYbd0DLgmqHc1PcP3cDjlfiK1NWUNeysNrh3BTbl32Dpvo2ti/cftPvcsztoaY7rkXkHuA5Vd3gv/4egKr+eKz61nFtjDHXbryO61qfKvwgMF9E5opIBNgO7K1yTMYYc8eo6eYmVXVF5EngbwwPgf2dqv67ymEZY8wdo6aTBICqvgq8Wu04jDHmTlTrzU3GGGOqyJKEMcaYiixJGGOMqciShDHGmIosSRhjjKmoph+mu1Yi0gtcOfWmMcaY8cxW1fRYP7itkoQxxpjJZc1NxhhjKrIkYYwxpiJLEsZUkYh8UUReqXYcxlRiScIYY0xFliSMuQoi8piIvCsih0XkBREJikhWRH4uIu+JyJsikvbrdorIARF5X0ReFpEmv/wuEXlDRP7lv2eev/ukiPxJRD4UkT3iL/cmIj8RkaP+fn5WpVM3dzhLEsZMQEQWAw8D61S1EygDjwIJ4D1VXQXsA77vv+X3wHdVdTlwZFT5HuB5VV0BfA7o9stXAk8BHUA7sE5EmoEtwBJ/Pz+6sWdpzNgsSRgzsfuB1cBBETnsv24HPOCPfp2XgHtFJAU0quo+v3w38HkRqQfaVPVlAFXNq2rOr/OuqnapqgccBuYAg0Ae+K2IbAUu1jXmprIkYczEBNitqp3+tlBVnxuj3ngPHY21FO9FhVHfl4GQqrrAWuDPwGbgtWuM2ZhJYUnCmIm9CTwkIlMBRKRZRGYz/PfzkF/nEWC/qg4A50TkPr98B7BPVQeBLhHZ7O8jKiJ1lQ4oIkkg5a+n8hTQeSNOzJiJ1PyiQ8ZUm6oeFZFngddFJACUgG8DQ8ASETkEDDDcbwHwNeDXfhI4Duz0y3cAL4jID/x9bBvnsPXAX0UkxvBdyNOTfFrGXBWblsOY6yQiWVVNVjsOY24ka24yxhhTkd1JGGOMqcjuJIwxxlRkScIYY0xFliSMMcZUZEnCGGNMRZYkjDHGVGRJwhhjTEX/A/YHBRbQHGwVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "\n",
    "errors, ns = stochastic_gradient(X, y,0.0001)\n",
    "plt.plot(ns,errors, label=\"$\\eta=0.0001$\")\n",
    "\n",
    "errors, ns = stochastic_gradient(X, y,0.0002)\n",
    "plt.plot(ns,errors, label=\"$\\eta=0.0002$\")\n",
    "\n",
    "errors, ns = stochastic_gradient(X, y,0.0005)\n",
    "plt.plot(ns,errors, label=\"$\\eta=0.0005$\")\n",
    "\n",
    "plt.xlabel(\"epochs\")\n",
    "plt.ylabel(\"$J(\\omega)$\")\n",
    "plt.xticks(ticks=[])\n",
    "\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot magnitude of $J(\\omega)$ during during gradient descent by different methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD1CAYAAAClSgmzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3SUZdr48e89JZX0EAIkpNB7RxFQWBBRXBuu+gqKvSzuWhZXfd1Vt7iva10bKFY8rm3lh7qKqFgQLEiRjvQAoYYESC8zc//+eGaSSTJJJsmUZHJ9zpkzM8885ZqUuebuSmuNEEII4Ykp2AEIIYRouyRJCCGEaJAkCSGEEA2SJCGEEKJBkiSEEEI0yBLsAHwpOTlZZ2ZmBjsMIYRoV9auXXtca93Z02shlSQyMzNZs2ZNsMMQQoh2RSm1r6HXpLpJCCFEgyRJCCGEaJAkCSGEEA2SJCGEEKJBkiSEEEI0SJKEEEKIBkmSEEII0SBJEi1QYa9g0Y5FFFcWBzsUIYTwK0kSLfB5zuc89MNDzPhoBquPrA52OEII4TeSJFpgf9F+TMqE2WTm+s+u58k1T2J32IMdlhBC+JwkiRY4UHSA1KhU3v/1+8zoM4PXtrzGop2Lgh2WEEL4nCSJFjhQdID0mHSirFE8cPoDDE8ZzvwN8ymtKg12aEII4VMhNcFfoOQW5TIpfRIASinuHHknV396Nf/e9m9uHHJjkKPzzicbD7M6pwCrWWExm7CYFBaTCYtZGY/NJuM1k/O1WvsprGZjX7PrsfPeeG4cZ3bbz2oyYTbXHGtSxs9OCNG2SZJoppKqEgrKC0iPSa/eNjxlOBPTJ/Lq5lf5TZ/fEB8RH8QIvbMx9ySL1uVis2vsDk2Vw4HWgY2hOgm5JyaTwuxMKhYPr7vvV/v1mmTmSlq1z1NzvCtxuSfGugmu1rbq556Tn3uyNJsk8YnQIkmimQ4UHQColSQAbh9+OzP+O4OXNr3E3aPvDkZozXLfef2577z+tbbZHZoquwO7Q2OzG4nDtc1m19gcDmyu1+yu15zb7dr5moMq571r35rXHcb+do3d4Xm/KlfSsrud03l8ld1Bpc1BSaUdm90ttjrXqY7NeX5HAJOfUtRLHO6lLPfE4qlk5inZ1STHBrY1IxHWLSVa3eN022apE7fouCRJNJMrSaTFpNXa3iuhFxf0vIC3f3mb87LOY2DywFqvl9vKibBEBCzOljCbFGaTOdhh+JzDUZNsjCRVO4l4THR1kmJDya/KlfDsDWyrTlx1EqPduL5rv9JKGzZHzX6uJO1+rPs2WwAzX2OJz5WI6iWZhqouze4ls8aTYO1SnXsCrXue+olQkp7vSJJopoZKEgC3DbuNHw//yNWfXs2fTv8TF/e+mBPlJ3ju5+d4f+f7PHHWE0zJmBLokDs8k0kRZlKEhVA/Da2NEpetkVJX3WTmSnDuJcPGEmTdxFedIN2SV93ruZ+7rErXKe3VTZy1YwxU3nNPeq7EUV19WC9J1S95uRJSmCsRVVd/NpwEPR7nlsA8H187+dWt4rSYVEDa9SRJNNOBogPEh8cTExZT77Uu0V149/x3+eO3f+SB7x/gq/1fsfbYWkqrSom0RPL+jvclSQifUMrVmQAirKFR+nM43EpKtUpStZOMvdZ+Do8lQk+JqaGkV2l31CrdeTpXeZUDm91W61z1kq1bLIFKeO5VjucOTuXRS4f6/ho+P2OIyy3K9ViKcEmMSOSFKS/w3M/P8crmVzi96+ncO+ZePt7zMa9ufpXjZcdJjkxu9Bp2h51nf36W8d3HMyp1VKtjPll+ku8Pfc93h74j0hLJvWPuxWKSX71oW0wmRbjJTHgI/Gk2lvA8lbqq6uxXO/nVLQnWr/6ssjsY0DXWL+8lBH4dgXWg6ABDOg9pdB+LycIdI+/g6oFXkxCegFKK6VnTeXnTy3yW8xkz+89s9PiXNr3EK5tf4ZsD37D4wsWtKlI+tfYpXt/yOg7tICYshqLKImwOGw+OfVC6oArhJ6GU8EKnkjYAqhxVHCk50mhJwl1iRGL1B3GvhF70S+zHx7s/bvSY1UdWM3/DfDJiM9h9ajcrD65scbzfH/qeVze/ytSMqbx53pusuHwFNw6+kUU7F/HChhdafF4hRMchSaIZDhcfxq7tXieJuqZnTWdz/mb2Fe7z+HpBeQH3fnsv6THp/Pu8f5MSmcLCrQtbdK3iymIe+v4hMmMz+du4vzG081DMJjO/G/47Lux5IfM2zGPRDplKRAjROEkSzdBYzyZvnJt1LgrFJ3s+qfea1po/rfwTJytO8vhZjxMXHsfMATNZdXgVvxT8Um/f7w9+z+xPZ3PhBxfyx+V/5OVNL/PzsZ9xaAcAT6x9gqOlR/nbuL/V6nqrlOLBMx5kRMoInlv/XIvehxCi45Ak0QzVYyQ6pTWxp2ddorswJnUMn+z5BF1nePMnez9hxcEV3DXqLvol9gPg0j6XEmWJYuEWozShtWb1kdVcs/Qabl52M4dLDtMjpgcb8jbw9LqnufrTqzn7/bO5f+X9vL/jfa4ecDXDUobVi8NqsjKk8xBZD0MI0aQQaFYJnANFBwg3h9M5qnOLzzE9ezoPfP8A6/PWMzxlOACnKk7x2OrHGJw8mCv6XlG9b2xYLJf0voR3fnmHEV1GsHjnYjYd30RyZDL/e9r/MqP3DMLMYdXnWHlwJZ/lfManez8lOy6bOcPmNBhHuDmcCnsFWmtpwBZCNEiSRDMcKDpAWqc0TKrlBbApGVN4et3T3Pn1nSyYuoA+CX14Zt0znKw4yQtTXqg34nlm/5m89ctb/PWHv9Ijpgd/Pv3PXNDzgnqjt+PC45iePZ3p2dMpqSpBoRod4R1hiUCjqXJUVScaIYSoS5JEM7imCG+NmLAYXp32Kjd+fiPXLr2W3w3/Hf/Z8R9m9p9J/6T+9fZPi0nj0TMfxWKyMDFtolfTZkRbo5vcJ8xkJIZye7kkCSFEgwLeJqGUMiulflZKfex8nqiU+kIptdN5n+C2731KqV1Kqe1KqXMCHas7rTUHiw/Wm7OpJbLjslk4bSExYTE8vOphOkd15rbhtzW4/zmZ5zC5x2SfzqvkKmVU2it9dk4hROgJRsP17cA2t+f3Al9qrXsDXzqfo5QaAFwBDASmAfOUUkGbfyC/PJ8yW1mrSxIuaTFpLJy2kIlpE/nbuL959e3fl1ylh3JbeUCvK4RoXwKaJJRSacB04GW3zRcCrsEAC4GL3La/o7Wu0FrvBXYBYwIVa12t7f7qSZfoLjw7+VnO6HaGz87prQizlCSEEE0LdEniX8AfAYfbti5a68MAzvsU5/buwAG3/XKd22pRSt2klFqjlFqTl5fnn6gx5myC+lOEt1fh5nDAaJNoLq01pypO1dtuc9jYmLexXvdeIUT7FbAkoZQ6HzimtV7r7SEettX79NFaL9Baj9Jaj+rcueVdU5tyvOw4AClRKU3s2T64kkSFvaJZx20+vplrP7uWCe9M4OVNL1cnhNKqUm7/+nZmLpnJioMrfB6vECI4Atm7aRxwgVLqPCACiFVKvQkcVUp11VofVkp1BY45988F3Ot20oBDAYy3loLyAsLN4URZooIVgk+FW5qXJEqqSvjrD39lyd4lJEYkMrbbWJ5e9zQ7TuzgjhF38Idv/sDWgq1YTBa+zf2WM9PO9Gf4QogACVhJQmt9n9Y6TWudidEg/ZXWehbwETDbudts4EPn44+AK5RS4UqpLKA38FOg4q2roLyg1oR97Z2rTaLC5l2S+GDXByzZu4QbBt/AJxd/wgtTXuD2EbezdO9Szvt/57Hr5C6emvgU47qN47uD3/kzdCFEALWFcRKPAO8ppa4H9gO/AdBab1FKvQdsBWzAHK21PVhB5pflkxSRFKzL+1x17yYv2yT2nNxDbFgsvx/+++pEecPgG+gZ15PXtrzGXSPvYljKMI6WHmV57nL2F+6nR2wPv8UvhAiMoCQJrfU3wDfOx/nA5Ab2exh4OGCBNaKgvKBV03G0Nc3t3ZRTmENmXGa9ktSkHpOY1GNS9fNx3cYBsPLgSq6MvdJH0QohgkUm+PNSfnk+iRGJwQ7DZ5pbkth7ai9ZsVlN7tcjtgfpMel8d0iqnIQIBZIkvKC1pqC8IKSqm5oz4rq4spi8sjwy4zK9Ove4buNYfWS1jMEQIgRIkvBCYWUhNoctpEoS1eMkvBhxnVOYA+BVSQJgXPdxlNnKWHdsXYvjE0K0DZIkvFBQXgBAUmTolCRc1U3edIHde2ovAFlx3iWJMaljsJgsfH/w++ptMsBOiPZJkoQX8svyAUKqJGFSJsJMYV4nCbMyez0lSZQ1ipEpI1l5aCXHy47zj1X/YPS/R3P5x5fz8qaXq5OOEKLtkyThBVdJIpSSBBgD6rxJEjmFOaTFpGE1W70+9xndz2DniZ2c9//O473t7/GrHr/Coiw8ve5pLvjgAr7a/1VrQhdCBEhbGCfR5uWXGyWJUKpuAqNdwts2iczYzGade3KPySzYuIAz087ktmG3VY+ZOFJyhNmfzua9HUbiEEK0bZIkvFBQXoBCER8eH+xQfCrcHN5kDyS7w86+U/uqxz94KyM2gx+v/LHe9tToVM7NOpfXt7xePYpdCNF2SXWTFwrKCkiISMBiCq2cGm4Ob3KcxOGSw1Q6KptdkmjMuVnnYtd2vsj5wmfnFEL4hyQJL4TaQDoXb0oSrkZmb8dIeKNPQh96xvVkyd4lPjunEMI/JEl4IVSrRSIsEU2WJKrHSHjZ/dUbSimmZU1j3bF1HCk54rPzCiF8T5KEF0JttLVLmDmsyVlg957aS2xYLAnhCY3u11znZZ0HwGc5n/n0vEII35Ik4YX8snwSI0OwJGGOaLILbE5hDllxWT6fIr1HbA8GJg2UKich2jhJEk2osFdQXFUcktVN4eamx0nsPbXXp43W7s7NOpet+VvJOZXjl/MLIVpPkkQTTpSfAAjJ6qamkkRxZTHHy477tNHa3TmZ52AxWbjh8xtYunepTN0hRBskSaIJoTglh0tTI6790WjtLjU6ldfOeY3EiETu/vZurv3sWg4UHfDLtYQQLSNJogmhOtoanCWJRhqutxdsB7yf/bUlhqUM4+3pb/Pg2AfZcWIHv132W4oqi/x2PSFE80iScCnYA2Un6m0O6ZJEI9VN2wu288SaJ8iKy/L7MqRmk5lL+1zK05OeJrcol/tW3IdDO/x6TSGEdyRJuDwzHOaNrbc5VCf3A6N3U6Wjst4H8r7Cfdz8xc1EWaN4YcoLARtpPjp1NHePvpvlucuZt35eQK4phGicJAl3RYfrbSooLyDSEkmUNSoIAfmXpzUl8krzuOnzm3BoBwumLqBbp24Bjel/+v0PF/W6iBc3vsiyfcsCem0hRH2SJJoQqlNygOclTBfvWsyhkkPMnzKf7LjsgMeklOJPp/+JQUmDuH/l/ew5tSfgMQghakiSaEJBWWiOtgbPS5ieqjhFpCWSgckDgxUW4eZwnpr0FBGWCO74+g6KK4uDFosQHZ0kiSYUlBeE5GhrqEkS7tVNJVUldLJ2ClZI1VKjU3nszMfYX7ifP3/3ZxlDIUSQSJJoQn55fsiXJNyTRHFVMdHW6GCFVMuYrmO4c+SdLNu/jCfWPCE9noQIgtBaIMHHHNrBifITIdsm0VCSaAslCZerB1zNgaIDLNy6kH1F+3hkwiNtJokJ0RFISaIRhRWF2LU9JAfSgTHiGmonidKq0jb1IayU4v7T7ufeMfeyIncFs5bM4mDxwWCHJUSHIUmiEa7R1qFakogwG72b3Eddt6XqJhelFDP7z2T+lPkcLTnKfSvukzYKIQJEkkQjXAPpQrVNwjVOwn3hoZLKEjqFtZ3qJndju43l9hG38/Oxn1l5cGWwwxGiQ5Ak0YhQnpIDakoS7uMk2mJJwt0lvS+he6fuPPvzs9KQLUQASJJoRHV1U6h2gXW2SbhKElrrNtMFtiFWs5XfDvst2wq2yYhsIQJAkkQj8svyMSsz8eHxwQ7FL6p7NznbJMrt5di1vU2XJACmZ00nOy6b59c/j91hD3Y4QoQ0SRKNcE3JYVKh+WOq2wW2pKoEoM0nCbPJzJxhc9hzag+f7P0k2OEIEdJC89PPR/LL8kO2+yu03yQBMCVjCv0T+/PihhexOWzBDkeIkCVJohH5Zc7R1lVlwQ7FL6wmKwpVnSSKq4w5ktpym4SLSZm4eejN7C/az6d7Pw12OEKELEkSjcgvzydJm+DhVPgl9Ko1lFJEWCJqShKVRkmirXaBrWtS+iR6J/TmpU0vSduEEH4SsCShlIpQSv2klNqglNqilPqLc3uiUuoLpdRO532C2zH3KaV2KaW2K6XOCVSsYPT0yS/LJ6nSWYrYviSQlw+YMHNY9SywrpJEe6huAmdpYsjN7D21ly/2fRHscIQISYEsSVQAv9JaDwWGAdOUUqcD9wJfaq17A186n6OUGgBcAQwEpgHzlFLmQAVbVFVEpaOSJFfViyM0++SHm8OpdBjjJNpTm4TL2Rlnkx2XzYsbX5RxE0L4QcCShDa4FgawOm8auBBY6Ny+ELjI+fhC4B2tdYXWei+wCxgTqHhdA+mSLM4V6UK0cTTCHFFdkmiPScKkTNw05CZ2ndzF4p2La62NIYRovYC2SSilzEqp9cAx4Aut9Sqgi9b6MIDzPsW5e3fggNvhuc5tdc95k1JqjVJqTV5ens9irU4SWhkbtB0qiuDATz67RlsQZg5rlw3X7qZlTiM7LpuHfniI0946jfMXn89ff/gr+wv3Bzs0Idq9gE4VrrW2A8OUUvHAYqXUoEZ2V55O4eGcC4AFAKNGjfLZrG+u0dZJdmcVxuZFsH8VFObCH/dCVGiMwo4wR9TqAmtRluquse2F2WTmjXPfYNXhVew6uYsdJ3bw4a4PWbRzEedknMMNQ26gT0KfYIcpRLsUlPUktNYnlVLfYLQ1HFVKddVaH1ZKdcUoZYBRckh3OywNOBSoGKtLEraaeY0ozDXuy06ETJKoVZKoLCY6LBqlPOXnti0uPI6pmVOZylQAjpcd542tb/De9vf4NOdTpmVO47fDfktWXFaQIxWifQlYklBKdQaqnAkiEpgC/BP4CJgNPOK8/9B5yEfAW0qpJ4FuQG8gYHU9x8uOY1Im4is8jJEoOQ5JPQMVil9FWCKqq5na+rxNzZEcmcxdI+/i+kHXs3DLQt7c9iaf7/ucOcPmcNOQm/x23aqqKnJzcykvl7YR0fZERESQlpaG1Wr1+phAliS6AgudPZRMwHta64+VUj8A7ymlrgf2A78B0FpvUUq9B2wFbMAcZ3VVQBSUF5AYkYi5orD+iyW+a/sItnBzeHWpqbiqmChrVJAj8q248Dh+P+L3zOw/kz8s/wPv/vKuX5NEbm4uMTExZGZmtssSmQhdWmvy8/PJzc0lK8v7EnXAkoTWeiMw3MP2fGByA8c8DDzs59A8qh5tnV8IUUlgq4TKIuPFEEsSruqm0qrSkClJ1JUUmcTQzkPZmLcRrbXfPsDLy8slQYg2SSlFUlISze3gIyOuG5Bfnk+SQ0PBbkg/DUbOrnmx5HjwAvMx9yTR1teSaK2E8ASqHFXVXX39RRKEaKta8rcpSaIBx0vzSMr9GYoOQ3isUZpw+frvwQvMx2pNyxFCbRKeJEQYg/lPVJwIciRCtB+SJDzQWhslCbuzCSQirnaSAKgsDXxgflB3Wo6QLkm4kkS5JAkhvCVJwoPiqmIqHVUkuyeJ6OTaOxUdDnxgfhBhjqhevjTkSxLhRpI4WXEyyJEI0X5IkvCgem3rxkoShQcDHJV/hJnDsGkblfZKymxlIV2SiI8wVhgsKC8IciTtx9KlS+nbty+9evXikUceafZ+zd0OkJmZyeDBgxk2bBijRo1qNL6cnBwGDWpsTG59J0+eZN68eT4779GjR7nyyivJzs5m5MiRjB07lsWLF1e/bjabGTZsGAMHDmTo0KE8+eSTONrRXHCSJDyoN9o6IhYiE2rvVBiwcX1+FWGOAGo+OEM5SSRGGAMgT5ZLScIbdrudOXPm8Omnn7J161befvtttm7d6vV+zd3u7uuvv2b9+vWsWbPG5+/LmyThLa01F110EWeeeSZ79uxh7dq1vPPOO+Tm5lbvExkZyfr169myZQtffPEFS5Ys4S9/+YtPrh8IkiQ8OF5m9F6q3SbhrG4aPsu4D5GSRLjFmILDlRjby1oSLRFlicJqslJQEfoliX/+85/cdNNNXHTRRWRlZfH44483+xw//fQTvXr1Ijs7m7CwMK644go+/PBDr/dr7vaWstlszJ49myFDhnDppZdSWlrTXnjRRRcxcuRIBg4cyIIFCwC499572b17N8OGDePuu+8G4I033mDIkCEMHTqUq666CjCS34033sjAgQOZOnUqZWX1B9Z+9dVXhIWFccstt1Rvy8jI4He/+53HWFNSUliwYAHPPfccWvtsFiG/Csq0HG1d9ZQctdokkuD2DRDbHbZ9HDIlCdc8Ta73HMolCaUUCREJAStJ/OW/W9h6yMNgzFYY0C2WB389sMn9Nm3ahNlsZtGiRezZs4cZM2Ywd+7c6tcnTJhAUVFRveMef/xxpkyZAsDBgwdJT6+ZGSctLY1Vq1bVO6ah/Zq73UUpxdSpU1FKcfPNN3PTTY0Pfty+fTuvvPIK48aN47rrrmPevHnV7/XVV18lMTGRsrIyRo8ezYwZM3jkkUfYvHkz69evB2DLli08/PDDfPfddyQnJ1NQUEBhYSE7d+7k7bff5qWXXuKyyy5j0aJFzJo1q9a1t2zZwogRIxqNr67s7GwcDgfHjh2jS5cuzTo2GCRJeJBfno8JRYKr3jAizrhPyDTuY7tDYWg0XLuShKu6KZQbrsFovO4IvZs2btzI4sWLMZvNmM1mEhNrzzW2YsWKJs/h6Zuup372De3X3O0u3333Hd26dePYsWOcffbZ9OvXjzPPPLPBONPT0xk3bhwAs2bN4plnnqlOEs8880x1+8CBAwfYuXMnqamptY7/6quvuPTSS0lONmoLEhMTKSwsJCsri2HDhgEwcuRIcnJyGozBZc6cOaxcuZKwsDBWr17d4H7tpRQBkiQ8yi/LJ8ESRfUKR5bI2jvEdg2Z6iZXm0RHKEmA0Q02UOMkvPnG7w9VVVUcP36cnj2N+cU2btzI4MGDa+3jTUkiLS2NAwdqZuvPzc2lW7du9Y5paL/mbndxPU5JSeHiiy/mp59+ajRJ1E1crufffPMNy5Yt44cffiAqKoqJEyd6nFOroRH44eE1syGbzWaP1U0DBw5k0aJF1c+ff/55jh8/3miD+549ezCbzaSkpDS4T1sibRJ1ORzGGAlTWM22TnV+mbHdQqYLbJjZeJ8doeEaOkZJYvv27fTv37/6+fr16xk6dGitfVasWMH69evr3VwJAmD06NHs3LmTvXv3UllZyTvvvMMFF1xQ73oN7dfc7QAlJSXVyaukpITPP/+8upfR5MmTOXiw/pez/fv388MPPwDw9ttvM378eABOnTpFQkICUVFR/PLLL/z4448AxMTE1EqQkydP5r333iM/3/iiVFDgfZvVr371K8rLy5k/f371Nvc2kbry8vK45ZZbuO2229rNyHxJEnVVFhvzNmkzhMXA/x6uPy14TDcoPgb2quDE6EMRltq9m0K+uimAJYlg2bRpE0OGDKl+7ilJeMNisfDcc89xzjnn0L9/fy677DIGDjRKR+eddx6HDh1qdL/mbgejO+n48eMZOnQoY8aMYfr06UybNg2Hw8GuXbvqVZsB9O/fn4ULFzJkyBAKCgq49dZbAZg2bRo2m40hQ4bw5z//mdNPPx2ApKQkxo0bx6BBg7j77rsZOHAg999/P2eddRZDhw7lrrvu8vpnpJTigw8+YPny5WRlZTFmzBhmz57NP//5z+p9ysrKqrvATpkyhalTp/Lggw82+/cRLKo91Y01ZdSoUbrFXeYecrY73LmFc764nhEVFfzf8RNGY3Vda1+H/94Od2yG+PT6r7cjG/I2MGvJLMZ2HcsPh3/gu//5jtiw2GCH5TfzN8xn3vp5rLtqHVaT99Mle2vbtm21vsW3Bb169WLTpk1ERkY2vXMbtXnzZl599VWefPLJYIfS7nn6G1VKrdVae6wjk5JEHbq80Khu0goamjY71rmKagj0cKo3TsIS2tVNieHGN9FTFaeCHElgnDx5krCwsHadIAAGDRokCSJIJEnUUVKWT4W9gmSHAksDy3jGdDXu960MXGB+Ut0FtjyfSEskZpO5iSPat4426jo+Pt7jADghvCVJoo7jrjESDgc46+vriXX2xPjyr1DWvkfvupLEifITId9oDTLqWojmkiRRR75zNG7S0W1G47QnkQnQ1eg/zfEdAYrMP1wjru3aHvKN1gDx4c6SRAcYdS2EL0iSAHBrvM93do9MstuNBYc8UQpmvOw8oIF92glXmwSEfvdXqJkuXEoSQnhHkgTUThKVRoNm9ZQcDYnPAGVqOJG0E65xEhD63V/BWPMaZE0JIbzV7CShlIpWSoVs66ZZa7pbYoi3NzGVryUM4nu0+5KExWTBooyB9x2hJGE1WYkNiw35sRJC+EqTSUIpZVJKXamU+kQpdQz4BTislNqilHpMKdXb/2H6W01J4rLkESztebV385Uk9mz3JQmoKU2E8gyw7hIiQn/UtRC+4k1J4mugJ3AfkKq1TtdapwATgB+BR5RSsxo7Qbtit4HD5t2+ST0hf0+t6qr2yDXqOsrSwLiQEJMQHvqjroXwFW++ME/RWtebf0JrXQAsAhYppXw/dDWQ3D/kHVVQ5Zx75bf1p0WuJbEnVBZBSV79+Z3aEVc32I5SkoiPiOdQcfsfCClEIDRZknAlCKXU00qpz5VSnymlHlVKDau7T/vlliTsziRhskJKv8YPSzJm2Wzv7RKuJNER2iTAGCshvZu809aXL21Ia1afa8mSqN7GcMYZZ3h9fFtZFrU5DdfbgMeAp4FjwJtKqdt8HlGwOaqgqgysXkxjUJ0kdvk3Jj+rLkl0gN5NYIyVKKgoaFdz+gdDe16+1JdLlLaUpxi+//57r45tS8uiep0ktNYvaK2/0Fov0Vo/DowCbvZ5RMHg/k5sj/EAACAASURBVGFht4G9Ety6hjYoPsPYL0QG1HWkkoTNYaOkqiTYofhNR1m+tKSkhOnTpzN06FAGDRrEu+++C3heovTJJ59k0KBBDBo0iH/961/V52ju0qWelkT1FIenGDp16tTodV3a0rKozV50SCl1C9ALiAF8uzZjW+CoAltlw/M2uTOZIalX+08SHbAkAXCi4oR/22E+vReObPLtOVMHw7kNV/u4dJTlS5cuXUq3bt345JNPAGMNCaDeEqVr167ltddeY9WqVWitOe200zjrrLMICwtr9tKlnpZE/eabb+rFcdppp9WKwZ2nJVPrvt5WlkVtycp0S4CzgUuA//NZJEHllnnLT0HZCe9KEgDJfeDIRv+EFSAdrU3CNer6RPkJ0mPa91TvDekoy5cOHjyYuXPncs8993D++eczYcIEj/utXLmSiy++mOho42/8kksuYcWKFSilmr10qaclUT3FceJEwz3oPC2Z2phgLovqdZJQSr0HPKi13ga8opR6DfgZ+NjnUQXTt48Z952baLR2Se4D2z4CW4V3pY82qDpJhHWQJBHunJqjws+N11584/eHjrR8aZ8+fVi7di1LlizhvvvuY+rUqTzwwAP19mvow7O5S5c2tCSqpziuvvpqj9ds7LoubWlZ1OY0XL8JvKuU2qSUeh14D/B9U3owePoDUl7+aJL7gHZAwR7fxhRAHa26yVWSCNXpwjvS8qWHDh0iKiqKWbNmMXfuXNatWwfUX6L0zDPP5IMPPqC0tJSSkhIWL17MhAkTmr10aUNLonqKo24M7pq6bltaFtXrkoTW+iPgI6XUEGAYRoJZ4tNogsZDkqiqv+i5R8nOAed52yGlba1I5i3XYLqOVt0Uqt1gPS1fOn369Gafx32ZUbvdznXXXVdr+dKXX36Zbt26Nbpfc7cfPXqUiy++GACbzcaVV17Z6PKlmzZt4u6778ZkMmG1Wqs/VN2XKD333HN57LHHuOaaaxgzZgwAN9xwA8OHDweoXrrUbDYzfPhwHnrooQZ/JtOmTeOFF15gyJAh9O3bt3pJVE9xeIrBxX3JVNd1X3/99erXXcui3nnnnTz66KN07tyZ6Ohoj8uiVlVVYbFYuOqqq5q19Kq3mly+VCmldBM7ebNPILR4+dKqMng4tfa22O5wlxeLtVSWwD+6waT74aw/Nv/abcDDPz7MO9vfYfXM1dUJI5RprRn15ihmDpjJXSN9+08ly5f6hyxf6jv+WL70a6XU75RSPeqcNEwp9Sul1EJgdosjbgs85TdbhXfHhkUbXWGPbfNtTAEUbY0mzBRWXe0U6pRSxEfEh2xJwp0sXypay5vqpmnAdcDbSqks4CQQiZFgPgee0lrX7+PV3nmbJMDolnhonZFsfFwfGAhX9LuCUamjfF6X2ZYlRiR2iEn+ZPlS0VpNJgmtdTkwD5jnnKMpGSjTWofQ1zBPJQkv2yQAek2GXz6G4zuhcx/fhRUgqdGppEanNr1jCIkPj5dJ/oTwgjdThT+plLpGKTUCMGmtD7ckQSil0pVSXyultjmnGb/duT1RKfWFUmqn8z7B7Zj7lFK7lFLblVLnNPeaXvNU3dRjrPfHpxuNVxz62TfxCL+T6cKF8I43bRK7gNOBZzHWkdiqlHpHKfW/SqmzlVLeVmTbgD9orfs7zzdHKTUAuBf4UmvdG/jS+Rzna1cAAzGqvOYFbLGj5D5wUTPmfUnuA9YoSRLtSPdO3TlUfIiy5pQYheiAvJkFdp7W+hat9TitdSIwHXjLeeytwDZvvuU7SyDrnI+LMCYM7A5cCCx07rYQuMj5+ELgHa11hdZ6L0ayGtOsd+e1OiWJtDHGqnPeMlsgdYgkiXZkeMpwbNrG5uObgx2KEG1as5cv1Vrv1Vp/pLX+u9b6EmAc8I/mnEMplQkMB1YBXbTWh53nPgy4hgt2Bw64HZbr3Fb3XDcppdYopdbk5eU19+14ZmrB0t/dhhvTc9i9XLBIBNXQzkNRKNYdXRfsUIRo01rwaVib84P9LW/3V0p1wlis6A6tdWMTBHrqalOv8UBrvUBrPUprPapz587ehlH3JHWu3IJarW7DjXUo2vlkfx1FXHgcvRJ6se6YJAkhGtPqJAGgtX7Cm/2cvaMWAf/WWv8/5+ajSqmuzte7YqxVAUbJwX32tTQgMMuJmVqYJECqnNqRESkjWH9sPTZvl6sVogPySZLwhjI64b8CbNNau4+K+YiawXizgQ/dtl+hlAp3js/oDfzkn+h8UJJI6gVhnSRJtCPDU4ZTaitl54mdwQ5FiDYrYEkCo+3iKuBXSqn1ztt5wCPA2UqpnRhTkD8CoLXegjGJ4FZgKTBHa233S2R1q5taUpIwmaDrMEkS7cjILiMBpMqpEe11+VJPvFk6tKF9mrukqbcr43l73qAuZaq1DpnbyJEjdYuUntD6wdia22f3t+w8n92v9V87a11V3rLjRcCd/Z+z9V1f3+Wz823dutVn5wo2m82ms7Oz9e7du3VFRYUeMmSI3rJli9f7NXe7S0ZGhs7LywvkW23S3r179cCBA32+vzf7ORwOffrpp+v58+dXb8vJydHPPPNM9fPo6Ojqx0ePHtWTJ0/WDzzwgMfzefobBdboBj5XA1mSaMN8UN0ExqA6ewUclG+m7cXwlOH8fOznkFvvuqMsX5qTk0O/fv244YYbGDRoEDNnzmTZsmWMGzeO3r1789NPNTXUrqVDc3Jy6N+/v8flSd2XF63LZrMxe/ZshgwZwqWXXlo9dben5Uw9LV3a0HKljS2VCsFfyrQlK9OFvpZUN0HNKO1P/wi3NL3ylwi+ESkjWLJ3CbnFuT5fpe6fP/2TXwp+8ek5+yX2454x9zS5X0dZvhRg165d/Oc//2HBggWMHj2at956i5UrV/LRRx/xj3/8gw8++KDeMY0tT9qQ7du388orrzBu3Diuu+465s2bx9y5cz0uZ1p3+dTGlittKpZgL2UqSQJ80wUWIDrJaMA+shHKCyEitvWxCb8a0cX451t3dF1ILWXaUZYvBcjKyqpeeW/gwIFMnjwZpRSDBw+utexo3WMaWp60Ienp6YwbNw6AWbNm8cwzzzB37lyPy5mmptaeC62x5UqbG0uglzKVJOFJS0sSAOc/BQt/Dfu+g77n+i4m4Rc943sSExbDz8d+5sJeF/r03N584/eHjrR8KdReatRkMlU/N5lM2Gyeuzc3tDypy/PPP89LL70EwJIlxtpqdROkUqrB5Uzr0o0sV9pULMFeylTaJNy51lNozRRRaWPAEgF7lvsmJuFXJmVieMpw1h5dG+xQfKYjLV/qL3PmzKn+mbiS1v79+/nhhx8AePvttxk/fnyDy5nWXbq0ucukugv2UqZSkoCa6iZrpNHw3JrBVdYIo21i91e+iU343dDOQ/k291uKKouICYsJdjit1pGWLw2k/v37s3DhQm6++WZ69+7Nrbfeitls9ricqaelSxtbrrQxwV7KtMnlS9uTFi9fWpIPj2UbS5YWHoQzfg9T/9byQH6YB5/dB7dvgITMlp9HBMS3ud8y58s5vHbOa4xKbV2/fFm+1D9k+VLf8cfypR2AM1FGOpeyKM1v3en6OCfF3fF5684jAqJ/ovEP4+ueSG2BLF8qWkuSBNRUNyX1Mu5P5LTufEk9Iak37FjauvOIgOgc1ZmkiCS2FbTfdcobIsuXitaSNgl36acZPZvG3dH6c/U5B35aABXFEN7wAB3RNvRL6heSJQkhWktKEkB1dZPZCpe+Cl2HNL67N/qcA/ZK2P1l688l/K5/Yn/2nNxDhb0i2KEI0aZIknDng+5i1XqcAdEpsOk/vjun8Jt+if2waRu7Tuxq9blCqTOICC0t+duUJAH1R1z7gtkCg2bAjs+g7KTvzy98ytV43dp2iYiICPLz8yVRiDZHa01+fj4RERHNOk7aJGrxYUkCYMhlsGo+bP0QRs5uen8RNGkxaURbo1vdLpGWlkZubi4+W0pXCB+KiIggLS2tWcdIkgA8rIrqG92GGz2mNv1HkkQbZ1Im+ib0bXVJwmq1kpWV5aOohAg+qW6CmuomX7ZJuM43+DLIWQGncn17buFz/ZP6s/PETuwO/6xtJUR7JEmiFh8nCYAhvzHu35zhn7YP4TP9E/tTZitjX+G+YIciRJshSQLwW3UTQGI2xGdA3i9wwE9LdAuf6JfYD2h947UQoUSShDtfVze53LDMmFl27Wv+Ob/wiez4bMJMYTKoTgg3kiTA/9VAnVJg5DWweREU7PHvtUSLWU1WeiX0kpKEEG4kSQA11U1+KkkAnGmsc8v3z/nvGqLV+if255eCX2ScgxBOkiTc+au6CSC2qzFuYv1bUHzMf9cRrTI8ZTinKk6xtUAmxRMCJEkYAvWtcdwdoO3w1d8Dcz3RbBPSJqBQLD8gKwsKAZIk6vBjSQIguTeMuBrWLZTlTduoxIhEhnYeyjcHvgl2KEK0CZIkAL92ga1r/J3G/RsXQN72wF1XeO2s9LPYVrCNoyVHgx2KEEEnScKdP9skXOLS4Jx/GI+//Kv/ryeabWLaRACW50ppTwhJEhD4kdBj58DE/4VfPobcFqzJLfyqZ3xPunfqzre53wY7FCGCTpIEEJAusHWNnQNRybDsIZmuo41RSnFW2ln8ePhHymxlwQ5HiKCSJOEuENVNLuGd4Kx7jMn/tn0UuOsKr5yVfhYV9gpWHV6F3WFn3vp5zPhoBoeLDwc7NCECSpIEBO+b/KjrIHUwvHc1PBQH2z8NThyintFdRhNtjeaDXR9w4xc3Mn/DfHaf3M09K+7B5rAFOzwhAkaSRC0BLEmAsXrdjFdrnr99BZyQGUjbAqvZyhndzuDL/V+y+fhm/j7u7/xj/D/4+djPzFs/L9jhCREwsugQENAusHV17gP3HzHmdHrlHHh5Ctz4FcSnBy8mAcCs/rOosFdw18i76BnfE4BVR1bx8qaXGZ06mrHdxgY5QiH8T5IE+G/RIW9ZI6HLQLjiTXjrcvjXIDjtFjj7b2AJC05MghFdRjCiy4ha2+4dcy8bjm1g7vK5TEqfRL/EfmTGZVJUWUR+WT4V9gp+3fPXpESlBClqIXxLkkQtQUoSLtkT4ddPw+KbYdULxhoUM98HszW4cYlqkZZInpr0FI+veZyVB1fy4e4P6+3z4sYXuXbgtcweOJsoa1QQohTCdyRJtDVDr4Duo2D5I8ba2J/+EaY/GbxSjqgnKy6L5yc/D8DxsuPsL9xPbFgsSZFJFFUW8fS6p5m3YR7/2fEfftP3N1zc62JSo1ODHLUQLROwJKGUehU4HzimtR7k3JYIvAtkAjnAZVrrE87X7gOuB+zA77XWnwUgSL9fwivJvWDGyxDbHb77FyRkwrjbgx2V8CA5MpnkyOTq5wkRCTwx8YnqBu556+cxf/18xnYbS3ZcNvHh8SRFJnF2xtnEhccFMXIhvBPIksTrwHPAG27b7gW+1Fo/opS61/n8HqXUAOAKYCDQDVimlOqjtfbPCvVtdTDb5Afh5D744gGwRsGYG4MdkfDS8JThvDT1JQ4UHeCDXR/wec7nbMjbQElVCQDP/vws94y+h3OzzkW1lS8nQngQsC6wWutvgYI6my8EFjofLwQuctv+jta6Qmu9F9gFjPFjdM77NvbPajLBJS9B3/Ngyd2w6f1gRySaKT0mnd8N/x3/vfi//Hjlj6ydtZa3p79Nt+hu3LPiHm798lbWHl2LQzuqj6mwV/DzsZ8prCwMYuRCGILdJtFFa30YQGt9WCnl6hLSHfjRbb9c57Z6lFI3ATcB9OjRo3XRtMVvdGYrXPoqvDnDaNA+vMHo+RTn8cch2rgwcxiDkgfx5nlv8s72d3j252e5Zuk1pESl8Kv0X3Gk5AirjqyizFZGcmQyD5z+AJN6TAp22KIDa6uD6Tx9WnusE9JaL9Baj9Jaj+rcuXPLrtZWq5tcrJHwP29D12Hw/TPw1AD44kFwOJo+VrRJZpOZmf1n8uVvvuSRCY8wIGkAi3YuYtfJXVzY80L+b8L/kRSRxO+//j33rbiPA4UHZElVERTBLkkcVUp1dZYiugKudT1zAffRZGnAoYBH15ZExMH1n8Pa143G7O/+BacOwEXzwRIe7OhEC0Vbo5mePZ3p2dOxOWyYlbm6jeKcjHNYsGkBL298mY/3fExKVAoju4xkQvcJTEyfSExYTJCjFx1BsJPER8Bs4BHn/Ydu299SSj2J0XDdG/jJf2G0k29oJjOMvt6Y8+m7p2HZg1B4GC57Azq1sBQl2gyLqfa/o9VsZc6wOVyQfQHfH/qetUfXsvrIaj7d+ylWkzFtSGp0Kvll+eSX55MSlcKk9ElMSJtAbFhskN6FCDWB7AL7NjARSFZK5QIPYiSH95RS1wP7gd8AaK23KKXeA7YCNmCO33o21Q7S75fwCaVg/B3GAkYfzoF5p8GEuTDyGgiTwVuhJj02nctjL+fyfpfj0A42Hd/E5zmf8+X+L1mft56kiCQSIxJZe3Qtn+V8hkVZGJ82nln9ZzEmdYz0nhKtokKpnnPUqFF6zZoWLOJz7Bfjg/bSV2HQDN8H5k+H1sOSuZC7GjqlwjkPQ/cREJ9p9I4SHYYrgXy570s+3P0hBeUF9E3oyx9H/5ExXf3YOVC0e0qptVrrUR5fkyQBHNsG806HS1+DQZf4PjB/0xq2LzGWQ837pWZ7t+HG/E9ZE4IXmwiKCnsFn+z5hHnr59HJ2okPLvog2CGJNqyxJBHsNom2pb0Wy5WCftOh19mw8R2jcTu5L+xdDgvPN0ZuawfE94AxN8HAS6SUEeLCzeFc0vsS9hXu442tb2B32DGbzMEOS7RDkiSg7XeB9ZYlDEZcbdwAqsqMhPHDPCg8CEWH4cAq+Ox/odcUGHsbdBkQ1JCFf2XGZmJz2DhUfIj0WJl+XjSfJIla2mlJoiHWSDj9VuMGxriKrYvhp5dg/b9hzzdw19aghij8KzMuE4CcwhxJEqJFpM4BaDddYFvLZDIa5q9banSjrSwJdkTCzzJiMwAjSQjREpIkIPiLDgWDOTx0qtlEgxLCE4gNi2VfoSyLK1pGkkQtHShJKBMEYOiJCC6lFJmxmeScygl2KKKdkiQBdJjqJncmk9HjSYS8zLhM9hbuDXYYop2SJOGuI1U3KRM4pCTREWTEZnCs9BilVaXBDkW0Q5IkoGPWzSspSXQUmbGZANIuIVpEkkQtHakkYZYk0UG4ejhJkhAtIUkC6JBtEtJw3WG4koS0S4iWkCQBHbMLrGuKho5Y1dbBRFgi6BrdVUoSokUkSdTSgZKEcv7qpcqpQ5BusKKlJEkAHbO6yZkQpYdTh5ARm8G+wn2yBKpoNkkS7jpSdZNyVTdJSaIjyIzLpLiqmPzy/GCHItoZSRLQMevlpbqpQ3F1g5UqJ9FckiSAmuqmjlSScCUJqW7qCNxngxWiOSRJuOtI1U0mqW7qSFKjUgkzhbGvcB85p3KYu3wu1312HYeKDwU7NNHGSZIAiEo2ps6OSwt2JIHjKklIw3WHYDaZ6RHbgw93fchFH17EitwVbM3fyuUfX873h74Pdni+ZauEwsPBjiJkSJIASMiA85+CLgODHUngVFc3dcD2mA5qQNIAiiqLuKzvZSy5ZAnvTH+H5Mhkbl12K0+ve5pdJ3Y1u/eTQzs8zglVWlXKhrwNFFUWtTruKnsVxZXFFFcWU1JVQmlVKRX2CqocVdgd9uqYHdpBua2cov/ehuPJflBV3uprC1Ch1CVu1KhRes2aNcEOo3346SVYMhfm7oJOnYMdjQiAkqoSymxlJEcmV28rrSrlLz/8hSV7lwDQNborg5MHo9FU2auIDY/lmoHX0DuhN2B8EH+691M+3PUhB4sPcrjkMFWOKnrE9GBYyjAyYjNYd3Qdq4+sptJRiUmZ6JfYj2Gdh+HQDk5VnqKkqoSUqBQyYjLIiM1gYPJAUqJSasVaYa9gZe5KluYsZXnucspsZc16r9OLS3jk2jUQldjoflprHl39KJ/s+YTunbqTEZdB34S+TEqfVN2O0xEopdZqrUd5fE2SRAe1+hX45C74w3aISQ12NCLIjpQcYeXBlazIXcHuU7uxmqxYTVYOFB2gpKqE87PPZ3z38byy+RV2nNhBZmwmfRP70q1TN6It0WzJ38KGvA0UlBeQGZvJmWlnMrTzUHad3MXqI6vZfHwzYeYw4sLjiLJEcbT0KCcrTlZfv1t0NwYlD6LMVsaBogPkFudic9hICE/g7Iyz6RHbAzA+1B04cGgHdocdBw7QYNd2zCYzVpOV7358ku1mxYoZSzHHNb5k6/wN85m3fh7ju4/H7rCzr3Afh0qMdppe8b04N+tcLu97OXHhcf774bcBkiREfWtfh//eDndtg9huwY5GtFGnKk7xyuZXeGvbW1TYK0iPSee2YbcxLWsaJlW7tlprTWFlodcfqKcqTrH31F42Hd/EhrwNbD6+mZiwGNJj0kmPSee01NMY03UMFpOlWTF/8mw/7o218u6EJxiQPbV6+/Prn2dl7kqu7H8l07Km8fHuj3ng+we4oOcF/H3c31HOjitHSo6wbN8yvtj3BeuOrSPaGs2V/a7k7Iyz+enIT3y1/yu2FWyja3RXesT2IDsum0npkxjaeWj1OdobSRKivnVvwEe/gzu3dKwGe9EiR0uOsv3EdsZ2G4vVZA12OI06+q8BTEkwM7fvTGaffi8AdoedSe9NoriqmCpHFV2iunC87DhjUsfw/OTnsZo9v6cdJ3awYOMCPs/5HO3sKu+qPjtWeoz9RfvJKczB5rDRNbork3tMplunbsSHxxMbFkuprZTCikLKbGV0ie5Cekw6XaO7UmorpaC8gFMVp+gR04MesT3qJd1AaixJNC9Fi9AhvZtEM3SJ7kKX6C7BDsMrXTCTUVXFmvwtzHZu23h8IycqTvDomY8SbY3m9S2vkxaTxpMTn2wwQQD0SejD42c9zu6hu9mYt5ExXcfQvVP3WvsUVRbxzYFvWJqzlHe3v0uVo6rZMcdYYxiQPIDe8b3JissiMzYTpRSFlYUUVxYTbgknITyB+PB4AMpsZZTby9FaYzFZsJqsJEYkVlfL+ZIkiY5KpuUQocpkZVR5MZ+f3I7dYbRVfHPgGyzKwvju44kJi+HMtDObdcqe8T3pGd/T42sxYTH8uuev+XXPX+PQDooqizhRfoKiyiKirdHEhMUQbgnnaMlR9hft50jJEaIsUSRFJhETFsPeU3vZfHwzW/K3sGjnomY30rtMy5zGY2c91qJjGyNJoqPydloOWyWU5hu9RCzh/o9LiNYyWRhVVs6imDJ2nNhB/6T+LD+wnJFdRhITFuPfSysTceFxHttlYsNiq3uJuRueMpxLel8CGL3HjpUeI6cwBxMmYsJi6GTtRLm9nJMVJzlZcRITJsIt4YSbw1EobNqG3WEnMaLxnlwtJUmio/I2SXz7GHz7KJjD4aw/wvCrwBoBZScgpqskDtH2mMMYVV4BwOojq+kU1ondp3Yzo8+MIAfWNJMykRqdSmp02+lxKEmiozJ5kSSWPQQrnzIex3aFr/5m3FyikiBjHAy9AjLOgOM74fgOyBwPCZk1+9kqwV4BDhuc3A+RiUZjeTvtCSLauMh4Uu120q2xrDm6BrNzCpqJaRODG1c7JUmio2qo4fq7p6GyBIb+T02CuPlbSB0Ce76Bbf+FNa9Al0GQmAUHVsG2j+qfPyETUgYaJY5DP0PdetawTtC5L3TuV3NL6QexaXBwDRQdhoQsiIiD8BjjeUQ8RCaANVISjGiYNQqA0ZZ4lh1dS2lVKdlx2aTHNj5mQngmSaKjaqi66YsHjPvl/zTur1oMXYcaj3tOMm7nP1mzv90Gu76AnJUQHgtdhxilhb3fGiWLikLIGGuUNCqKjOShNeRth7xfYNcyWP9vt7jMTc9Maw4zkkVkgpE4opOdt85GUjGHg9li7GcOM95j4UHI323E0XMyzDsdLn4RVr0Agy+F4bNa/KNsF7TuOInV2btolN3M/6ssZNWRVVw76NrmnUNrKD4Knbp0nJ9bAyRJdFQN9W6K7gwledBlMPQ7D3r+qvHzmC3Q91zj5u60m72PpbTAqKY6tg3yd4HZCn3Pg8JDRmIpPQ7RKcY/f9kJKDvpvHfeCvbCgZ+M/ZpqY9mz3PjHLz8Jb1/u3PY1rHoRpv4Nss4KrQ8Fhx0WTISwaLhuabCjCQy7M0kUHIJOxqaJXcc1nihtFcZxOSth//ew5QM4uc/4f+g6zCjlpgyApF7GF6xTuWCyQKcU4xadAmFRAXqDgSVJoqPytJ6E1saH7oQ/wOQHAhdLVCL0ON24tYbDDpXFRunGXmncHDbjFt0Zlj9qlFrcJ6SzRhmJ8JeP4Y0LjVJI9kTIPsuYHTh9jFEKiUw0RqZbIozGem8SSd4OI2mFRRnVa9Yoz8dWlUPRoZrJFg+vN0phDhugjN5l5SeN/cpPGud0lZIs4TWPy08aCb44z+hcYA6HY1tqEqfdBnu/gbge0LlP637WnpQXGh+02g62cuOLiDXS8/vW2vh9OaqM35PJauxrrzQS/om9zvdlNc7jmtq+/JQxw+vRzVCwxyiNWqOMUmx4J+NLRfFRALrm7SQtZTRFVaUMXTDVqNIcdb3x89z9FRQfMUqix7ZVlz5qCY81Sp1HN8Pe5UZsjQmLMeZB69TF+HuzRhnxhsfUvlUUGb+rsE7G87BOxvu0hNfeFt7J+HvL32X8rMxhxpeyspPGlyoAlFFaP7kfug2HM+e2/vdYhySJjsr1T1dVZkzRoUzwyR+MD6aopKCG1mIms1Hd1JCIOOMfqtS5hOdZ9xpVVqffYrTDbHofDq2Dnctg52cNnycyAZL7QnIviO1uzH2lNez73vh5hncyku3Oz+sfq0w1CSMs2vjHP77DcxWbMhkf8CYLxPcwjomIM57byo33YnMmQ3uF8YHSqQtkjjPisJXD0U015/ub2+91ABIqLQAABgxJREFU9I0w5HKj+q8kz4g1Mdu4xaXBkY2w4zPjw8u1zRxutEE5bMbP6+BaI36HDY5tNc7T4O/GAtZo433aK53f9uvO9qA8bGtARDykDobRNxjXrygybq7EkjEOPvkDtxdVUlGQgxmMhPLp3TXn6NzP+Jk6qiBjPGRNMGIYNMN4zya3EdB2m5G48rYbvxdXKaIkD4qPGYmp+BiUHDPu87Ybv5+oJOOLS0WRkURdySgywSi9eJhBt9ms0cbvJ2VA68/lQZtPEkqpacDTgBl4WWv9SJBDCg2uksTqV2Dz+zXbM8ZD76mej2nvzM4/94/vNO7HzoGIWONxWDSMnG3ctDYaykvzjW+12mH8UxcdMf7hiw4bpYSdXxgfCO4rG3YZWPOB1WUQnPF74wO8shSqSoz7ypKax1Wl0GWAUZpxfeBGJhptJ67YWtOe8NHvYd3C2tv6nAtrX4PVLzV9vMniLNF4EN3ZSF7lp4wP3P4XGEkqIq6m5OB6j5XFRuIyWYyb2Wq8X9dje5Wxn73K6NDQfZSR9F0lE4fd+Dlbo43zRybU/D4bUn6KaV/82Xh82i1wzj9gy2Ijjm7DIXVQ0+/fxWyB5N7GrTVsFcbN9bt1L/3ayqCi2Pj7qCxyPi42EmJcd+NnYa8yYkkdavx8tK6dzPygTScJpZQZeB44G8gFViulPtJabw1uZCHAGmncuxLEuNsh29kwHapK8mseu4r1nihlVC3FdjO+rTbGbjO+PZbkQVJv/9RLt6aNxFXVNO2f0P/XxocNGCWdXV8acdvKjS8GtnKj2uLUQSMRDpsJyX2gMLdmW7fhEJduHB+dXFMibYvG/d7oSKF1zd/14EuDG5MlvPbYoqZKv00JQPtZm04SwBhgl9Z6D4BS6h3gQkCSRGulnw79zjfq4ntPhbP/GuyI/G/c7UZxf8pDxjcyX/yDmS01CaUtclVnJPWsSRBgfBP39IHZfWT9bQmZtce9AMS0j3mcyJ4Y7AjavbaeJLoDB9ye5wKnue+glLoJuAmgRw/fT24VsswWuOLfRiNgQ9+oQ01sV5j+RLCjCKzJDxrjTbJDuIQo/KqtL1/q6aterZYtrfUCrfUorfWozp1lhbVmi+1qNLSK0JSQAZP/3HT9vRANaOtJIhdwHyaZBhwKUixCCNHhtPUksRrorZTKUkqFAVcAHuaAEEII4Q9tugyqtbYppW4DPsPoAvuq1npLkMMSQogOo00nCQCt9RJgSbDjEEKIjqitVzcJIYQIIkkSQgghGiRJQgghRIMkSQghhGiQ0trLWRfbAaVUHrAv2HEIIUQ7k6G19jgaOaSShBBCCN+S6iYhhBANkiQhhBCiQZIkhBBCNEiShBBBpJSaqJT6ONhxCNEQSRJCCCEaJElCCC8opWYppX5SSq1XSr2olDIrpYqVUk8opdYppb5USnV27jtMKfWjUmqjUmqxUirBub2XUmqZUmqD85ieztN3Ukq9r5T6RSn1b6WMJfOUUo8opbY6z/N4kN666OAkSQjRBKVUf+ByYJzWehhgB2YC0cA6rfUIYDnwoPOQN4B7tNZDgE1u2/8NPK+1HgqcARx2bh8O3AEMALKBcUqpROBiYKDzPH/377sUwjNJEkI0bTIwElitlFrvfJ4NOIB3nfu8CYxXSsUB8Vrr5c7tC4EzlVIxQHet9WIArXW51tq5ADU/aa1ztdYOYD2QCRQC5cDLSqlLANe+QgSUJAkhmqaAhVrrYc5bX631Qx72a2xkqqeleF0q3B7bAYvW2gaMARYBFwFLmxmzED4hSUKIpn0JXKqUSgFQSiUqpTIw/n8ude5zJbBSa30KOKGUmuDcfhWwXGtdCOQqpS5yniNcKRXV0AWVUp2AOOd6KncAw/zxxoRoSptfdEiIYNNab1VK/Qn4XCllAqqAOUAJMFAptRY4hdFuATAbeMGZBPYA1zq3XwW8qJT6q/Mcv2nksjHAh0qpCIxSyJ0+fltCeEXmbhKihZRSxVrrTsGOQwh/kuomIYQQDZKShBBCiAZJSUIIIUSDJEkIIYRokCQJIYQQDZIkIYQQokGSJIQQQjTo/wNpV6IdnCgQogAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "\n",
    "errors, ns = batch_gradient(X, y,0.0005)\n",
    "plt.plot(ns,errors, label=\"$\\eta=0.0005$, batch GD\")\n",
    "\n",
    "errors, ns = stochastic_gradient(X, y,0.0005)\n",
    "plt.plot(ns,errors, label=\"$\\eta=0.0005$, stochastic GD\")\n",
    "\n",
    "errors, ns = minibatch_gradient(X, y,0.0005)\n",
    "plt.plot(ns,errors, label=\"$\\eta=0.0005$, mini-batch GD\")\n",
    "\n",
    "\n",
    "plt.xlabel(\"epochs\")\n",
    "plt.ylabel(\"$J(\\omega)$\")\n",
    "plt.xticks(ticks=[])\n",
    "\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}