model_sizing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Welcome! In this notebook we aim to estimate the compute and memory requirements needed to train a theoretical model architecture using LightGPT. We'll start by first defining the parameters of the architecture."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model\n",
    "vocabulary_size = 50257\n",
    "embedding_dimensions = 1024\n",
    "num_attention_heads = 16\n",
    "num_hidden_layers = 24\n",
    "feed_forward_ratio = 4\n",
    "\n",
    "# Training set\n",
    "tokens_per_sample = 1024\n",
    "samples_per_epoch = 4096"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we'll estimate the total number of trainable parameters in the network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAILCAYAAAAABSOZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABsAklEQVR4nO3dd1gUV9sG8HvpNsAGiEHB3hAVG/aCYhd7L1hiNEYjVizYJRo1GruxG3s0JPaCJRqxiy32BiogNkBU2j7fH37MywaMosACc/+uay/dmbPDM6vM3nvmzBmNiAiIiIiIVMRA3wUQERERpTcGICIiIlIdBiAiIiJSHQYgIiIiUh0GICIiIlIdBiAiIiJSHQYgIiIiUh0GICIiIlIdBiAiIiJSHQYgIkoVGo0GkyZNSvHrHjx4AI1GgzVr1qR6TUREH8IARJSFrFmzBhqNBhqNBidOnEiyXkRgZ2cHjUaDFi1a6KHCz3f06FFl3zQaDYyNjVGkSBH07NkT9+7d03d5aeqff/7BpEmT8ODBA32XQpRlMAARZUFmZmbYuHFjkuXHjh3Do0ePYGpqqoeqUseQIUOwfv16LF++HM2bN8eWLVtQpUoVPHnyRN+lpZl//vkHkydPZgAiSkUMQERZULNmzbBt2zbExcXpLN+4cSOcnZ1hY2Ojp8q+XO3atdG9e3d4eHhgwYIFmD17Nl68eIG1a9d+0XZFBG/fvk2lKjOHqKgofZdApDcMQERZUJcuXfD8+XMcPHhQWRYTE4PffvsNXbt2TfY1UVFRGD58OOzs7GBqaoqSJUti9uzZEBGddtHR0Rg2bBjy58+PXLlyoVWrVnj06FGy23z8+DH69OkDa2trmJqaomzZsli1alXq7SiABg0aAADu378PAFi9ejUaNGgAKysrmJqaokyZMliyZEmS19nb26NFixbYv38/KleujGzZsmHZsmWftY2jR48q23B0dMTRo0cBADt27ICjoyPMzMzg7OyMixcvJtnGjRs30L59e+TJkwdmZmaoXLky/vzzT2X9mjVr0KFDBwBA/fr1lVOACT8DAPbu3YvatWsjR44cyJUrF5o3b45r167p/JzevXsjZ86cuHv3Lpo1a4ZcuXKhW7duAIDbt2+jXbt2sLGxgZmZGb766it07twZ4eHhn/rPQJTpGOm7ACJKffb29nBxccGmTZvQtGlTAO8/JMPDw9G5c2f8/PPPOu1FBK1atcKRI0fQt29fVKhQAfv378fIkSPx+PFj/PTTT0rbfv364ddff0XXrl1Ro0YNHD58GM2bN09SQ2hoKKpXrw6NRoPBgwcjf/782Lt3L/r27YuIiAh8//33qbKvd+/eBQDkzZsXALBkyRKULVsWrVq1gpGREXbu3IlBgwZBq9Xi22+/1XntzZs30aVLFwwYMAD9+/dHyZIlU7yNO3fuoGvXrhgwYAC6d++O2bNno2XLlli6dCnGjh2LQYMGAQB8fHzQsWNH3Lx5EwYG7797Xrt2DTVr1kTBggUxZswY5MiRA1u3boW7uzu2b9+ONm3aoE6dOhgyZAh+/vlnjB07FqVLlwYA5c/169ejV69ecHNzw8yZM/HmzRssWbIEtWrVwsWLF2Fvb6/UGhcXBzc3N9SqVQuzZ89G9uzZERMTAzc3N0RHR+O7776DjY0NHj9+jF27duHVq1ewsLBIlX8nogxHiCjLWL16tQCQs2fPysKFCyVXrlzy5s0bERHp0KGD1K9fX0REChcuLM2bN1de5+vrKwBk2rRpOttr3769aDQauXPnjoiIBAQECAAZNGiQTruuXbsKAJk4caKyrG/fvlKgQAF59uyZTtvOnTuLhYWFUtf9+/cFgKxevfo/9+3IkSMCQFatWiVhYWHy5MkT2b17t9jb24tGo5GzZ8+KiCjbTczNzU2KFCmis6xw4cICQPbt25ekfUq3cfLkSWXZ/v37BYBky5ZNHj58qCxftmyZAJAjR44oyxo2bCiOjo7y7t07ZZlWq5UaNWpI8eLFlWXbtm1L8loRkcjISLG0tJT+/fvrLA8JCRELCwud5b169RIAMmbMGJ22Fy9eFACybdu2JPtMlJXxFBhRFtWxY0e8ffsWu3btQmRkJHbt2vXB01979uyBoaEhhgwZorN8+PDhEBHs3btXaQcgSbt/9+aICLZv346WLVtCRPDs2TPl4ebmhvDwcFy4cOGz9qtPnz7Inz8/bG1t0bx5c0RFRWHt2rWoXLkyACBbtmxK2/DwcDx79gx169bFvXv3kpzScXBwgJubW5KfkZJtlClTBi4uLsrzatWqAXh/aq5QoUJJlidcsfbixQscPnwYHTt2RGRkpPL+PH/+HG5ubrh9+zYeP378n+/FwYMH8erVK3Tp0kXnPTY0NES1atVw5MiRJK8ZOHCgzvOEHp79+/fjzZs3//nziLISngL7iL/++gs//vgjzp8/j+DgYPz+++9wd3dP0Tb279+PiRMn4tq1azAzM0OdOnUwZ84cna5potSWP39+uLq6YuPGjXjz5g3i4+PRvn37ZNs+fPgQtra2yJUrl87yhNMsDx8+VP40MDBA0aJFddolnDpKEBYWhlevXmH58uVYvnx5sj/z6dOnn7Vf3t7eqF27NgwNDZEvXz6ULl0aRkb/O5T9/fffmDhxIvz9/ZN8oIeHh+uc0nFwcEj2Z6RkG4lDDvC/QGFnZ5fs8pcvXwJ4f+pMRDBhwgRMmDAh2TqePn2KggULJrsOeD92B/jfOKh/Mzc313luZGSEr776SmeZg4MDPD09MXfuXGzYsAG1a9dGq1at0L17d57+oiyNAegjoqKi4OTkhD59+qBt27Ypfv39+/fRunVreHp6YsOGDQgPD8ewYcPQtm3bz/4GTPSpunbtiv79+yMkJARNmzaFpaVluvxcrVYLAOjevTt69eqVbJvy5ct/1rYdHR3h6uqa7Lq7d++iYcOGKFWqFObOnQs7OzuYmJhgz549+Omnn5S6EiTu6fncbRgaGiZby4eWy/8PKk/YzogRI5LthQKAYsWKJbs8QcI21q9fn+yVfYmDIQCYmpoq448SmzNnDnr37o0//vgDBw4cwJAhQ+Dj44NTp04lCUxEWQUD0Ec0bdpUGUSanOjoaIwbNw6bNm3Cq1evUK5cOcycORP16tUDAJw/fx7x8fGYNm2acuAZMWIEWrdujdjYWBgbG6fHbpBKtWnTBgMGDMCpU6ewZcuWD7YrXLgwDh06hMjISJ1eoBs3bijrE/7UarW4e/euTq/PzZs3dbaXcIVYfHz8B8NKWti5cyeio6Px559/6vTMJHcqKC238SmKFCkCADA2Nv7oe6TRaJJdntATZ2Vl9cXvs6OjIxwdHTF+/HicPHkSNWvWxNKlSzFt2rQv2i5RRsUxQF9o8ODB8Pf3x+bNm3H58mV06NABTZo0UbqmnZ2dYWBggNWrVyM+Ph7h4eFYv349XF1dGX4ozeXMmRNLlizBpEmT0LJlyw+2a9asGeLj47Fw4UKd5T/99BM0Go3yJSDhz39fRTZv3jyd54aGhmjXrh22b9+Oq1evJvl5YWFhn7M7H5XQ6yKJLt0PDw/H6tWr03Ubn8LKygr16tXDsmXLEBwcnGR94vcoR44cAIBXr17ptHFzc4O5uTlmzJiB2NjY/9zGh0RERCSZL8rR0REGBgaIjo7+lF0hypTYA/QFAgMDsXr1agQGBsLW1hbA+96dffv2YfXq1ZgxYwYcHBxw4MABdOzYEQMGDEB8fDxcXFyUwaREae1Dp6ASa9myJerXr49x48bhwYMHcHJywoEDB/DHH3/g+++/V3oaKlSogC5dumDx4sUIDw9HjRo14Ofnhzt37iTZ5g8//IAjR46gWrVq6N+/P8qUKYMXL17gwoULOHToEF68eJHq+9q4cWOYmJigZcuWGDBgAF6/fo1ffvkFVlZWyYaMtNrGp1q0aBFq1aoFR0dH9O/fH0WKFEFoaCj8/f3x6NEjXLp0CcD7993Q0BAzZ85EeHg4TE1NlXmKlixZgh49eqBSpUro3Lkz8ufPj8DAQOzevRs1a9ZMEmr/7fDhwxg8eDA6dOiAEiVKIC4uDuvXr1dCLFFWxQD0Ba5cuYL4+HiUKFFCZ3l0dLQyJ0lISAj69++PXr16oUuXLoiMjIS3tzfat2+PgwcPfrBrmyg9GRgY4M8//4S3tze2bNmC1atXw97eHj/++COGDx+u03bVqlXInz8/NmzYAF9fXzRo0AC7d+9OMujX2toaZ86cwZQpU7Bjxw4sXrwYefPmRdmyZTFz5sw02Y+SJUvit99+w/jx4zFixAjY2Nhg4MCByJ8/P/r06ZNu2/hUZcqUwblz5zB58mSsWbMGz58/h5WVFSpWrAhvb2+lnY2NDZYuXQofHx/07dsX8fHxOHLkCKysrNC1a1fY2trihx9+wI8//ojo6GgULFgQtWvXhoeHx0drcHJygpubG3bu3InHjx8je/bscHJywt69e1G9evVU3V+ijEQj8q9pXumDNBqNzlVgW7ZsQbdu3XDt2rUkAx5z5swJGxsbTJgwAfv27cPZs2eVdY8ePYKdnR38/f15gCEiItID9gB9gYoVKyI+Ph5Pnz5F7dq1k23z5s2bJFddJISlf19NQkREROmDg6A/4vXr1wgICEBAQACA95e1BwQEIDAwECVKlEC3bt3Qs2dP7NixA/fv38eZM2fg4+OD3bt3AwCaN2+Os2fPYsqUKbh9+zYuXLgADw8PFC5cGBUrVtTjnhEREakXT4F9xNGjR1G/fv0ky3v16oU1a9YgNjYW06ZNw7p16/D48WPky5cP1atXx+TJk+Ho6AgA2Lx5M2bNmoVbt24he/bscHFxwcyZM1GqVKn03h0iIiICAxARERGpEE+BERERkeowABEREZHq8CqwZGi1Wjx58gS5cuXiPD1ERESZhIggMjIStra2yd73LjEGoGQ8efIkyaRuRERElDkEBQV99Ea+DEDJSLgZZFBQEMzNzfVcDREREX2KiIgI2NnZ6dzU+UMYgJKRcNrL3NycAYiIiCiT+ZThKxwETURERKrDAERERESqwwBEREREqsMARERERKrDAERERESqwwBEREREqsMARERERKrDAERERESqwwBEREREqsMARERERKrDAERERESqwwBEREREqsMARERERKqj1wDk4+ODKlWqIFeuXLCysoK7uztu3rz50ddt27YNpUqVgpmZGRwdHbFnzx6d9SICb29vFChQANmyZYOrqytu376dVrtBREREmYxeA9CxY8fw7bff4tSpUzh48CBiY2PRuHFjREVFffA1J0+eRJcuXdC3b19cvHgR7u7ucHd3x9WrV5U2s2bNws8//4ylS5fi9OnTyJEjB9zc3PDu3bv02C0iIiLK4DQiIvouIkFYWBisrKxw7Ngx1KlTJ9k2nTp1QlRUFHbt2qUsq169OipUqIClS5dCRGBra4vhw4djxIgRAIDw8HBYW1tjzZo16Ny580friIiIgIWFBcLDw2Fubp46O0ekQvZjduu7hEzlwQ/N9V0CUaaWks/vDDUGKDw8HACQJ0+eD7bx9/eHq6urzjI3Nzf4+/sDAO7fv4+QkBCdNhYWFqhWrZrS5t+io6MRERGh8yAiIqKsK8MEIK1Wi++//x41a9ZEuXLlPtguJCQE1tbWOsusra0REhKirE9Y9qE2/+bj4wMLCwvlYWdn9yW7QkRERBlchglA3377La5evYrNmzen+8/28vJCeHi48ggKCkr3GoiIiCj9GOm7AAAYPHgwdu3ahb/++gtfffXVf7a1sbFBaGiozrLQ0FDY2Ngo6xOWFShQQKdNhQoVkt2mqakpTE1Nv2APiIiIKDPRaw+QiGDw4MH4/fffcfjwYTg4OHz0NS4uLvDz89NZdvDgQbi4uAAAHBwcYGNjo9MmIiICp0+fVtoQERGRuum1B+jbb7/Fxo0b8ccffyBXrlzKGB0LCwtky5YNANCzZ08ULFgQPj4+AIChQ4eibt26mDNnDpo3b47Nmzfj3LlzWL58OQBAo9Hg+++/x7Rp01C8eHE4ODhgwoQJsLW1hbu7u172k4iIiDIWvQagJUuWAADq1auns3z16tXo3bs3ACAwMBAGBv/rqKpRowY2btyI8ePHY+zYsShevDh8fX11Bk6PGjUKUVFR+Prrr/Hq1SvUqlUL+/btg5mZWZrvExEREWV8GWoeoIyC8wARpQ7OA5QynAeI6Mtk2nmAiIiIiNIDAxARERGpDgMQERERqQ4DEBEREakOAxARERGpDgMQERERqQ4DEBEREakOAxARERGpDgMQERERqQ4DEBEREakOAxARERGpDgMQERERqQ4DEBEREakOAxARERGpDgMQERERqQ4DEBEREakOAxARERGpDgMQERERqQ4DEBEREakOAxARERGpDgMQERERqQ4DEBEREakOAxARERGpDgMQERERqQ4DEBEREakOAxARERGpDgMQERERqQ4DEBEREakOAxARERGpDgMQERERqQ4DEBEREakOAxARERGpjl4D0F9//YWWLVvC1tYWGo0Gvr6+/9m+d+/e0Gg0SR5ly5ZV2kyaNCnJ+lKlSqXxnhAREVFmotcAFBUVBScnJyxatOiT2s+fPx/BwcHKIygoCHny5EGHDh102pUtW1an3YkTJ9KifCIiIsqkjPT5w5s2bYqmTZt+cnsLCwtYWFgoz319ffHy5Ut4eHjotDMyMoKNjU2q1UlERERZS6YeA7Ry5Uq4urqicOHCOstv374NW1tbFClSBN26dUNgYKCeKiQiIqKMSK89QF/iyZMn2Lt3LzZu3KizvFq1alizZg1KliyJ4OBgTJ48GbVr18bVq1eRK1euZLcVHR2N6Oho5XlERESa1k5ERET6lWkD0Nq1a2FpaQl3d3ed5YlPqZUvXx7VqlVD4cKFsXXrVvTt2zfZbfn4+GDy5MlpWS4RERFlIJnyFJiIYNWqVejRowdMTEz+s62lpSVKlCiBO3fufLCNl5cXwsPDlUdQUFBql0xEREQZSKYMQMeOHcOdO3c+2KOT2OvXr3H37l0UKFDgg21MTU1hbm6u8yAiIqKsS68B6PXr1wgICEBAQAAA4P79+wgICFAGLXt5eaFnz55JXrdy5UpUq1YN5cqVS7JuxIgROHbsGB48eICTJ0+iTZs2MDQ0RJcuXdJ0X4iIiCjz0OsYoHPnzqF+/frKc09PTwBAr169sGbNGgQHBye5gis8PBzbt2/H/Pnzk93mo0eP0KVLFzx//hz58+dHrVq1cOrUKeTPnz/tdoSIiIgyFY2IiL6LyGgiIiJgYWGB8PBwng4j+gL2Y3bru4RM5cEPzfVdAlGmlpLP70w5BoiIiIjoSzAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHq6DUA/fXXX2jZsiVsbW2h0Wjg6+v7n+2PHj0KjUaT5BESEqLTbtGiRbC3t4eZmRmqVauGM2fOpOFeEBERUWaj1wAUFRUFJycnLFq0KEWvu3nzJoKDg5WHlZWVsm7Lli3w9PTExIkTceHCBTg5OcHNzQ1Pnz5N7fKJiIgokzLS5w9v2rQpmjZtmuLXWVlZwdLSMtl1c+fORf/+/eHh4QEAWLp0KXbv3o1Vq1ZhzJgxX1IuERERZRGZcgxQhQoVUKBAATRq1Ah///23sjwmJgbnz5+Hq6ursszAwACurq7w9/f/4Paio6MRERGh8yAiIqKsK1MFoAIFCmDp0qXYvn07tm/fDjs7O9SrVw8XLlwAADx79gzx8fGwtrbWeZ21tXWScUKJ+fj4wMLCQnnY2dml6X4QERGRfun1FFhKlSxZEiVLllSe16hRA3fv3sVPP/2E9evXf/Z2vby84OnpqTyPiIhgCCIiIsrCMlUASk7VqlVx4sQJAEC+fPlgaGiI0NBQnTahoaGwsbH54DZMTU1hamqapnUSERFRxpGpToElJyAgAAUKFAAAmJiYwNnZGX5+fsp6rVYLPz8/uLi46KtEIiIiymD02gP0+vVr3LlzR3l+//59BAQEIE+ePChUqBC8vLzw+PFjrFu3DgAwb948ODg4oGzZsnj37h1WrFiBw4cP48CBA8o2PD090atXL1SuXBlVq1bFvHnzEBUVpVwVRkRERKTXAHTu3DnUr19feZ4wDqdXr15Ys2YNgoODERgYqKyPiYnB8OHD8fjxY2TPnh3ly5fHoUOHdLbRqVMnhIWFwdvbGyEhIahQoQL27duXZGA0ERERqZdGRETfRWQ0ERERsLCwQHh4OMzNzfVdDlGmZT9mt75LyFQe/NBc3yUQZWop+fzO9GOAiIiIiFKKAYiIiIhUhwGIiIiIVIcBiIiIiFSHAYiIiIhUhwGIiIiIVIcBiIiIiFSHAYiIiIhUhwGIiIiIVIcBiIiIiFSHAYiIiIhUhwGIiIiIVIcBiIiIiFSHAYiIiIhUhwGIiIiIVIcBiIiIiFQnxQFo7dq12L17t/J81KhRsLS0RI0aNfDw4cNULY6IiIgoLaQ4AM2YMQPZsmUDAPj7+2PRokWYNWsW8uXLh2HDhqV6gURERESpzSilLwgKCkKxYsUAAL6+vmjXrh2+/vpr1KxZE/Xq1Uvt+oiIiIhSXYp7gHLmzInnz58DAA4cOIBGjRoBAMzMzPD27dvUrY6IiIgoDaS4B6hRo0bo168fKlasiFu3bqFZs2YAgGvXrsHe3j616yMiIiJKdSnuAVq0aBFq1KiBsLAwbN++HXnz5gUAnD9/Hl26dEn1AomIiIhSW4p6gOLi4vDzzz9j9OjR+Oqrr3TWTZ48OVULIyIiIkorKeoBMjIywqxZsxAXF5dW9RARERGluRSfAmvYsCGOHTuWFrUQERERpYsUD4Ju2rQpxowZgytXrsDZ2Rk5cuTQWd+qVatUK46IiIgoLaQ4AA0aNAgAMHfu3CTrNBoN4uPjv7wqIiIiojSU4gCk1WrTog4iIiKidPNFN0N99+5datVBRERElG5SHIDi4+MxdepUFCxYEDlz5sS9e/cAABMmTMDKlStTvUAiIiKi1JbiADR9+nSsWbMGs2bNgomJibK8XLlyWLFiRaoWR0RERJQWUhyA1q1bh+XLl6Nbt24wNDRUljs5OeHGjRupWhwRERFRWkhxAHr8+LFyN/jEtFotYmNjU6UoIiIiorSU4gBUpkwZHD9+PMny3377DRUrVkzRtv766y+0bNkStra20Gg08PX1/c/2O3bsQKNGjZA/f36Ym5vDxcUF+/fv12kzadIkaDQanUepUqVSVBcRERFlbSm+DN7b2xu9evXC48ePodVqsWPHDty8eRPr1q3Drl27UrStqKgoODk5oU+fPmjbtu1H2//1119o1KgRZsyYAUtLS6xevRotW7bE6dOndcJX2bJlcejQIeW5kVGKd5OIiIiysBQng9atW2Pnzp2YMmUKcuTIAW9vb1SqVAk7d+5Eo0aNUrStpk2bomnTpp/cft68eTrPZ8yYgT/++AM7d+7UCUBGRkawsbFJUS1ERESkHp/VNVK7dm0cPHgwtWtJMa1Wi8jISOTJk0dn+e3bt2FrawszMzO4uLjAx8cHhQoV+uB2oqOjER0drTyPiIhIs5qJiIhI/1I8BqhIkSJ4/vx5kuWvXr1CkSJFUqWoTzV79my8fv0aHTt2VJZVq1YNa9aswb59+7BkyRLcv38ftWvXRmRk5Ae34+PjAwsLC+VhZ2eXHuUTERGRnqQ4AD148CDZ+31FR0fj8ePHqVLUp9i4cSMmT56MrVu3wsrKSlnetGlTdOjQAeXLl4ebmxv27NmDV69eYevWrR/clpeXF8LDw5VHUFBQeuwCERER6cknnwL7888/lb/v378fFhYWyvP4+Hj4+fnB3t4+VYv7kM2bN6Nfv37Ytm0bXF1d/7OtpaUlSpQogTt37nywjampKUxNTVO7TCIiIsqgPjkAubu7A3h/x/devXrprDM2Noa9vT3mzJmTqsUlZ9OmTejTpw82b96M5s2bf7T969evcffuXfTo0SPNayMiIqLM4ZMDUMJd4B0cHHD27Fnky5fvi3/469evdXpm7t+/j4CAAOTJkweFChWCl5cXHj9+jHXr1gF4f9qrV69emD9/PqpVq4aQkBAAQLZs2ZQeqREjRqBly5YoXLgwnjx5gokTJ8LQ0BBdunT54nqJiIgoa0jxGKD79+8r4edL7wZ/7tw5VKxYUbmE3dPTExUrVoS3tzcAIDg4GIGBgUr75cuXIy4uDt9++y0KFCigPIYOHaq0efToEbp06YKSJUuiY8eOyJs3L06dOoX8+fN/Ua1ERESUdWhERFLyAq1Wi+nTp2Pp0qUIDQ3FrVu3UKRIEUyYMAH29vbo27dvWtWabiIiImBhYYHw8HCYm5vruxyiTMt+zG59l5CpPPjh46f1iejDUvL5neIeoGnTpvFu8ERERJSp8W7wREREpDq8GzwRERGpjl7vBk9ERESkD3q9GzwRERGRPqS4ByjhbvCHDh1S7gZ//fr1z7obPBEREZE+ZOq7wRMRERF9js8KQAlev36tzBCdgPPmEBERUUb3WTNBN2/eHDly5ICFhQVy586N3Llzw9LSErlz506LGomIiIhSVYp7gLp37w4RwapVq2BtbQ2NRpMWdRERERGlmRQHoEuXLuH8+fMoWbJkWtRDRERElOZSfAqsSpUqCAoKSotaiIiIiNJFinuAVqxYgW+++QaPHz9GuXLlYGxsrLO+fPnyqVYcERERUVpIcQAKCwvD3bt34eHhoSzTaDQQEWg0GsTHx6dqgURERESpLcUBqE+fPqhYsSI2bdrEQdBERESUKaU4AD18+BB//vlnsjdEJSIiIsoMUjwIukGDBrh06VJa1EJERESULlLcA9SyZUsMGzYMV65cgaOjY5JB0K1atUq14oiIiIjSQooD0DfffAMAmDJlSpJ1HARNREREmUGKA9C/7/1FRERElNmkeAwQERERUWb3WXeDj4qKwrFjxxAYGIiYmBiddUOGDEmVwoiIiIjSSooD0MWLF9GsWTO8efMGUVFRyJMnD549e4bs2bPDysqKAYiIiIgyvBSfAhs2bBhatmyJly9fIlu2bDh16hQePnwIZ2dnzJ49Oy1qJCIiIkpVKQ5AAQEBGD58OAwMDGBoaIjo6GjY2dlh1qxZGDt2bFrUSERERJSqUhyAjI2NYWDw/mVWVlYIDAwEAFhYWPAu8URERJQppHgMUMWKFXH27FkUL14cdevWhbe3N549e4b169ejXLlyaVEjERERUapKcQ/QjBkzUKBAAQDA9OnTkTt3bgwcOBBhYWFYvnx5qhdIRERElNpS1AMkIrCyslJ6eqysrLBv3740KYyIiIgoraSoB0hEUKxYMY71ISIiokwtRQHIwMAAxYsXx/Pnz9OqHiIiIqI0l+IxQD/88ANGjhyJq1evpkU9RERERGkuxVeB9ezZE2/evIGTkxNMTEyQLVs2nfUvXrxIteKIiIiI0kKKA9C8efNS7Yf/9ddf+PHHH3H+/HkEBwfj999/h7u7+3++5ujRo/D09MS1a9dgZ2eH8ePHo3fv3jptFi1ahB9//BEhISFwcnLCggULULVq1VSrm4iIiDK3FAegXr16pdoPj4qKgpOTE/r06YO2bdt+tP39+/fRvHlzfPPNN9iwYQP8/PzQr18/FChQAG5ubgCALVu2wNPTE0uXLkW1atUwb948uLm54ebNm7Cyskq12omIiCjz0oiIfO6L3717l+Ru8Obm5p9XiEbz0R6g0aNHY/fu3Trjjzp37oxXr14pl+NXq1YNVapUwcKFCwEAWq0WdnZ2+O677zBmzJhPqiUiIgIWFhYIDw//7P0hIsB+zG59l5CpPPihub5LIMrUUvL5neIeoKioKIwePRpbt25N9mqw+Pj4lG7yk/n7+8PV1VVnmZubG77//nsAQExMDM6fPw8vLy9lvYGBAVxdXeHv7//B7UZHRyM6Olp5HhERkbqFU4bAD+NPxw9iIsrqUnwV2KhRo3D48GEsWbIEpqamWLFiBSZPngxbW1usW7cuLWpUhISEwNraWmeZtbU1IiIi8PbtWzx79gzx8fHJtgkJCfngdn18fGBhYaE87Ozs0qR+IiIiyhhSHIB27tyJxYsXo127djAyMkLt2rUxfvx4zJgxAxs2bEiLGtOcl5cXwsPDlQcneiQiIsraUnwK7MWLFyhSpAiA9+N9Ei57r1WrFgYOHJi61f2LjY0NQkNDdZaFhobC3Nwc2bJlg6GhIQwNDZNtY2Nj88HtmpqawtTUNE1qJiIioownxT1ARYoUwf379wEApUqVwtatWwG87xmytLRM1eL+zcXFBX5+fjrLDh48CBcXFwCAiYkJnJ2dddpotVr4+fkpbYiIiIhSHIA8PDxw6dIlAMCYMWOwaNEimJmZYdiwYRg5cmSKtvX69WsEBAQgICAAwPvL3AMCAhAYGAjg/ampnj17Ku2/+eYb3Lt3D6NGjcKNGzewePFibN26FcOGDVPaeHp64pdffsHatWtx/fp1DBw4EFFRUfDw8EjprhIREVEWleJTYInDhqurK27cuIHz58+jWLFiKF++fIq2de7cOdSvX1957unpCeD9XENr1qxBcHCwEoYAwMHBAbt378awYcMwf/58fPXVV1ixYoUyBxAAdOrUCWFhYfD29kZISAgqVKiAffv2JRkYTUREROr1yfMAabVa/Pjjj/jzzz8RExODhg0bYuLEiUluhZEVcB6grImXwX+61LoMnu95ynD6AaIvk5LP708+BTZ9+nSMHTsWOXPmRMGCBTF//nx8++23X1wsERERUXr75AC0bt06LF68GPv374evry927tyJDRs2QKvVpmV9RERERKnukwNQYGAgmjVrpjx3dXWFRqPBkydP0qQwIiIiorTyyQEoLi4OZmZmOsuMjY0RGxub6kURERERpaVPvgpMRNC7d2+dCQPfvXuHb775Bjly5FCW7dixI3UrJCIiIkplnxyAevXqlWRZ9+7dU7UYIiIiovTwyQFo9erVaVkHERERUbpJ8UzQRERERJkdAxARERGpDgMQERERqQ4DEBEREanOJwWgSpUq4eXLlwCAKVOm4M2bN2laFBEREVFa+qQAdP36dURFRQEAJk+ejNevX6dpUURERERp6ZMug69QoQI8PDxQq1YtiAhmz56NnDlzJtvW29s7VQskIiIiSm2fFIDWrFmDiRMnYteuXdBoNNi7dy+MjJK+VKPRMAARERFRhvdJAahkyZLYvHkzAMDAwAB+fn6wsrJK08KIiIiI0sonzwSdQKvVpkUdREREROkmxQEIAO7evYt58+bh+vXrAIAyZcpg6NChKFq0aKoWR0RERJQWUjwP0P79+1GmTBmcOXMG5cuXR/ny5XH69GmULVsWBw8eTIsaiYiIiFJVinuAxowZg2HDhuGHH35Isnz06NFo1KhRqhVHRERElBZS3AN0/fp19O3bN8nyPn364J9//kmVooiIiIjSUooDUP78+REQEJBkeUBAAK8MIyIiokwhxafA+vfvj6+//hr37t1DjRo1AAB///03Zs6cCU9Pz1QvkIiIiCi1pTgATZgwAbly5cKcOXPg5eUFALC1tcWkSZMwZMiQVC+QiIiIKLWlOABpNBoMGzYMw4YNQ2RkJAAgV65cqV4YERERUVr5rHmAEjD4EBERUWaU4kHQRERERJkdAxARERGpDgMQERERqQ4DEBEREanOZwWgwYMH48WLF6ldCxEREVG6+OQA9OjRI+XvGzduxOvXrwEAjo6OCAoKSv3KiIiIiNLIJwegUqVKoXDhwujatSvevXunhJ4HDx4gNjb2i4pYtGgR7O3tYWZmhmrVquHMmTMfbFuvXj1oNJokj+bNmyttevfunWR9kyZNvqhGIiIiyjo+OQC9evUK27Ztg7OzM7RaLZo1a4YSJUogOjoa+/fvR2ho6GcVsGXLFnh6emLixIm4cOECnJyc4ObmhqdPnybbfseOHQgODlYeV69ehaGhITp06KDTrkmTJjrtNm3a9Fn1ERERUdbzyQEoNjYWVatWxfDhw5EtWzZcvHgRq1evhqGhIVatWgUHBweULFkyxQXMnTsX/fv3h4eHB8qUKYOlS5cie/bsWLVqVbLt8+TJAxsbG+Vx8OBBZM+ePUkAMjU11WmXO3fuFNdGREREWdMnzwRtaWmJChUqoGbNmoiJicHbt29Rs2ZNGBkZYcuWLShYsCDOnj2boh8eExOD8+fPK/cUAwADAwO4urrC39//k7axcuVKdO7cGTly5NBZfvToUVhZWSF37txo0KABpk2bhrx58ya7jejoaERHRyvPIyIiUrQfRERElLl8cg/Q48ePMX78eJiamiIuLg7Ozs6oXbs2YmJicOHCBWg0GtSqVStFP/zZs2eIj4+HtbW1znJra2uEhIR89PVnzpzB1atX0a9fP53lTZo0wbp16+Dn54eZM2fi2LFjaNq0KeLj45Pdjo+PDywsLJSHnZ1divaDiIiIMpdPDkD58uVDy5Yt4ePjg+zZs+Ps2bP47rvvoNFoMGLECFhYWKBu3bppWWsSK1euhKOjI6pWraqzvHPnzmjVqhUcHR3h7u6OXbt24ezZszh69Giy2/Hy8kJ4eLjy4FVtREREWdtnT4RoYWGBjh07wtjYGIcPH8b9+/cxaNCgFG0jX758MDQ0TDKAOjQ0FDY2Nv/52qioKGzevBl9+/b96M8pUqQI8uXLhzt37iS73tTUFObm5joPIiIiyro+KwBdvnwZX331FQCgcOHCMDY2ho2NDTp16pSi7ZiYmMDZ2Rl+fn7KMq1WCz8/P7i4uPzna7dt24bo6Gh07979oz/n0aNHeP78OQoUKJCi+oiIiChr+qwAZGdnBwOD9y+9evXqF42Z8fT0xC+//IK1a9fi+vXrGDhwIKKiouDh4QEA6Nmzp84g6QQrV66Eu7t7koHNr1+/xsiRI3Hq1Ck8ePAAfn5+aN26NYoVKwY3N7fPrpOIiIiyjk++CiytdOrUCWFhYfD29kZISAgqVKiAffv2KQOjAwMDlbCV4ObNmzhx4gQOHDiQZHuGhoa4fPky1q5di1evXsHW1haNGzfG1KlTYWpqmi77RERERBmb3gMQ8P7eYoMHD052XXIDl0uWLAkRSbZ9tmzZsH///tQsj4iIiLIY3g2eiIiIVIcBiIiIiFSHAYiIiIhUhwGIiIiIVIcBiIiIiFSHAYiIiIhUhwGIiIiIVIcBiIiIiFSHAYiIiIhUhwGIiIiIVIcBiIiIiFSHAYiIiIhUhwGIiIiIVIcBiIiIiFSHAYiIiIhUhwGIiIiIVIcBiIiIiFSHAYiIiIhUhwGIiIiIVIcBiIiIiFSHAYiIiIhUhwGIiIiIVIcBiIiIiFSHAYiIiIhUhwGIiIiIVIcBiIiIiFSHAYiIiIhUhwGIiIiIVIcBiIiIiFSHAYiIiIhUhwGIiIiIVIcBiIiIiFSHAYiIiIhUJ0MEoEWLFsHe3h5mZmaoVq0azpw588G2a9asgUaj0XmYmZnptBEReHt7o0CBAsiWLRtcXV1x+/bttN4NIiIiyiT0HoC2bNkCT09PTJw4ERcuXICTkxPc3Nzw9OnTD77G3NwcwcHByuPhw4c662fNmoWff/4ZS5cuxenTp5EjRw64ubnh3bt3ab07RERElAkY6buAuXPnon///vDw8AAALF26FLt378aqVaswZsyYZF+j0WhgY2OT7DoRwbx58zB+/Hi0bt0aALBu3TpYW1vD19cXnTt3TpsdSQH7Mbv1XUKm8eCH5vougYiIsiC99gDFxMTg/PnzcHV1VZYZGBjA1dUV/v7+H3zd69evUbhwYdjZ2aF169a4du2asu7+/fsICQnR2aaFhQWqVav2wW1GR0cjIiJC50FERERZl14D0LNnzxAfHw9ra2ud5dbW1ggJCUn2NSVLlsSqVavwxx9/4Ndff4VWq0WNGjXw6NEjAFBel5Jt+vj4wMLCQnnY2dl96a4RERFRBqb3MUAp5eLigp49e6JChQqoW7cuduzYgfz582PZsmWfvU0vLy+Eh4crj6CgoFSsmIiIiDIavQagfPnywdDQEKGhoTrLQ0NDPzjG59+MjY1RsWJF3LlzBwCU16Vkm6ampjA3N9d5EBERUdal1wBkYmICZ2dn+Pn5Kcu0Wi38/Pzg4uLySduIj4/HlStXUKBAAQCAg4MDbGxsdLYZERGB06dPf/I2iYiIKGvT+1Vgnp6e6NWrFypXroyqVati3rx5iIqKUq4K69mzJwoWLAgfHx8AwJQpU1C9enUUK1YMr169wo8//oiHDx+iX79+AN5fIfb9999j2rRpKF68OBwcHDBhwgTY2trC3d1dX7tJREREGYjeA1CnTp0QFhYGb29vhISEoEKFCti3b58yiDkwMBAGBv/rqHr58iX69++PkJAQ5M6dG87Ozjh58iTKlCmjtBk1ahSioqLw9ddf49WrV6hVqxb27duXZMJEIiIiUieNiIi+i8hoIiIiYGFhgfDw8DQZD8R5gD5das4DxPf906XW+873PGU47xXRl0nJ53emuwqMiIiI6EsxABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6jAAERERkeowABEREZHqMAARERGR6mSIALRo0SLY29vDzMwM1apVw5kzZz7Y9pdffkHt2rWRO3du5M6dG66urkna9+7dGxqNRufRpEmTtN4NIiIiyiT0HoC2bNkCT09PTJw4ERcuXICTkxPc3Nzw9OnTZNsfPXoUXbp0wZEjR+Dv7w87Ozs0btwYjx8/1mnXpEkTBAcHK49Nmzalx+4QERFRJqD3ADR37lz0798fHh4eKFOmDJYuXYrs2bNj1apVybbfsGEDBg0ahAoVKqBUqVJYsWIFtFot/Pz8dNqZmprCxsZGeeTOnTs9doeIiIgyAb0GoJiYGJw/fx6urq7KMgMDA7i6usLf3/+TtvHmzRvExsYiT548OsuPHj0KKysrlCxZEgMHDsTz588/uI3o6GhEREToPIiIiCjr0msAevbsGeLj42Ftba2z3NraGiEhIZ+0jdGjR8PW1lYnRDVp0gTr1q2Dn58fZs6ciWPHjqFp06aIj49Pdhs+Pj6wsLBQHnZ2dp+/U0RERJThGem7gC/xww8/YPPmzTh69CjMzMyU5Z07d1b+7ujoiPLly6No0aI4evQoGjZsmGQ7Xl5e8PT0VJ5HREQwBBEREWVheu0BypcvHwwNDREaGqqzPDQ0FDY2Nv/52tmzZ+OHH37AgQMHUL58+f9sW6RIEeTLlw937txJdr2pqSnMzc11HkRERJR16TUAmZiYwNnZWWcAc8KAZhcXlw++btasWZg6dSr27duHypUrf/TnPHr0CM+fP0eBAgVSpW4iIiLK3PR+FZinpyd++eUXrF27FtevX8fAgQMRFRUFDw8PAEDPnj3h5eWltJ85cyYmTJiAVatWwd7eHiEhIQgJCcHr168BAK9fv8bIkSNx6tQpPHjwAH5+fmjdujWKFSsGNzc3vewjERERZSx6HwPUqVMnhIWFwdvbGyEhIahQoQL27dunDIwODAyEgcH/ctqSJUsQExOD9u3b62xn4sSJmDRpEgwNDXH58mWsXbsWr169gq2tLRo3boypU6fC1NQ0XfeNiIiIMia9ByAAGDx4MAYPHpzsuqNHj+o8f/DgwX9uK1u2bNi/f38qVUZERERZkd5PgRERERGlNwYgIiIiUh0GICIiIlIdBiAiIiJSHQYgIiIiUh0GICIiIlIdBiAiIiJSHQYgIiIiUh0GICIiIlIdBiAiIiJSHQYgIiIiUh0GICIiIlIdBiAiIiJSHQYgIiIiUh0GICIiIlIdBiAiIiJSHQYgIiIiUh0GICIiIlIdBiAiIiJSHQYgIiIiUh0GICIiIlIdBiAiIiJSHQYgIiIiUh0GICIiIlIdBiAiIiJSHQYgIiIiUh0GICIiIlIdBiAiIiJSHQYgIiIiUh0GICIiIlIdBiAiIiJSHQYgIiIiUp0MEYAWLVoEe3t7mJmZoVq1ajhz5sx/tt+2bRtKlSoFMzMzODo6Ys+ePTrrRQTe3t4oUKAAsmXLBldXV9y+fTstd4GIiIgyEb0HoC1btsDT0xMTJ07EhQsX4OTkBDc3Nzx9+jTZ9idPnkSXLl3Qt29fXLx4Ee7u7nB3d8fVq1eVNrNmzcLPP/+MpUuX4vTp08iRIwfc3Nzw7t279NotIiIiysD0HoDmzp2L/v37w8PDA2XKlMHSpUuRPXt2rFq1Ktn28+fPR5MmTTBy5EiULl0aU6dORaVKlbBw4UIA73t/5s2bh/Hjx6N169YoX7481q1bhydPnsDX1zcd94yIiIgyKr0GoJiYGJw/fx6urq7KMgMDA7i6usLf3z/Z1/j7++u0BwA3Nzel/f379xESEqLTxsLCAtWqVfvgNomIiEhdjPT5w589e4b4+HhYW1vrLLe2tsaNGzeSfU1ISEiy7UNCQpT1Ccs+1ObfoqOjER0drTwPDw8HAERERKRgbz6dNvpNmmw3K0rNfwO+758utd53vucpk1rve7mJ+1NlO2pxdbKbvkugVJLwOyQiH22r1wCUUfj4+GDy5MlJltvZ2emhGkrMYp6+K1Anvu/6wfddP/i+Zz2RkZGwsLD4zzZ6DUD58uWDoaEhQkNDdZaHhobCxsYm2dfY2Nj8Z/uEP0NDQ1GgQAGdNhUqVEh2m15eXvD09FSea7VavHjxAnnz5oVGo0nxfmU2ERERsLOzQ1BQEMzNzfVdjmrwfdcPvu/6wfddP9T2vosIIiMjYWtr+9G2eg1AJiYmcHZ2hp+fH9zd3QG8Dx9+fn4YPHhwsq9xcXGBn58fvv/+e2XZwYMH4eLiAgBwcHCAjY0N/Pz8lMATERGB06dPY+DAgclu09TUFKampjrLLC0tv2jfMiNzc3NV/IJkNHzf9YPvu37wfdcPNb3vH+v5SaD3U2Cenp7o1asXKleujKpVq2LevHmIioqCh4cHAKBnz54oWLAgfHx8AABDhw5F3bp1MWfOHDRv3hybN2/GuXPnsHz5cgCARqPB999/j2nTpqF48eJwcHDAhAkTYGtrq4QsIiIiUje9B6BOnTohLCwM3t7eCAkJQYUKFbBv3z5lEHNgYCAMDP53sVqNGjWwceNGjB8/HmPHjkXx4sXh6+uLcuXKKW1GjRqFqKgofP3113j16hVq1aqFffv2wczMLN33j4iIiDIejXzKUGnK0qKjo+Hj4wMvL68kpwIp7fB91w++7/rB910/+L5/GAMQERERqY7eZ4ImIiIiSm8MQERERKQ6DEBERESkOgxARERElCoy07BiBiDK8LRarc7zzPQLRkSkFlqtVrl7wr179/RczccxAFGGptVqlXmgTp8+jXfv3qni9iSUfhioMz/+G2YMCcfqMWPGYMKECXj69KmeK/pvDECUYSUOPxMmTMDgwYOxefNmaLVaHvAoVdy9exdTp06Fh4cHNmzYgKCgIH2XRJ8g4ff/3bt3Or0OpB+Jj8dnz57F3r17MWTIEFhZWemxqo/jPECU4Y0dOxbLly/Hb7/9hnLlyiFfvnz6LknvRAQajQZBQUEwMTGBgYEB8ufPrxMa6b9dunQJjRs3RsmSJREcHIyHDx+id+/emDZtWoY/cKtZwv/9Xbt2Yf369QgPD4enpyeqVq2qyns4ZiQ//vgj7t27h/j4eOX2VBkZj5SUoV25cgU7d+6Er68v6tWrB0NDQ9y4cQNz5szBtWvXAKiz+1uj0WDHjh2oU6cO6tSpgxYtWuCvv/6CgYFBkjFTlNTVq1dRo0YNfPfddzh48CBu376NCRMmYM2aNbh8+bK+y6P/oNFo8Pfff6Nr167InTs34uLi0LlzZyxevBjBwcH6Lk/VHj9+jGXLluHMmTMZ/vQXwABEGcy/P7wNDQ0RGhqKN2/e4OrVqxg3bhzc3d2xaNEiVK1aFVeuXFFV93dC2AsMDMQ333yDkSNHYuTIkShVqhQaNWqEQ4cOMQR9xPPnz1GvXj04Oztj1KhRyu0Bhg0bBisrK9y9e1fPFdLHhISEYMSIEVi6dCkOHTqk/H3FihUMQekkuWPMvHnzMGXKFFy+fBkbNmxAVFSUHir7dHq/GSpRYgmnb65evQoHBwcUKlQIjRo1Qp8+ffDy5Ut4eHhg2rRpaN++PRwdHbFr1y44Ojrquer0o9FocPToUdy7dw/9+/fHoEGDAADNmzeHmZkZmjRpgr1796JRo0Y8HfYBefPmRYcOHXDkyBEsW7YMnTp1UoLP06dPUahQIX2XSP+ScNrrwoULePDgAU6dOgU7Oztl/dixYwEAS5YsgaGhIXr16oWCBQvqq9wsL/Gx5dy5c3j79i20Wi3q1q2L8ePHIzIyEqNGjUL27NnRo0cPZM+eXc8Vf4AQZQDx8fHK37dt2yZFixaV9evXi4hIcHCw7Nu3T44fP660e/v2rVSvXl3WrFmjl3r1JTIyUtq3by8ajUbatGmjs+7Jkyfy9ddfi5mZmezatUtPFWZsif+fDRkyROzt7WXNmjVy4cIF+eqrr2TIkCF6rI7+y44dO8TExERKlSolGo1G6tWrJ3fu3NFp88MPP0i2bNlk1qxZEhcXp6dKszatVqv8fcyYMVK2bFmxt7eXKlWqSIMGDZR1Xl5eYmxsLMuXL5fIyEh9lPpRDECkd4k/lDZu3CgzZ84UQ0NDKV68uGzevFnevHmjrH/79q3cvHlTWrRoIZUqVZLY2Fh9lKxXZ8+ele7du4uZmZmcP39eRP53UAoODpYuXbpIvnz5JCoqSudgRe8l/mD87rvvxM7OTiwtLcXDw0NZnvj/JOlfUFCQ9OzZU5YtWyYREREyd+5cKVeunAwdOjRJCJo7d67cunVLT5Wqx5w5cyRv3rzi7+8vsbGxMnXqVNFoNHLo0CGlzZgxY0Sj0Yivr68eK/0wBiDKMMaNGyd58uSRlStXyuLFi6V69epSpEgR2bBhg7x9+1ZERNasWSNNmjSRWrVqSUxMjIhIlv6mlxBg4uLiJDo6Wll+8+ZNadmypVhbWycJQSEhIfLkyZP0LzaDS/z/JPHfx44dK+bm5jJv3jx58eKFPkqj/xAQECCtW7eWBg0ayMOHD5Xl8+fPlwoVKsjgwYPl7t27eqxQfWJiYqRnz56yYsUKERH5448/xNzcXJYvXy4iIhEREUrbxYsXZ9gvqgxAlCEEBgYqYSexJk2aiJ2dnWzatEni4+Plxo0bsm3bNuUDLKP+YqWGhECzd+9e6dSpk9SqVUsGDhwoZ8+eFRGRO3fuSJs2bcTGxkYuXryo8xp678GDBzJx4kR59+6diOj27CQOQUOGDBEHBwdZuHChhIWFpXudpCvh//GtW7fEx8dHKlWqJJaWlnLlyhWddvPnz5cqVapI79695f79+3qoVB3+fVyJi4uTqlWryqpVq2Tfvn2SM2dOWbx4sYi8PybPnj07ybE8Ix6rOUKSMgRjY2OICExMTAC8n+AMAPbu3QtTU1PMnDkTv/32G4oVK4b27dvD0NAQ8fHxMDLKuuP4NRoNdu7cidatW8PS0hI1a9aEn58fhg4dik2bNqFo0aKYMWMG6tSpg6pVq+Ly5cuquiLuU2zbtg2//vorJk6ciJiYGJ0r5AwNDREXFwcAmD9/Ptq0aQMvLy/s2LGDV9HpmUajwe+//47GjRujWbNmGDlyJOzt7TF69Ghcv35daTdkyBC0a9cO9+7dQ7Zs2fRYcdaWcFz59ddfceLECRgaGqJu3brYuHEjOnbsiB9//BEDBw4EADx79gxHjhzBq1evdLaRIY/V+k5gpD7Jja+Ii4sTJycn6dixo7Is4ZRPy5YtxcHBQcqVKyeXLl364DYyu8TdxlqtVl6+fCk1a9aU6dOnK8tfvHghbdu2FRcXF+Xb8Pnz56Vnz55y8+bNdK85o7p//774+flJXFycTJ8+XSpXriwjRoxQ/k99qCdo6tSpHD+iRwk9DZGRkdK3b1+ZPXu2sm79+vVSr149adeundy4cUPndTx1mfYePnwoJUqUkEWLFomIyIkTJ8TKykqqVaumHHuePHkizZo1ExcXl0wxNIEBiNJV4g+e69evS3BwsDx79kxERPz8/CRnzpzy3Xff6bymR48ecu7cOSlbtqxOQMpKfvzxR5k4caLOQePNmzdSvnx5WbhwoYiIMubp5cuXYm9vLyNGjFDaJh4fpHaPHz+WfPnySfHixeWPP/6Q+Ph4mTJlygdDUHR0tIwdO1Z++uknPVZNCf7++28pVaqU1KxZU44fP66zbt26dVKvXj3p1KmTXL16VU8Vqtfo0aOlUKFCSuA8dOiQ5M+fX5ydnaVkyZJSo0YNqVy5cqYZn5kB+6QoK0uYO8LLywu//fYboqKi0KRJE/Tv3x8NGjTAwoUL8e233+LixYsoWrQobt68iRcvXsDZ2Rlubm64dOmSnvcgbcTGxqJLly4wNDRETEwMTExMEB8fDwBKl7+RkRFiY2NhaWmJRo0a6UzYl3DqkIBbt27hxYsXcHBwwC+//IK4uDiMGzcOAPDnn39i3LhxmD59OkxMTPD27VuMHDkSS5cuRUBAgH4LJwBAsWLFYG5ujpMnTyqzCSfMO9OjRw8YGBhg9uzZmDVrFlasWAFjY2M9V5z1/HsOsdjYWBgbG6Nfv344cuQIfH190bt3bzRs2BBHjhzB1atXcffuXZQuXRqtWrVSTi9nyNNeiek7gVHWp9VqdQbR7d69W7766ivZu3evzJo1S5o3by41a9aUEydOiIjIlStXpGvXrtK1a1cZMGCA8m2iXbt20rt3b4mPj8+yg32PHz8uo0aNkqCgIBER2bp1qxgaGsqCBQt02rVp00YGDBiQZd+HL9WnTx+pUKGCtGvXTurWrSu+vr5JeoIiIiJk+PDhkj17duVKOsoYQkNDpUqVKlKyZEnllGTi/+ubN2+WBw8e6Ks81Vi7dq3cv39fuQpXq9VKu3btpGHDhv/5uoze85OAAYjS1Z9//ilDhw6Vn3/+WVnm5+cnbdq0kRo1asjhw4eTvCY8PFyGDx8u+fPnl3/++Sc9y00ziU8FJr46YubMmVKsWDEZN26cBAcHi4jItGnTxMDAQPr06SNTpkyRQYMGSc6cOeXatWvpXndGl3C11+7du6V3796yf/9+adu2rdSsWVPndFi1atWkaNGiOnMpUfpLfLXXiRMn5NKlS8op8bCwMHFychJHR0flMncG/vRz69YtqVChglhaWkr//v1l27ZtyvLixYvL2rVr9Vzhl2MAojTTuXNnZZ4IEZHLly9L1apVxdLSUmbOnKnT9vDhw9K2bVupXbu27N69W1l+584dmTp1qpQoUUK51DurCAoKUg7of/75p8yfP19ERKZMmSIVKlSQMWPGKB8Gvr6+UqNGDaldu7Y0a9ZMGQxO76dQ2LFjh86yp0+fSqlSpWThwoXy9OlTadu2rdSqVUsJQWPHjpUyZcrwfdSjhP/727dvFxsbGylRooSYmppKkyZNZPPmzSLyvxBUqVIlDk5PY/8Olwlf0lavXi3ffPONmJiYSNeuXWXq1KnSo0cPGTVqlE67zIgBiNLEkydPZMGCBcrpqwSbNm2SqlWriqOjY5JAc+TIEalbt64MGDBAWabVauXGjRtKb0hW8ebNGylXrpzUq1dPtmzZIhqNRjZu3KisnzRpkhKCEvY9YUbsxDNjq11gYKDkzZtXNBqNNGvWTLZs2aJckfLnn39K7dq15enTp/LPP/9I27ZtpX79+rJ161bRarVKuCT9OX36tJibm8uiRYvkyZMncvDgQenWrZs4OzvL1q1bReR9mLW3t5eaNWsmOZ5Q6kgcYh4/fix37tzRubBCq9XKuXPn5Ouvv5a6deuKRqMRY2NjCQgI0Ee5qYYBiFLd69evReR/Vy0tXrxYvLy8lPVbtmyR+vXri7u7e5JfoPPnzyu/jJn5m8XHaLVauX79uuTJk0fMzMyUe5olnMIReR+CKlasKOPGjZPAwECd19J7Dx48kMqVK4uLi4tUqlRJ+vXrJ4ULF5Zly5bJli1bpEWLFrJnzx4REbl27Zq4urpKs2bNMuy9idQi4dgwb948qV27ts66gIAA6dixo7i7uyvHkrCwMLl3716616kGiY+z3t7eUq1aNcmWLZv06NFDuR9jgrdv30pkZKTMmjVLKlWqJMOGDcvUYzIZgChVjR49WmxsbOTVq1ciIvL8+XMZOnSoFC1aVGbMmKG027BhgzRo0EDc3d2TPQ2RlcNPgocPH4qRkZHkzJlTmjdvrixP/M1rypQpUrhwYZk8eXKmGViY3m7duiVt27YVd3d32bFjh/z+++9Sr149cXd3F41GI9WqVVPe0xs3bigDzCl9PXjwQOcYICKyZMkSKVWqlISGhuos/+OPP8TY2JhzW6Ujb29vyZ8/v+zYsUNOnTol9erVE0dHR2WGZxHd8Yo+Pj5SvHhxZYB0ZsQARKnqwIED4uLiIhUrVpSXL1+KiMjdu3dlwoQJUrJkSZk6darSdsOGDdKoUSOpVauW3L59W08V69ft27clICBAbG1txc3NTVmeOAQtWLCA9zr6iBs3bkjTpk2lcePGcvPmTXn9+rX4+/tLixYtlG+xmfVbalYQHx8v06dPF3t7e/H29laW7969W8zNzWX9+vU6X3pu3LghpUuXTnLrC0obx48fF0dHR/nrr79EROTYsWNiZmYmtWrVkvLly8svv/yitE3ovUs4NXn69Gm91JwaGIAo1Z04cUIaNGggTk5OEh4eLiLvQ9C4ceOShKBffvlFhgwZoooen4QP4Bs3bshff/2lc2PHEydOiK2trTRt2lRpN2/evCSXv9OH3bp1Sxo3biyNGzdWplSgjCMkJES8vb2lYsWKMnbsWGX5999/r5wGfvDggbx7905GjhwpRYsWladPn+qxYvUIDg6Wn376SWJiYuTAgQOSN29eWbVqlYSEhEjRokWlZMmSSS5cmTx5suTOnTtTj89kAKJUkTjAbN++Xby9vUWj0YiLi4vSE3Tv3j0ZN26clC5dWuf2DsltI6vavn27WFhYiIODgxgbG8uCBQuUWVVPnDghX331lZQpU0Z69OghRkZGcvnyZT1XnLncunVLmjRpIm5ubklmESb9CwkJkbFjx0rFihVlzJgxyvJhw4ZJ3rx5xc7OTipXriz58+eXCxcu6LFS9YmMjJTY2Fhp27atjBs3Tjnl3qZNGylXrpwMGTJEpxd10aJFmX4KCQYgSlWenp7KPDZt2rQRW1tbcXJy0glBEyZMkNy5cysDf7O6hIPGgwcPpGLFirJkyRK5d++eTJ06VXLmzClTp05Vrki6d++edOvWTXr37s3w85lu3bolLVq0kOrVq4u/v7++y1GtmzdvyvLly8XPz09E/nfqJDg4WMaPHy/ly5fXCUFHjhyRTZs2KZPvUer72HidmJgYqVSpkvLvEh0dLV26dJHNmzcrx7GsNBaRAYi+SOJvBBcuXBBbW1s5dOiQsmzXrl3i7OwsFStWVAZG37p1S5YtW5alfpE+5tChQzJ37lydma1FRGbPni25cuWSqVOn6gwETXw1GKXc9evXpX379jqnGSn9PH/+XHLmzCkajUbMzc2latWq0qtXL9m7d6+EhoaKVquVcePGSe3atWXkyJH6LlcVtm3bJp07d9a5ojQxrVYrkZGR0qtXL2nQoIF8//330rBhQ6lYsWKWvTI3g9+ogzKq9u3bo0WLFujduzdEBBqNBuHh4QgPD0ehQoWUdo0bN8br16/Rs2dPtGzZEr6+vihevDiKFy8OAIiPj4ehoaG+diPd7Nq1C/Pnz0fJkiXx8uVLWFlZAQCGDx8OjUaD6dOn482bNxg6dCisra1hamqq54ozt1KlSmHDhg28R5qe5MmTB0OHDsWyZcvQo0cPREdH4+3bt+jatSssLCzQsGFDWFlZoUSJEti3bx9MTU0xdepUfZedpVlZWWHLli3Ili0bpk6dioIFC+qs12g0yJkzJ4YOHYoFCxYgICAAefLkwd69e2FgYJDk/mBZgr4TGGU+b9++lQEDBoiRkZEyWZnI+/P7jo6OMn/+fJ2eoRcvXki5cuXEyMhIunXrJiLqvCJn0qRJotFoZPHixRIVFaWzburUqVKoUCFOzkeZXuJeghEjRkjp0qXlp59+kri4OLl9+7bs2bNHWrRooUyop9FopGDBgvy/n4YSetv//vtvMTExkZ49e8qjR4+U9f8+Hj958kRnfp/El79nJQxA9FkiIyNl1KhRYmBgoNwjJioqSrp37y516tSR33//XWkbGhoqbdu2lf3792e5LtTkJBw0goKC5Pbt23L16lVl3dChQ8XExERWrlyZZEbn58+fp2udRGkl8entESNGiJ2dncyePVsJOTExMRIbGyu7du2SOXPmyPXr1/VVqmokHHuPHz+ebAgSeT8+q0qVKvLDDz8oy7Lyl1UGIEqRxN8ELl68KJ07dxZDQ0Ml8ISEhEjDhg2levXq0q9fP1m1apXUrVtX6tWrp/wCZuWxPwkHix07dkilSpXEwcFBqlevLm3atFHaDB8+XExMTGT16tU6PUFZ+UBD6pP493zUqFFSqFAhmTNnTpJJDyn9/DsE9ejRQwlBYWFhUrduXSlSpIhqbjnCAESfZcyYMVKlShVp1aqV5MmTRwwMDJR7WT19+lQmTZoktWrVEmdnZ2nZsqXyC6WGHqCDBw9KtmzZZMmSJRIYGCirVq0SjUajc/fkESNGiEajSTLVPFFm8+/TuYn9OwQVLlxYfvrpJ87vk8YSjrMJX6oSf7n6dwjq3bu3XLx4UWrXri2lSpVSjtVZ9bRXYgxAlGKbNm2SHDlyiL+/v0RFRck///wjgwYNEgMDA9m0aZOIiHL++Pnz51n+PHJiWq1WRo0apVxG+ujRIylcuLB8++23SdqOHTtW/vnnn/QukSjV3Lp1Szp16iTLly//YJvEIcjLy0ty5colixYtUsWXIX1IfGr93zecTpDw3p84cUKyZ88uGo1GnJycVBV+RESy2JBuSg+PHj1C5cqVUb16dWTPnh2lS5fG5MmT0b17d/To0QO7d++GgYEBNBoN8uTJA41GAxGBkVHWuehQRJJdrtFocOXKFZiamiIsLAzVq1eHm5sbFixYAADYsGED1q5dCwCYPn06SpcunW41E6WmK1euoF69esiRIwe0Wu0H2xkaGiI+Ph4AMGPGDHh6eqJx48ZZ74qiDGDr1q3w9vYGAHz//fdo3bo1Xr16laRdwlVdNWvWxP79+9GgQQOcPXsWxsbGiIuLy1LH6v/C/4GUYpaWlggICEBISAiA92EgX758aNOmDeLj49GyZUv4+fnpvEaj0eij1DSj0WgQFhaG58+fAwB8fX2xbds2AED9+vVx8+ZNVKpUCU2bNsWyZcsAAG/fvsVff/2F+/fvIyYmRm+1E32pu3fvolmzZujVqxeWLVuGAQMGJNsuIRglDkGTJk1CsWLF0q1WNYmJicGcOXPg4uKCtWvXYvfu3bC0tEz2C5uBgQHi4+NRq1YtHDp0SHXhB2AAov/woW91rq6uKFGiBKZPn46goCAl3BQsWBB9+vTBL7/8grp166ZnqelKRBAeHo7SpUtj/vz5WLVqFdq2bYu4uDgAQO3atXH06FFkz54dw4cPB/D+wDRt2jTs2bMH3bp14/w0lKlt374dlStXxoQJE5R5vB4/fozTp09j6dKlOH/+PGJiYnR6edQw35e+yPvhLOjevTsaN26MM2fOoG3btihRogSAD38B/fe/iZrCDwBo5EN9+aRqiSe9WrFiBa5fv45Xr17Bzc0N7du3x8qVK7Fq1SoUKVIEgwcPRs6cOTFmzBjkypULmzdvBoAs/21ix44d6Ny5M+Lj47FgwQIMGjRImRTy4MGDaN++PSpVqoT4+Hjky5cPJ06cwP79+1GxYkV9l070RXr06IGnT59i//79AN4Hoi1btuDw4cN48+YNbG1tMWzYMAwaNCjL9f5mNAnHnATTpk2DiGDixIkYNWoURo4cibx58370dWqUdT+d6IskhJ9Ro0Zh7dq16NevH2JiYjBmzBicPHkS8+bNQ2xsLHbt2oWaNWuiePHiyJEjB3x9fQEgy435SRwIo6OjYWpqivLlywN4v6/Pnj3Ds2fPkC9fPogIGjVqhIMHD+LkyZO4ePEinJ2dMXPmTGUGbKLM5t27dzAyMoKRkREaN24MLy8vTJgwAc+fP8dvv/2Gjh07YsOGDXBzc0O7du2wceNGeHh4IHv27PouPUtLCDGrV6+GqakpvLy8YGhoiEKFCsHDwwPA++N4njx5AADHjx9H7dq1VR9+AHAmaPqwgwcPSpEiReT06dMi8n5uGzMzM1m9erVOu7Nnz8qlS5eUKwuy6hUEgYGB8uDBAxER+fPPP2XVqlVy9epV2bZtm2g0GvHy8uJstpQlXb16VVxdXZUbmwYGBsq4ceOkfPnyUrFiRdm5c6fOpe3Lli2TMmXKcHLPdPLu3TupU6eOVK5cWdatW6dczbV27VrRaDTi6ekpf//9t7Rq1UoqVarEOcf+HwMQfdCvv/4q1atXF5H3N9LLlSuXLFmyREREIiIi5PDhw0kuZc2qkxxGRkaKu7u7VK1aVRYtWiQajUbnNiAJB5rx48dLWFiYiIjMmjVLtm/frq+SiVJFfHy81KhRQzQajVSqVEmOHz+urHv79m2y8wANHDhQ2rVrl2S2c0odyQWYly9fSuvWrcXFxUXWrl2rhKBff/1V8ufPL+XKlZMqVaqoZpLDT8EAREkkhJjNmzdLu3btZNeuXZIzZ04l/IiI/PHHH+Lp6SnBwcH6KjPdHThwQMqWLStGRkYyZ84cERGJjo5WDkZr164VIyMj6dq1q3Tr1k1MTEw+OA8HUWbi6+srrq6uUrp0abGxsZG//vpLOU4k/hIUHh4uY8aMkXz58uncAobSxv3793Wev3r1Slq2bCkuLi6yfv16Jez8888/cv78+SzfS59SvApM5UREuTw1QcKVAVWrVsX+/fvRsmVL/Pzzz/jmm28AvB8LsGTJEjx//hzW1tbpXnN6SO4KuJIlSyImJgYODg7YvXs37t+/DxMTE8TFxUFE0LNnT2zcuBFRUVGIiIjA2bNnUaFChfQvniiV2dvbAwCWLl2qXAhx8uRJAP8bL7hkyRL06dMHmzdvxoEDB1C2bFl9lasKq1evRrt27XDw4EFlmYWFBdauXQsjIyNMmzYNW7duRWxsLEqXLo1KlSopl75npfGZX4JXganYixcvlIFxAPDrr7/i7t27yJMnD+rUqQMnJyfs2bMHnTt3Rrdu3dCmTRtotVrMnTsXISEhuHDhAoyMjLLs1QQ3btzA+vXr8fXXX6NQoUIAgAcPHuDmzZuYNWsW4uPjsWbNGjg4OCA2NhbGxsYAgPj4eMTFxcHU1FSf5RN9tpiYmCRTNQwdOhRnzpyBn58funXrhlOnTmHr1q2oXbs2QkJCsGnTJjx79gx9+vRB0aJF9VS5ejx8+BCtWrWCtbU1Ro0aBVdXV2XduXPn0KBBAxQqVAizZ89GkyZN9FhpBqbX/ifSm7Fjx0qLFi3kyZMnIiLi6ekplpaWUrVqValQoYIYGRkp96n6448/xMHBQezs7KRy5crSpk0bpWs1q475iYmJkSpVqohGo5HixYuLp6enctd7EZH9+/dLnTp1pF69enLv3j0REZk9e7YsXbqUU/xTpnbp0iWpWbOm+Pj46Iz3efLkiTRu3FhOnTolIiJubm5SoEABpU1MTIxER0frpeas7kPHlAcPHkilSpWkfv36cuDAAWX5oUOHpFevXjJixIgse4xODewHUykLCwuEh4djzJgx8PDwwI0bN3Dw4EE4Ozvj+fPn+Pnnn+Hh4YGcOXPC3d0dtWrVQnh4OMzMzGBjYwONRpOl5/kxNjZGhw4d0KVLF5QrVw5///03vv76a+zYsQMNGzZEnz59EB8fj6VLl6JOnTpo0KAB1q9fj4CAAE7xT5mWVqvFoEGDcPLkSRgbG8PHxwfffPMNqlevjjZt2iB37txYsmQJqlWrhn379qF169Zo0KABjh07BhcXF32XnyWJiHJM+fXXX3H79m3kzp0bderUQaVKleDr64s2bdpg5syZuHPnDho0aID58+ejYsWKmDx5MoD3vdKciDIpngJTGUl0umrJkiXYtm0bsmXLhvDwcOzZswfm5uZK22HDhmHbtm3w9/eHnZ2dznYSz4uTVR09ehStW7eGn58fKleujODgYCxfvhw+Pj6oVq0aevToAY1Gg6dPnyIgIADe3t4c90CZXlhYGOrUqYO8efOiV69eOH78OB48eAALCwvUq1cPU6dOxa5du1CrVi0AQKdOnTB16lRl1mFKPYmP1yNHjsSKFStQqlQpREdH4/Lly1i2bBn69u2LoKAgDBkyBAEBAYiNjcVXX32F48ePw9jYOMsOUUgV+ux+Iv1I3J36888/S4UKFcTCwkIePXokIv87rXX06FGxtbWVgIAAvdSZEYwYMUK6desmb9++FRGRTp06SalSpaR79+7SsGFDMTY2llWrVrHrn7KEhCsaQ0NDxcbGRtq0aSOnT5+W58+fS//+/cXV1VU0Go34+/vruVJ1uXDhgrRs2VLOnDkjIiLPnj2TSZMmiaGhoWzatElE3l8BdvXqVTl+/LhyDOfVXv+NAUhFPnQeedmyZVKiRAlxd3dXJvoTEbl165bY2dnJsWPH0qvEDGfbtm3i4uIi8fHx0rdvX7G2tlYu771+/br89NNPvNyXMrVnz57JtWvX5PLlyzrLQ0JCpECBAlKrVi3luBAcHKzqL0TpJfE8Pxs2bJBatWqJi4uLvHr1SqfdiBEjxNbWVue4nYBjfz6Op8BUIvEpq0OHDsHMzAzZsmWDs7MzAGDhwoXYuHEjjI2NMWnSJMTExGD+/PkICQnB2bNnVX3+uG7dujhx4gRsbGywZ88eODk56bskolRx9epV9OnTB2FhYRARNG7cGMuXL1fWh4aGwtnZGYULF8bKlStRqlQpALyPVFr69/CCZcuWYeHChXj06BEuXbqEQoUKKWN6jh8/js6dO2P37t2ccuMzZO1BHKRI+IUaPXo0OnbsiM6dO6NXr15YsGABAGDw4MHo0aMHgoKC0KpVKyxYsABlypTB6dOnYWhomGSuIDVI+G4wevRoFCtWDIsWLYKTkxP4nYGygkuXLqF69eqoU6cOVq9ejRYtWmDt2rVYsmQJgPeXwltbW+P8+fN4+PAhBg0ahMuXLwP48N3F6cscPnwYYWFhAIAxY8Zg6tSpGDBgAIYPHw5ra2sMGTIE9+/fV76Q2trawtDQEOHh4fosO/PSZ/cTpb3EXak3b96UqlWrysWLF+X48ePi7e0tFhYWMmvWLKXNsmXLpFSpUjJ9+nTltWo/jxwSEiLFihWT8ePH67sUolRx+/ZtMTMz0/k/fe/ePTExMZHhw4cnaR8SEiKmpqbSrFkzjndLI+Hh4fLVV19J9erVpV+/fmJubi6XLl1S1i9dulRcXFykVq1acvDgQdm7d680a9ZMKlasyNNdnylrXsNMAHS7Ut+9e4fIyEiUK1cOjo6OMDQ0RPHixWFsbIxp06ZBo9FgxIgR+Prrr2FhYYEOHTpAo9Fkubu6fw5ra2tMnDgR33zzDVq2bImqVavquySiz6bVarFq1SrkypULefPmVZZv3rwZsbGxuH37NubNm4e8efMqxwFra2s8fPgQERERSSZIpNRhbm6OGzduwNraGleuXIGvry/Kly+vnO4aMGAADA0N4ePjg1atWqFRo0YoX748tm/frvTSq3mowudQ9ydbFpcQfiZPnoxDhw4pszYn/JJYW1ujf//+AIAZM2YgMjISkydPRqdOnQCo41L3T1W/fn1UqVIFtra2+i6F6IsYGBhg8ODBePPmDTZv3gxTU1NERkbixx9/xLhx41ChQgVs2LABQUFBGDduHIoXL47vvvsO7u7uWfbWN/qUcJwVETx79gwGBgbIlSsXpk6dilKlSuGrr75S2vTr1w8GBgZYuXIlLC0t8c0338DMzAzR0dGcef4zcBB0FpQ4uCxYsAAzZsyAh4cHHj9+jPXr12PKlCkYP3680v7p06eYO3cuLly4gP379wPgOf7kvHv3DmZmZvougyhVhISEYPr06Th48CDu3r2L/fv3o0GDBgCgTHK6cOFCXLhwASNGjECZMmX0XHHWdvLkSdSoUQMA8OzZM1SvXh02NjbYsmULChYsqNN27dq1WLFiBYoUKYKJEyeiSJEi+ig589PrCThKU/7+/jJr1izx9fUVEZHIyEhZsGCBGBoayowZM3TavnjxQhnzk3jcEBFlXSEhITJkyBApX768zJ49W1meeJyP2scApjWtVit///23aDQamTFjhoSGhorI+zu9Fy1aVOrUqSP379+X6Oho6dixo8yZM0dE3o/XLFeunHz99df8N/pMPAWWhZw7dw7FihWDpaUlrl27pnybWL16NQAgZ86c6NevHzQaDb7//nsYGBhg9OjRAIDcuXMD4OWtRGpibW0NLy8vaLVabNu2DXFxcRg9ejRMTEyUXiC1jwFMaxqNBjVq1MDkyZMxZ84cGBoawsPDA/b29vDz80OjRo1Qq1Yt5M+fH2/fvsXatWsBAF9//TWMjY3RoEED/ht9Jg7wyCIWL16M9u3bIyQkBABQtmxZ/Pbbb8iePTv8/f3x7t07AICZmRn69euHn3/+GV5eXli/fr3Odhh+iNTFxsYG48aNQ5UqVbBz505MnDgRAPihmkYk0aiTxNOLTJgwAcOGDcOMGTOwevVqPHv2DIULF8aVK1cwYMAA9OzZE1evXlXG/ACAh4cHChcunO77kGXouwuKvtyyZcvE0NBQtm/fnmTdhg0bxNDQUMaOHavcwV1E5O3bt7J9+3Z2nRKRiLyf5bl3797i6uoqz54903c5Wd7MmTPll19+UW6zk2Dq1KlibGwss2bNksePHyd5HS95Tz2M+Jnc8uXL8e2332Lr1q1o27atsvz8+fNwdHRE165dAQA9e/YE8P6KMCMjI5iZmSnts/Jd3Yno09jY2OCHH34AAJ3L4+nLSaKhBQl/v3TpErZu3Yrs2bOjbdu2ygUW48ePx9WrVzF37ly8efMGQ4YMUYYoAOCl7qmIn3qZ2Pbt2/HNN99g48aNOuGnWbNmyJ07N1auXAkA6Nq1KzQaDTw8PBAREYF58+bp/BIx/BARAF7mnkaePn2KmJgYvHz5Evnz50eBAgWwYcMGWFhYoG/fvtBqtWjXrh2yZcsGAChYsCDy5s2LU6dOwdvbW8/VZ1385MukRASHDh1C4cKF8fz5c+VbRfv27REUFIRFixbpXLLdpUsXREZG4tdff+XcPkRE6WTjxo1YunQp7ty5g5CQEDg4OMDNzQ2LFy/G4sWLodVq0b9/f4gIGjZsCFtbWzx+/Bhr1qyBs7OzMiEtx2emPs4DlInFxsZi6NChuHDhArp3745Dhw7hwYMH+P333+Hg4KDzS/Pq1StYWloqy/gLRUSUtlavXo1BgwZhzpw5KFWqFIyNjbFq1Sps2rQJdevWVeZd+/bbb7F9+3YUKFAA8fHxiIuLw+XLl2FkZMQJadMQA1AmlTDteWxsLAYPHow9e/YgNjYWR44cQenSpXXG9bRr1w5FixbFrFmzAPBSdyKitHbx4kV06NABM2bMQMeOHZXlz58/x9atWzFixAi0atUKmzZtAgBl9u13795h/PjxMDIy4u0t0hhPgWVSCfd+MTY2xsKFCzF8+HCcOHEC+/btg52dHXLmzIn4+Hi0bNkSt27dwubNm5XXMvwQEaWtoKAg5MyZE3Xq1FGCjIggb9686NKlC548eYIFCxbg8OHDaNCgAbp166bzeoaftMd+tUwscQiaM2cOqlSpgk2bNuGXX35BVFQU2rVrh3v37uH69eswNjZGXFycvksmIlKFixcvIiQkBDY2Nkr4SfjyaWlpiR49eiAqKgpPnjxJ9vUMP2mPASiT+3dPUKVKlbB161aULFkS169fx5UrV5Tww6u9iIjSR+nSpREZGYkDBw4ASNrzXqRIEdjY2OD169f6KI/AAJThabXaj7ZJHIIWLFiAYsWKoXTp0rh69SrDDxGRHlSuXBnGxsZYvnw5AgMDleUJsz8HBgYiX758KFGihL5KVD0Ogs7AEneZbtu2DU+ePIG9vT1atGiRbPdowtUC8fHx0Gg0MDAwYPghItKTTZs2wcPDA+3atcPw4cNRqVIlAMCbN2/QsWNHREZG4siRI7zKS08YgDKoxOFn7NixmD9/PsqVK4ezZ8+iX79+GDlyJIoXL57kdYkvmeTlk0RE+hMXF4c1a9bg22+/Rf78+eHk5ARLS0sEBgYiMjISZ8+ehbGxMQc86wk/HTOohPBz/fp1nDp1CkePHsXp06dx5MgR/Pbbb5g2bRpu3bqltE/IsYkDD8MPEZH+GBkZoV+/fjh9+jRat26Nt2/fwtjYGC1atMC5c+eUIQoMP/rBHqAMzMfHB2fOnIGZmRnWrFkDU1NTAMDRo0fRrl07tGzZEuPGjUu2J4iIiDI29vzoF7sIMrBChQrhjz/+gL+/v3KppIigXr162LFjB/bu3Yvhw4cjKChIz5USEdF/Sa6vgeFHvxiAMojkrvbq1q0bduzYgcDAQMyfPx9hYWHKqbG6deti/fr1iIuLQ8GCBdO7XCIiSgFOQJvx8BRYBpB4sPLRo0cRFhYGQ0NDNGrUCLly5cKmTZvQrVs3DBs2DF5eXsiXL1+S21lwwDMREdGn4/XRGUBCcBk9ejR8fX1hYmKCvHnzYujQoTh16hS6dOkCIyMjdOrUCYaGhhgxYgSsrKyS3QYRERF9HD81M4ilS5di9erV+PXXX3HlyhV06NABjx8/xtmzZwEAHTp0wObNmzF79mxs3bpVz9USERFlbuwB0pN/n8L6559/MHz4cFSpUgW+vr7w8vLCsmXL4O7ujoiICBgbG6Njx47Imzcv6tatq8fKiYiIMj/2AOlB4vATExMDAAgJCUFMTAx2796NHj16YObMmejfvz+0Wi3Wr1+PZcuWIS4uDg0bNoSRkRFvbEpERPQFGID0ICH8TJ8+HVOmTAHw/r4xf/75J7p06YKZM2di4MCBAICXL19iz549ePfunc4tLXh7CyIios/HAJROpk2bhqdPnwL433wQx44dQ5kyZQAAHh4e0Gg0yJMnDxwdHREREYH79++jR48eePbsGUaMGKG32omIiLIaBqB0EBQUhIkTJ6J37954/vy5EoAiIiKUibDy58+PP/74Azlz5sSgQYNgb2+Pbt264eXLlzhx4gSMjIyUuwgTERHRl+F5lHRgZ2eHK1euwM3NDd27d8evv/6KvHnzIi4uTpkAMSYmBgUKFMDff/+Ny5cv4+7duyhSpAhq1qwJQ0ND3tWdiIgoFfETNZ2UKVMG+/btQ+PGjdG5c2ds3LgROXLkQPbs2ZU2ERERsLCwQMmSJVG7dm1leXx8PMMPERFRKuJM0Gno35e6A8C1a9fQoEEDFClSBMHBwQgJCYGjoyOeP3+O6Oho5MyZE7Vq1cLKlSv1VDUREVHWxwCURhLf5TfhkvWEXpyrV6+iY8eOePr0KebMmQM7Ozu8ffsWsbGxMDMzg6urK3t8iIiI0hADUBqIjIxErly5AABz5szBuXPncOvWLXTp0gV169ZFlSpVcO3aNTRu3BjVq1fH6tWrYW5urrONxAGKiIiIUhevAktl69evx08//QQAGDNmDGbMmIFSpUqhbNmy2LZtGzw9PXH48GGULVsW+/fvx9mzZ1G/fn28evVKZzsMP0RERGmH51lS0bJlyzBw4EDs2bMHt2/fhq+vL3777TfUr18fAHDkyBEsX74cPj4+KFy4MMqVK4edO3di4sSJSXqAiIiIKO2wByiVrF+/Ht999x127dqFJk2a4PXr1wgNDdUZy1O/fn306tULt2/fxqNHjwAATk5O8PX1hYGBgXJJPBEREaUtBqBUsGbNGvTq1Qv16tVDs2bNAADGxsawsrLCw4cPAfxv9ucmTZrA1NQUx48fT7IdAwP+cxAREaUHfuJ+oV9++QV9+/ZF3759ce3aNQwZMgQAUK5cOVStWhXDhw/HyZMnlcvhX758iezZs8POzk6fZRMREakarwL7AvPmzYOnpyd2796Npk2bYtmyZRg/fjw6deqEhQsXAgBatGiBU6dOoWfPnsifPz+OHDmCkJAQXLhwgZe6ExER6QkD0Bc4duwYgoOD0blzZwBAeHg4tmzZgnHjxumEIC8vL1y5cgUvX75EsWLFsGLFChgbG/NSdyIiIj1hAEoFiWd8joiIwObNm5OEoDdv3sDAwABmZmYAwHt7ERER6RE/gVNB4ttdmJubKz1C48ePh6GhIebPn69zzy8RYfghIiLSI34Kp4GEEKTRaDBgwAAUKVIEQ4cOVdb/+/5gRERElL54CiwNvXr1CseOHUOLFi041oeIiCgDYQBKJxzzQ0RElHEwABEREZHqcCJEIiIiUh0GICIiIlIdBiAiIiJSHQYgIiIiUh0GICIiIlIdBiAiIiJSHQYgIiIiUh0GICIiIlIdBiAiIiJSHQYgIiIiUh0GICIiIlIdBiAiIiJSnf8D6rPVJCJMpGwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token Embeddings               51,463,168      14.56%\n",
      "Attention                     100,663,296      28.48%\n",
      "MLP                           201,326,592      56.95%\n",
      "RMS Norm                           50,176       0.01%\n",
      "Output Layer                            0       0.00%\n",
      "Total parameters: 353,503,232\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "parameter_counts = {\n",
    "    \"Token Embeddings\": vocabulary_size * embedding_dimensions,\n",
    "    \"Attention\": (\n",
    "        embedding_dimensions**2 + embedding_dimensions * 3 * embedding_dimensions\n",
    "    )\n",
    "    * num_hidden_layers,\n",
    "    \"MLP\": embedding_dimensions\n",
    "    * feed_forward_ratio\n",
    "    * embedding_dimensions\n",
    "    * 2\n",
    "    * num_hidden_layers,\n",
    "    \"RMS Norm\": embedding_dimensions * num_hidden_layers * 2 + embedding_dimensions,\n",
    "    \"Output Layer\": 0,  # Tied to token embeddings\n",
    "}\n",
    "\n",
    "plt.bar(parameter_counts.keys(), parameter_counts.values())\n",
    "\n",
    "plt.title(\"Model Parameters\")\n",
    "plt.ylabel(\"# of Parameters\")\n",
    "plt.xticks(rotation=45)\n",
    "\n",
    "plt.show()\n",
    "\n",
    "total_parameter_count = sum(parameter_counts.values())\n",
    "\n",
    "for name, count in parameter_counts.items():\n",
    "    print(f\"{name:20s} {count:20,d} {count / total_parameter_count * 100:10.2f}%\")\n",
    "\n",
    "\n",
    "print(f\"Total parameters: {total_parameter_count:,}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To get an idea of the \"shape\" of our neural network we'll look at the ratio of embedding dimensions to number of layers. Generally, an aspect ratio between 50 and 100 is considered optimal according to certain scaling laws (Kaplan, 2020)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network has an aspect ratio of 42.67\n"
     ]
    }
   ],
   "source": [
    "aspect_ratio = embedding_dimensions / num_hidden_layers\n",
    "\n",
    "print(f\"Network has an aspect ratio of {aspect_ratio:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can do the same analysis for the ratio of embedding dimensions to the number of attention heads. In this case, a ratio of between 20 and 80 is considered optimal according to the same paper."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Heads ratio is 64.00\n"
     ]
    }
   ],
   "source": [
    "heads_ratio = embedding_dimensions / num_attention_heads\n",
    "\n",
    "print(f\"Heads ratio is {heads_ratio:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we'll estimate the size of the model in memory and on disk. Note that this does not include any intermediate variables that get memorized during training such as activations, gradients, optimizer state, and temporary buffers. Actual memory consumption will likely be much higher."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total gigabytes: 1.41G\n"
     ]
    }
   ],
   "source": [
    "bytes_per_parameter = 32 // 8  # Assuming 32-bit floating point\n",
    "\n",
    "total_bytes = total_parameter_count * bytes_per_parameter\n",
    "\n",
    "total_gigabytes = total_bytes / 1e9\n",
    "\n",
    "print(f\"Total gigabytes: {total_gigabytes:,.2f}G\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can estimate the optimal number of training tokens using the Chinchilla scaling law given the number of parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimal training tokens: 7,070,064,640\n",
      "Epochs required: 1,686\n",
      "\n"
     ]
    }
   ],
   "source": [
    "num_training_tokens = 20 * total_parameter_count\n",
    "\n",
    "num_epochs_required = round(\n",
    "    num_training_tokens / (samples_per_epoch * tokens_per_sample)\n",
    ")\n",
    "\n",
    "print(f\"Optimal training tokens: {num_training_tokens:,}\")\n",
    "\n",
    "print(f\"Epochs required: {num_epochs_required:,}\", end=\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we'll estimate the maximum number of floating point operations (FLOPs) required to perform a full forward pass of the network on a single sample."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHwCAYAAABaLU4/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABUJUlEQVR4nO3dd1gUV9sG8HspgkpRkWLBEjV2RbEAGsVKlKjYCwbssaDYFXvHEqPGghpfxajEEtsXYwkSe4sKxBZrFFBZlKiAgEvZ8/3hy7ysoAEFBob7d117xT1zZvZZNsDNmXNmVEIIASIiIiKF0JO7ACIiIqKcxHBDREREisJwQ0RERIrCcENERESKwnBDREREisJwQ0RERIrCcENERESKwnBDREREisJwQ0RERIrCcENEWaZSqTBnzpxs7/fo0SOoVCr4+/vneE0F0cd+HYkoaxhuiAoYf39/qFQqqFQqnD17NsN2IQRsbW2hUqnw1VdfyVDhpwsPD8fw4cNRqVIlGBkZwcrKCm5ubjh37pzcpWXZ4cOHGWCIZMJwQ1RAGRsbIyAgIEP7qVOn8PjxYxgZGclQ1ac7d+4c6tati59++gndu3fHunXr4O3tjZs3b+KLL77A6tWr5S4xSw4fPoy5c+dmui0xMREzZszI44qICg8DuQsgoo/TsWNH7NmzB99//z0MDP73rRwQEAB7e3tER0fLWN3HefnyJXr06IGiRYvi3LlzqFKlirRt/PjxcHFxwdixY2Fvbw8nJ6c8rS0+Ph7FixfPkWMZGxvnyHGIKHMcuSEqoPr27Yt//vkHgYGBUltSUhJ+/vln9OvXL9N94uPjMWHCBNja2sLIyAjVq1fHt99+CyGETj+NRoNx48bB0tISpqam6Ny5Mx4/fpzpMZ88eYJBgwbB2toaRkZGqF27NjZv3vxR72nDhg1Qq9VYtmyZTrABgKJFi2Lr1q1QqVSYN2+e1J52mu706dP45ptvYGFhATMzM3h4eODly5cZXuPIkSP44osvULx4cZiamsLV1RU3b97U6TNgwACYmJjgwYMH6NixI0xNTeHu7g4AOHPmDHr27IkKFSrAyMgItra2GDduHBITE3X2X7t2LQBIpxBVKpW0PbM5NyEhIejQoQPMzMxgYmKCNm3a4OLFizp90t7ruXPnMH78eFhaWqJ48eLo2rUrnj9/rtP3ypUrcHFxQenSpVG0aFFUrlwZgwYN+rePgEgROHJDVEBVqlQJjo6O+Omnn9ChQwcAb39xx8TEoE+fPvj+++91+gsh0LlzZ5w4cQKDBw+GnZ0djh07hkmTJuHJkydYsWKF1HfIkCHYvn07+vXrBycnJ/z+++9wdXXNUENUVBQcHBygUqng5eUFS0tLHDlyBIMHD0ZsbCzGjh2brff0yy+/wNjYGL169cp0e+XKldG8eXP8/vvvSExMRNGiRaVtXl5eKFGiBObMmYM7d+7Az88PYWFhOHnypBQstm3bBk9PT7i4uGDJkiVISEiAn58fmjdvjpCQEFSqVEk6XkpKClxcXNC8eXN8++23KFasGABgz549SEhIwIgRI2BhYYE//vgDq1evxuPHj7Fnzx4AwDfffIOnT58iMDAQ27Zt+9f3nXbKzczMDJMnT4ahoSE2bNgAZ2dnnDp1Ck2bNtXpP3r0aJQsWRKzZ8/Go0ePsHLlSnh5eWHXrl0AgGfPnqF9+/awtLTE1KlTUaJECTx69Aj79u3L+odBVJAJIipQtmzZIgCIy5cvizVr1ghTU1ORkJAghBCiZ8+eolWrVkIIISpWrChcXV2l/Q4cOCAAiAULFugcr0ePHkKlUon79+8LIYQIDQ0VAMTIkSN1+vXr108AELNnz5baBg8eLMqUKSOio6N1+vbp00eYm5tLdT18+FAAEFu2bPngeytRooSoX7/+B/uMGTNGABDXrl3T+XrY29uLpKQkqd/SpUsFAHHw4EEhhBBxcXGiRIkSYujQoTrHU6vVwtzcXKfd09NTABBTp07N8Ppp7yk9X19foVKpRFhYmNQ2atQo8b4fse9+Hd3c3ESRIkXEgwcPpLanT58KU1NT0aJFC6kt7b22bdtWaLVaqX3cuHFCX19fvHr1SgghxP79+6X/R4gKI56WIirAevXqhcTERBw6dAhxcXE4dOjQe09JHT58GPr6+hgzZoxO+4QJEyCEwJEjR6R+ADL0e3cURgiBvXv3olOnThBCIDo6Wnq4uLggJiYGwcHB2Xo/cXFxMDU1/WCftO2xsbE67cOGDYOhoaH0fMSIETAwMJDeT2BgIF69eoW+ffvq1Kqvr4+mTZvixIkTGV5rxIgRGdrSjxbFx8cjOjoaTk5OEEIgJCQk62/2v1JTU/Hbb7/Bzc0Nn332mdRepkwZ9OvXD2fPns30vaY/zfXFF18gNTUVYWFhAIASJUoAAA4dOoTk5ORs10RU0BXqcHP69Gl06tQJZcuWhUqlwoEDB7K1/5s3bzBgwADUrVsXBgYGcHNzy9AnMjIS/fr1w+effw49Pb1sD9MTfYilpSXatm2LgIAA7Nu3D6mpqejRo0emfcPCwlC2bNkM4aFmzZrS9rT/6unpZZjzUr16dZ3nz58/x6tXr7Bx40ZYWlrqPAYOHAjg7emR7DA1NUVcXNwH+6Rtf/d9VKtWTee5iYkJypQpg0ePHgEA7t27BwBo3bp1hnp/++23DLUaGBigfPnyGV4/PDwcAwYMQKlSpWBiYgJLS0u0bNkSABATE5P1N/tfz58/R0JCQoavL/D2s9FqtYiIiNBpr1Chgs7zkiVLAoA0x6hly5bo3r075s6di9KlS6NLly7YsmULNBpNtusjKogK9Zyb+Ph41K9fH4MGDUK3bt2yvX9qaiqKFi2KMWPGYO/evZn20Wg0sLS0xIwZM3TmNBDllH79+mHo0KFQq9Xo0KGD9Fd7btNqtQCA/v37w9PTM9M+9erVy9Yxa9asiZCQEGg0mvcuZb927RoMDQ0zhJms1rtt2zbY2Nhk2J5+xRkAGBkZQU9P9++/1NRUtGvXDi9evMCUKVNQo0YNFC9eHE+ePMGAAQOk18ht+vr6mbaL/04MV6lU+Pnnn3Hx4kX88ssvOHbsGAYNGoTly5fj4sWLMDExyZM6ieRSqMNNhw4dpImYmdFoNJg+fTp++uknvHr1CnXq1MGSJUvg7OwMAChevDj8/PwAvL02x6tXrzIco1KlSli1ahUAfPQKEqIP6dq1K7755htcvHhRmlCamYoVK+L48eMZTv3cvn1b2p72X61WiwcPHuiMJty5c0fneGkrqVJTU9G2bdsceS9fffUVLly4gD179qB///4Ztj969AhnzpxB27ZtdU4PAW9HZlq1aiU9f/36NSIjI9GxY0cAkEairKysPrre69ev4+7du9i6dSs8PDyk9vQr1tKkP230IZaWlihWrFiGry/w9rPR09ODra3tR9Xr4OAABwcHLFy4EAEBAXB3d8fOnTsxZMiQjzoeUUFRqE9L/RsvLy9cuHABO3fuxLVr19CzZ098+eWX0vA2UX5gYmICPz8/zJkzB506dXpvv44dOyI1NRVr1qzRaV+xYgVUKpUU9NP+++5qq5UrV+o819fXR/fu3bF3717cuHEjw+u9uzQ5K7755htYWVlh0qRJ+Pvvv3W2vXnzBgMHDoQQArNmzcqw78aNG3Xml/j5+SElJUV6Py4uLjAzM8OiRYsynYeSlXrTRkxEuqXzQgjpD5j00q6Jk9kfPe8es3379jh48KB0Cg14uxItICAAzZs3h5mZ2b/Wlt7Lly8zLO+3s7MDAJ6aokKhUI/cfEh4eDi2bNmC8PBwlC1bFgAwceJEHD16FFu2bMGiRYtkrpDof953Wii9Tp06oVWrVpg+fToePXqE+vXr47fffsPBgwcxduxYaWTDzs4Offv2xbp16xATEwMnJycEBQXh/v37GY65ePFinDhxAk2bNsXQoUNRq1YtvHjxAsHBwTh+/DhevHiRrfdhYWGBn3/+Ga6urmjYsCGGDBmCWrVqQa1Ww9/fH/fv38eqVasyvYBfUlIS2rRpg169euHOnTtYt24dmjdvjs6dOwMAzMzM4Ofnh6+//hoNGzZEnz59YGlpifDwcPz6669o1qxZhuD3rho1aqBKlSqYOHEinjx5AjMzM+zduzfT6+nY29sDeDsx28XFBfr6+ujTp0+mx12wYAECAwPRvHlzjBw5EgYGBtiwYQM0Gg2WLl2ara8hAGzduhXr1q1D165dUaVKFcTFxeGHH36AmZmZNJJFpGiyrdPKZwCI/fv3S88PHTokAIjixYvrPAwMDESvXr0y7O/p6Sm6dOnywddo2bKl8Pb2ztnCqdBJvxT8Q95dCi7E2+XQ48aNE2XLlhWGhoaiWrVqYtmyZTrLioUQIjExUYwZM0ZYWFiI4sWLi06dOomIiIgMS5iFECIqKkqMGjVK2NraCkNDQ2FjYyPatGkjNm7cKPXJ6lLw9P2HDh0qKlSoIAwNDUXp0qVF586dxZkzZ9779Th16pQYNmyYKFmypDAxMRHu7u7in3/+ydD/xIkTwsXFRZibmwtjY2NRpUoVMWDAAHHlyhWpj6enpyhevHimtd26dUu0bdtWmJiYiNKlS4uhQ4eKP//8M8P7S0lJEaNHjxaWlpZCpVLpLAvP7OsYHBwsXFxchImJiShWrJho1aqVOH/+fKbv9d3P/sSJEwKAOHHihHSsvn37igoVKggjIyNhZWUlvvrqK533SKRkKiHeGbsspFQqFfbv3y+teNq1axfc3d1x8+bNDJP3TExMMkxIHDBgAF69evXBFVfOzs6ws7PLMLxPRB/P398fAwcOxOXLl9GoUSO5yyGifICnpd6jQYMGSE1NxbNnz/DFF1/IXQ4RERFlUaEON69fv9aZR/Dw4UOEhoaiVKlS+Pzzz+Hu7g4PDw8sX74cDRo0wPPnzxEUFIR69epJl6K/desWkpKS8OLFC8TFxSE0NBTA/ybvAZDaXr9+jefPnyM0NBRFihRBrVq18uqtEhERFRqF+rTUyZMndZaOpvH09IS/vz+Sk5OxYMEC/Pjjj3jy5AlKly4NBwcHzJ07F3Xr1gXwdql32sXP0kv/Zc1sSWjFihV1VkYQ0cfhaSkielehDjdERESkPLzODRERESkKww0REREpSqGbUKzVavH06VOYmppm+fLoREREJC8hBOLi4lC2bNkM9317V6ELN0+fPv3o+7QQERGRvCIiIlC+fPkP9il04SbthoERERHZvl8LERERySM2Nha2trY6N/59n0IXbtJORZmZmTHcEBERFTBZmVLCCcVERESkKAw3REREpCgMN0RERKQoDDdERESkKAw3REREpCgMN0RERKQoDDdERESkKAw3REREpCgMN0RERKQoDDdERESkKAw3REREpCgMN0RERKQoDDdERESkKAw3REREpCgMN0RERKQoBnIXQFQQVJr6q9wlFFqPFrvKXQIRFTAcuSEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRck34Wbx4sVQqVQYO3bsB/vt2bMHNWrUgLGxMerWrYvDhw/nTYFERERUIOSLcHP58mVs2LAB9erV+2C/8+fPo2/fvhg8eDBCQkLg5uYGNzc33LhxI48qJSIiovxO9nDz+vVruLu744cffkDJkiU/2HfVqlX48ssvMWnSJNSsWRPz589Hw4YNsWbNmjyqloiIiPI72cPNqFGj4OrqirZt2/5r3wsXLmTo5+LiggsXLrx3H41Gg9jYWJ0HERERKZesVyjeuXMngoODcfny5Sz1V6vVsLa21mmztraGWq1+7z6+vr6YO3fuJ9VJREREBYdsIzcRERHw9vbGjh07YGxsnGuv4+Pjg5iYGOkRERGRa69FRERE8pNt5Obq1at49uwZGjZsKLWlpqbi9OnTWLNmDTQaDfT19XX2sbGxQVRUlE5bVFQUbGxs3vs6RkZGMDIyytniiYiIKN+SbeSmTZs2uH79OkJDQ6VHo0aN4O7ujtDQ0AzBBgAcHR0RFBSk0xYYGAhHR8e8KpuIiIjyOdlGbkxNTVGnTh2dtuLFi8PCwkJq9/DwQLly5eDr6wsA8Pb2RsuWLbF8+XK4urpi586duHLlCjZu3Jjn9RMREVH+JPtqqQ8JDw9HZGSk9NzJyQkBAQHYuHEj6tevj59//hkHDhzIEJKIiIio8FIJIYTcReSl2NhYmJubIyYmBmZmZnKXQwVEpam/yl1CofVosavcJRBRPpCd39/5euSGiIiIKLsYboiIiEhRGG6IiIhIURhuiIiISFEYboiIiEhRGG6IiIhIURhuiIiISFEYboiIiEhRGG6IiIhIURhuiIiISFEYboiIiEhRGG6IiIhIURhuiIiISFEYboiIiEhRGG6IiIhIURhuiIiISFEYboiIiEhRGG6IiIhIURhuiIiISFEYboiIiEhRGG6IiIhIURhuiIiISFEYboiIiEhRGG6IiIhIURhuiIiISFEYboiIiEhRGG6IiIhIURhuiIiISFEYboiIiEhRGG6IiIhIURhuiIiISFEYboiIiEhRZA03fn5+qFevHszMzGBmZgZHR0ccOXLkvf39/f2hUql0HsbGxnlYMREREeV3BnK+ePny5bF48WJUq1YNQghs3boVXbp0QUhICGrXrp3pPmZmZrhz5470XKVS5VW5REREVADIGm46deqk83zhwoXw8/PDxYsX3xtuVCoVbGxs8qI8IiIiKoDyzZyb1NRU7Ny5E/Hx8XB0dHxvv9evX6NixYqwtbVFly5dcPPmzQ8eV6PRIDY2VudBREREyiV7uLl+/TpMTExgZGSE4cOHY//+/ahVq1amfatXr47Nmzfj4MGD2L59O7RaLZycnPD48eP3Ht/X1xfm5ubSw9bWNrfeChEREeUDKiGEkLOApKQkhIeHIyYmBj///DM2bdqEU6dOvTfgpJecnIyaNWuib9++mD9/fqZ9NBoNNBqN9Dw2Nha2traIiYmBmZlZjr0PUrZKU3+Vu4RC69FiV7lLIKJ8IDY2Fubm5ln6/S3rnBsAKFKkCKpWrQoAsLe3x+XLl7Fq1Sps2LDhX/c1NDREgwYNcP/+/ff2MTIygpGRUY7VS0RERPmb7Kel3qXVanVGWj4kNTUV169fR5kyZXK5KiIiIiooZB258fHxQYcOHVChQgXExcUhICAAJ0+exLFjxwAAHh4eKFeuHHx9fQEA8+bNg4ODA6pWrYpXr15h2bJlCAsLw5AhQ+R8G0RERJSPyBpunj17Bg8PD0RGRsLc3Bz16tXDsWPH0K5dOwBAeHg49PT+N7j08uVLDB06FGq1GiVLloS9vT3Onz+fpfk5REREVDjIPqE4r2VnQhJRGk4olg8nFBMRkL3f3/luzg0RERHRp2C4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRTGQuwClqTT1V7lLKLQeLXaVuwQiIsoHOHJDREREisJwQ0RERIoia7jx8/NDvXr1YGZmBjMzMzg6OuLIkSMf3GfPnj2oUaMGjI2NUbduXRw+fDiPqiUiIqKCQNZwU758eSxevBhXr17FlStX0Lp1a3Tp0gU3b97MtP/58+fRt29fDB48GCEhIXBzc4Obmxtu3LiRx5UTERFRfqUSQgi5i0ivVKlSWLZsGQYPHpxhW+/evREfH49Dhw5JbQ4ODrCzs8P69euzdPzY2FiYm5sjJiYGZmZmOVZ3Gk4olk9uTijm5yofThQnIiB7v7/zzZyb1NRU7Ny5E/Hx8XB0dMy0z4ULF9C2bVudNhcXF1y4cCEvSiQiIqICQPal4NevX4ejoyPevHkDExMT7N+/H7Vq1cq0r1qthrW1tU6btbU11Gr1e4+v0Wig0Wik57GxsTlTOBEREeVLso/cVK9eHaGhobh06RJGjBgBT09P3Lp1K8eO7+vrC3Nzc+lha2ubY8cmIiKi/Ef2cFOkSBFUrVoV9vb28PX1Rf369bFq1apM+9rY2CAqKkqnLSoqCjY2Nu89vo+PD2JiYqRHREREjtZPRERE+Yvs4eZdWq1W5zRSeo6OjggKCtJpCwwMfO8cHQAwMjKSlpqnPYiIiEi5ZJ1z4+Pjgw4dOqBChQqIi4tDQEAATp48iWPHjgEAPDw8UK5cOfj6+gIAvL290bJlSyxfvhyurq7YuXMnrly5go0bN8r5NoiIiCgfkTXcPHv2DB4eHoiMjIS5uTnq1auHY8eOoV27dgCA8PBw6On9b3DJyckJAQEBmDFjBqZNm4Zq1arhwIEDqFOnjlxvgYiIiPIZWcPNf/7znw9uP3nyZIa2nj17omfPnrlUERERERV0+W7ODREREdGnYLghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRcl2uAkODsb169el5wcPHoSbmxumTZuGpKSkHC2OiIiIKLuyHW6++eYb3L17FwDw999/o0+fPihWrBj27NmDyZMn53iBRERERNmR7XBz9+5d2NnZAQD27NmDFi1aICAgAP7+/ti7d29O10dERESULdkON0IIaLVaAMDx48fRsWNHAICtrS2io6OzdSxfX180btwYpqamsLKygpubG+7cufPBffz9/aFSqXQexsbG2X0bREREpFDZDjeNGjXCggULsG3bNpw6dQqurq4AgIcPH8La2jpbxzp16hRGjRqFixcvIjAwEMnJyWjfvj3i4+M/uJ+ZmRkiIyOlR1hYWHbfBhERESmUQXZ3WLFiBdzd3XHgwAFMnz4dVatWBQD8/PPPcHJyytaxjh49qvPc398fVlZWuHr1Klq0aPHe/VQqFWxsbLJbOhERERUC2Q439evXx40bNzK0L1u2DPr6+p9UTExMDACgVKlSH+z3+vVrVKxYEVqtFg0bNsSiRYtQu3btTPtqNBpoNBrpeWxs7CfVSERERPlblk9LxcfHY8SIEShXrhwsLS3Rp08fPH/+XNpubGwMQ0PDjy5Eq9Vi7NixaNasGerUqfPeftWrV8fmzZtx8OBBbN++HVqtFk5OTnj8+HGm/X19fWFubi49bG1tP7pGIiIiyv+yHG5mzpyJbdu24auvvkK/fv3w+++/Y9iwYTlWyKhRo3Djxg3s3Lnzg/0cHR3h4eEBOzs7tGzZEvv27YOlpSU2bNiQaX8fHx/ExMRIj4iIiByrmYiIiPKfLJ+W2r9/P7Zs2YKePXsCADw8PODg4ICUlBQYGGT77JYOLy8vHDp0CKdPn0b58uWzta+hoSEaNGiA+/fvZ7rdyMgIRkZGn1QfERERFRxZHrl5/PgxmjVrJj23t7eHoaEhnj59+tEvLoSAl5cX9u/fj99//x2VK1fO9jFSU1Nx/fp1lClT5qPrICIiIuXI8pCLVqvNMKfGwMAAqampH/3io0aNQkBAAA4ePAhTU1Oo1WoAgLm5OYoWLQrg7QhRuXLl4OvrCwCYN28eHBwcULVqVbx69QrLli1DWFgYhgwZ8tF1EBERkXJkOdwIIdCmTRudU1AJCQno1KkTihQpIrUFBwdn+cX9/PwAAM7OzjrtW7ZswYABAwAA4eHh0NP73wDTy5cvMXToUKjVapQsWRL29vY4f/48atWqleXXJSIiIuXKcriZPXt2hrYuXbp80osLIf61z8mTJ3Wer1ixAitWrPik1yUiIiLl+qRwQ0RERJTffNQyp2vXrkl3Bv/8889Rr169HC2KiIiI6GNlK9z88ccfGDx4MG7duiWdUlKpVKhduzb+85//oHHjxrlSJBEREVFWZXkp+K1bt9CmTRsULVoU27dvR3BwMIKDg7Ft2zYYGRmhTZs2uHXrVm7WSkRERPSvsjxyM2fOHLRr1w579+6FSqWS2u3s7NC3b19069YNc+bMwe7du3OlUCIiIqKsyHK4OXHiBI4cOaITbNKoVCpMmzYNHTt2zNHiiIiIiLIry6el4uLiYG1t/d7tNjY2iIuLy5GiiIiIiD5WlsNNxYoV8ccff7x3+6VLl1CxYsUcKYqIiIjoY2U53PTp0wfjx4/HjRs3Mmy7fv06Jk6ciN69e+docURERETZleU5Nz4+Pjh+/Djs7OzQrl071KxZE0II/PXXXzh+/DiaNGmCadOm5WatRERERP8qy+HG2NgYJ06cwIoVK/DTTz/h1KlTAN5exG/BggUYN24cjIyMcq1QIiIioqzI8mkpAChSpAimTJmC0NBQJCQkICEhAaGhoZg6dSqeP3+OYcOG5VadRERERFmSrXDzIf/88w/+85//5NThiIiIiD5KjoUbIiIiovyA4YaIiIgUheGGiIiIFCXLq6W6dev2we2vXr361FqIiIiIPlmWw425ufm/bvfw8PjkgoiIiIg+RZbDzZYtW3KzDiIiIqIckeU5N3///TeEELlZCxEREdEny3K4qVatGp4/fy497927N6KionKlKCIiIqKPleVw8+6ozeHDhxEfH5/jBRERERF9Ci4FJyIiIkXJcrhRqVRQqVQZ2oiIiIjykyyvlhJCYMCAAdKdv9+8eYPhw4ejePHiOv327duXsxUSERERZUOWw42np6fO8/79++d4MURERESfite5ISIiIkXhhGIiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSlCyFm4YNG+Lly5cAgHnz5iEhISFXiyIiIiL6WFkKN3/99Zd0q4W5c+fi9evXOfLivr6+aNy4MUxNTWFlZQU3NzfcuXPnX/fbs2cPatSoAWNjY9StWxeHDx/OkXqIiIio4MvSUnA7OzsMHDgQzZs3hxAC3377LUxMTDLtO2vWrCy/+KlTpzBq1Cg0btwYKSkpmDZtGtq3b49bt25luDhgmvPnz6Nv377w9fXFV199hYCAALi5uSE4OBh16tTJ8msTERGRMqnEu3fEzMSdO3cwe/ZsPHjwAMHBwahVqxYMDDLmIpVKheDg4I8u5vnz57CyssKpU6fQokWLTPv07t0b8fHxOHTokNTm4OAAOzs7rF+//l9fIzY2Fubm5oiJiYGZmdlH1/o+lab+muPHpKx5tNg1147Nz1U+ufm5ElHBkZ3f31kaualevTp27twJANDT00NQUBCsrKw+vdJ3xMTEAABKlSr13j4XLlzA+PHjddpcXFxw4MCBTPtrNBpoNBrpeWxs7KcXSkRERPlWtldLabXaXAk2Wq0WY8eORbNmzT54ekmtVsPa2lqnzdraGmq1OtP+vr6+MDc3lx62trY5WjcRERHlLx+1FPzBgwcYPXo02rZti7Zt22LMmDF48ODBJxUyatQo3LhxQxohyik+Pj6IiYmRHhERETl6fCIiIspfsh1ujh07hlq1auGPP/5AvXr1UK9ePVy6dAm1a9dGYGDgRxXh5eWFQ4cO4cSJEyhfvvwH+9rY2CAqKkqnLSoqCjY2Npn2NzIygpmZmc6DiIiIlCvLN85MM3XqVIwbNw6LFy/O0D5lyhS0a9cuy8cSQmD06NHYv38/Tp48icqVK//rPo6OjggKCsLYsWOltsDAQDg6Omb5dYmIiEi5sj1y89dff2Hw4MEZ2gcNGoRbt25l61ijRo3C9u3bERAQAFNTU6jVaqjVaiQmJkp9PDw84OPjIz339vbG0aNHsXz5cty+fRtz5szBlStX4OXlld23QkRERAqU7XBjaWmJ0NDQDO2hoaHZnmjs5+eHmJgYODs7o0yZMtJj165dUp/w8HBERkZKz52cnBAQEICNGzeifv36+Pnnn3HgwAFe44aIiIgAfMRpqaFDh2LYsGH4+++/4eTkBAA4d+4clixZkmGJ9r/JwiV2cPLkyQxtPXv2RM+ePbP1WkRERFQ4ZDvczJw5E6ampli+fLl0uqhs2bKYM2cOxowZk+MFEhEREWVHtsONSqXCuHHjMG7cOMTFxQEATE1Nc7wwIiIioo+R7XCTHkMNERER5TcfdRE/IiIiovyK4YaIiIgUheGGiIiIFIXhhoiIiBTlo8KNl5cXXrx4kdO1EBEREX2yLIebx48fS/8OCAjA69evAQB169blnbaJiIgo38jyUvAaNWrAwsICzZo1w5s3bxAREYEKFSrg0aNHSE5Ozs0aiYiIiLIsyyM3r169wp49e2Bvbw+tVouOHTvi888/h0ajwbFjxxAVFZWbdRIRERFlSZbDTXJyMpo0aYIJEyagaNGiCAkJwZYtW6Cvr4/NmzejcuXKqF69em7WSkRERPSvsnxaqkSJErCzs0OzZs2QlJSExMRENGvWDAYGBti1axfKlSuHy5cv52atRERERP8qyyM3T548wYwZM2BkZISUlBTY29vjiy++QFJSEoKDg6FSqdC8efPcrJWIiIjoX2U53JQuXRqdOnWCr68vihUrhsuXL2P06NFQqVSYOHEizM3N0bJly9yslYiIiOhfffRF/MzNzdGrVy8YGhri999/x8OHDzFy5MicrI2IiIgo2z7qruDXrl1DuXLlAAAVK1aEoaEhbGxs0Lt37xwtjoiIiCi7Pirc2NraSv++ceNGjhVDRERE9Kl4bykiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFFnDzenTp9GpUyeULVsWKpUKBw4c+GD/kydPQqVSZXio1eq8KZiIiIjyPVnDTXx8POrXr4+1a9dma787d+4gMjJSelhZWeVShURERFTQGMj54h06dECHDh2yvZ+VlRVKlCiR8wURERFRgVcg59zY2dmhTJkyaNeuHc6dOyd3OURERJSPyDpyk11lypTB+vXr0ahRI2g0GmzatAnOzs64dOkSGjZsmOk+Go0GGo1Geh4bG5tX5RIREZEMClS4qV69OqpXry49d3JywoMHD7BixQps27Yt0318fX0xd+7cvCqRiIiIZFYgT0ul16RJE9y/f/+92318fBATEyM9IiIi8rA6IiIiymsFauQmM6GhoShTpsx7txsZGcHIyCgPKyIiIiI5yRpuXr9+rTPq8vDhQ4SGhqJUqVKoUKECfHx88OTJE/z4448AgJUrV6Jy5cqoXbs23rx5g02bNuH333/Hb7/9JtdbICIionxG1nBz5coVtGrVSno+fvx4AICnpyf8/f0RGRmJ8PBwaXtSUhImTJiAJ0+eoFixYqhXrx6OHz+ucwwiIiIq3GQNN87OzhBCvHe7v7+/zvPJkydj8uTJuVwVERERFWQFfkIxERERUXoMN0RERKQoDDdERESkKAw3REREpCgMN0RERKQoDDdERESkKAw3REREpCgMN0RERKQoDDdERESkKAw3REREpCgMN0RERKQoDDdERESkKAw3REREpCgMN0RERKQoDDdERESkKAw3REREpCgMN0RERKQoDDdERESkKAw3REREpCgMN0RERKQoDDdERESkKAw3REREpCgMN0RERKQoDDdERESkKAw3REREpCgMN0RERKQoDDdERESkKAw3REREpCgMN0RERKQoDDdERESkKAw3REREpCgMN0RERKQoBnK++OnTp7Fs2TJcvXoVkZGR2L9/P9zc3D64z8mTJzF+/HjcvHkTtra2mDFjBgYMGJAn9RIRUcFRaeqvcpdQaD1a7Crr68s6chMfH4/69etj7dq1Wer/8OFDuLq6olWrVggNDcXYsWMxZMgQHDt2LJcrJSIiooJC1pGbDh06oEOHDlnuv379elSuXBnLly8HANSsWRNnz57FihUr4OLikltlEhERUQFSoObcXLhwAW3bttVpc3FxwYULF967j0ajQWxsrM6DiIiIlKtAhRu1Wg1ra2udNmtra8TGxiIxMTHTfXx9fWFubi49bG1t86JUIiIikkmBCjcfw8fHBzExMdIjIiJC7pKIiIgoF8k65ya7bGxsEBUVpdMWFRUFMzMzFC1aNNN9jIyMYGRklBflERERUT5QoEZuHB0dERQUpNMWGBgIR0dHmSoiIiKi/EbWcPP69WuEhoYiNDQUwNul3qGhoQgPDwfw9pSSh4eH1H/48OH4+++/MXnyZNy+fRvr1q3D7t27MW7cODnKJyIionxI1nBz5coVNGjQAA0aNAAAjB8/Hg0aNMCsWbMAAJGRkVLQAYDKlSvj119/RWBgIOrXr4/ly5dj06ZNXAZOREREElnn3Dg7O0MI8d7t/v7+me4TEhKSi1URERFRQVag5twQERER/RuGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlKUfBFu1q5di0qVKsHY2BhNmzbFH3/88d6+/v7+UKlUOg9jY+M8rJaIiIjyM9nDza5duzB+/HjMnj0bwcHBqF+/PlxcXPDs2bP37mNmZobIyEjpERYWlocVExERUX4me7j57rvvMHToUAwcOBC1atXC+vXrUaxYMWzevPm9+6hUKtjY2EgPa2vrPKyYiIiI8jNZw01SUhKuXr2Ktm3bSm16enpo27YtLly48N79Xr9+jYoVK8LW1hZdunTBzZs339tXo9EgNjZW50FERETKJWu4iY6ORmpqaoaRF2tra6jV6kz3qV69OjZv3oyDBw9i+/bt0Gq1cHJywuPHjzPt7+vrC3Nzc+lha2ub4++DiIiI8g/ZT0tll6OjIzw8PGBnZ4eWLVti3759sLS0xIYNGzLt7+Pjg5iYGOkRERGRxxUTERFRXjKQ88VLly4NfX19REVF6bRHRUXBxsYmS8cwNDREgwYNcP/+/Uy3GxkZwcjI6JNrJSIiooJB1pGbIkWKwN7eHkFBQVKbVqtFUFAQHB0ds3SM1NRUXL9+HWXKlMmtMomIiKgAkXXkBgDGjx8PT09PNGrUCE2aNMHKlSsRHx+PgQMHAgA8PDxQrlw5+Pr6AgDmzZsHBwcHVK1aFa9evcKyZcsQFhaGIUOGyPk2iIiIKJ+QPdz07t0bz58/x6xZs6BWq2FnZ4ejR49Kk4zDw8Ohp/e/AaaXL19i6NChUKvVKFmyJOzt7XH+/HnUqlVLrrdARERE+Yjs4QYAvLy84OXllem2kydP6jxfsWIFVqxYkQdVERERUUFU4FZLEREREX0Iww0REREpCsMNERERKQrDDRERESkKww0REREpCsMNERERKQrDDRERESkKww0REREpCsMNERERKQrDDRERESkKww0REREpCsMNERERKQrDDRERESkKww0REREpCsMNERERKQrDDRERESkKww0REREpCsMNERERKQrDDRERESkKww0REREpCsMNERERKQrDDRERESkKww0REREpCsMNERERKQrDDRERESkKww0REREpCsMNERERKQrDDRERESkKww0REREpCsMNERERKQrDDRERESlKvgg3a9euRaVKlWBsbIymTZvijz/++GD/PXv2oEaNGjA2NkbdunVx+PDhPKqUiIiI8jvZw82uXbswfvx4zJ49G8HBwahfvz5cXFzw7NmzTPufP38effv2xeDBgxESEgI3Nze4ubnhxo0beVw5ERER5Ueyh5vvvvsOQ4cOxcCBA1GrVi2sX78exYoVw+bNmzPtv2rVKnz55ZeYNGkSatasifnz56Nhw4ZYs2ZNHldORERE+ZGs4SYpKQlXr15F27ZtpTY9PT20bdsWFy5cyHSfCxcu6PQHABcXl/f2JyIiosLFQM4Xj46ORmpqKqytrXXara2tcfv27Uz3UavVmfZXq9WZ9tdoNNBoNNLzmJgYAEBsbOynlP5eWk1CrhyX/l1ufaYAP1c55ebnSsrG71v55Mb3bdoxhRD/2lfWcJMXfH19MXfu3Azttra2MlRDucl8pdwVUG7g50pU8OTm921cXBzMzc0/2EfWcFO6dGno6+sjKipKpz0qKgo2NjaZ7mNjY5Ot/j4+Phg/frz0XKvV4sWLF7CwsIBKpfrEd6AcsbGxsLW1RUREBMzMzOQuh3IQP1vl4merTPxcMyeEQFxcHMqWLfuvfWUNN0WKFIG9vT2CgoLg5uYG4G34CAoKgpeXV6b7ODo6IigoCGPHjpXaAgMD4ejomGl/IyMjGBkZ6bSVKFEiJ8pXJDMzM34zKRQ/W+XiZ6tM/Fwz+rcRmzSyn5YaP348PD090ahRIzRp0gQrV65EfHw8Bg4cCADw8PBAuXLl4OvrCwDw9vZGy5YtsXz5cri6umLnzp24cuUKNm7cKOfbICIionxC9nDTu3dvPH/+HLNmzYJarYadnR2OHj0qTRoODw+Hnt7/FnU5OTkhICAAM2bMwLRp01CtWjUcOHAAderUkestEBERUT4ie7gBAC8vr/eehjp58mSGtp49e6Jnz565XFXhYmRkhNmzZ2c4hUcFHz9b5eJnq0z8XD+dSmRlTRURERFRASH7FYqJiIiIchLDDRERESkKww0REREpCsMNERFRIabEqbcMN0RERIWUVquVrtb/999/y1xNzmG4ISpklPhXGuU8/n9SOKRdR27q1KmYOXMmnj17JnNFOSNfXOeGiPLGgwcPsGPHDjx8+BBt27ZFixYteBNZAvA2zKhUKrx58wZFihTRuXgqKU/a5w0Aly9fxpEjR7Bx40ZYWVnJXFnO4HVuKEekfaNERERIPxgtLS2h1Wr5QzKf+PPPP9G+fXtUr14dkZGRCAsLw4ABA7BgwQLF/ECjj5P2/Xvo0CFs27YNMTExGD9+PJo0acJ78SncsmXL8PfffyM1NVVRtzHibx3KESqVCvv27UOLFi3QokULfPXVVzh9+jT09PSg1WrlLq/Qu3HjBpycnDB69GgEBgbi3r17mDlzJvz9/XHt2jW5yyOZqVQqnDt3Dv369UPJkiWRkpKCPn36YN26dYiMjJS7PMpFT548wYYNG/DHH38o5pQUwHBDnyht4C88PBzDhw/HpEmTMGnSJNSoUQPt2rXD8ePHGXBk9s8//8DZ2Rn29vaYPHmydEn3cePGwcrKCg8ePJC5QsoP1Go1Jk6ciPXr1+P48ePSvzdt2sSAoxCZ/RxeuXIl5s2bh2vXrmHHjh2Ij4+XobKcxzk39ElUKhVOnjyJv//+G0OHDsXIkSMBAK6urjA2NsaXX36JI0eOoF27djxFJRMLCwv07NkTJ06cwIYNG9C7d28p1Dx79gwVKlSQu0SSQdqpqODgYDx69AgXL17UmX81bdo0AICfnx/09fXh6emJcuXKyVUufaL0P3+vXLmCxMREaLVatGzZEjNmzEBcXBwmT56MYsWK4euvv0axYsVkrvgTCaJPEBcXJ3r06CFUKpXo2rWrzranT5+KYcOGCWNjY3Ho0CGZKizcUlNTpX+PGTNGVKpUSfj7+4vg4GBRvnx5MWbMGBmrI7nt27dPFClSRNSoUUOoVCrh7Ows7t+/r9Nn8eLFomjRomLp0qUiJSVFpkrpU2i1WunfU6dOFbVr1xaVKlUSjRs3Fq1bt5a2+fj4CENDQ7Fx40YRFxcnR6k5huGGPtnly5dF//79hbGxsbh69aoQ4n/fTJGRkaJv376idOnSIj4+XuebjPJG+l9Io0ePFra2tqJEiRJi4MCBUnv6EESFQ0REhPDw8BAbNmwQsbGx4rvvvhN16tQR3t7eGQLOd999J+7evStTpZRTli9fLiwsLMSFCxdEcnKymD9/vlCpVOL48eNSn6lTpwqVSiUOHDggY6WfjuGGsiUtnKSkpAiNRiO137lzR3Tq1ElYW1tnCDhqtVo8ffo074st5NKHmvT/njZtmjAzMxMrV64UL168kKM0klloaKjo0qWLaN26tQgLC5PaV61aJezs7ISXl5d48OCBjBVSTktKShIeHh5i06ZNQgghDh48KMzMzMTGjRuFEELExsZKfdetWyeSk5NlqTOncAIEZZn47zn6o0ePwt3dHW3atMHIkSNx5coVfP7551ixYgWcnJzg6uqK0NBQqFQqCCFgbW2NMmXKyF1+oRAWFoY5c+ZAo9FAX19fmkCor6+P1NRUAMDChQsxYMAArFq1CgEBAYiOjpazZMoj4r+T/+/du4cjR44gIiICwcHBiI2NlfqMGTMGAwcOxKVLlzB//nw8evRIpmrpU4l3rvKip6eH27dvQ09PD8eOHYO7uzsWL16MoUOHIiUlBRs3bkRAQAAAYMSIETAwMEBKSoocpecIhhvKMpVKhV9++QVdunRBiRIl0KxZMwQFBcHb2xs//fQTqlSpgkWLFqFFixZo0qQJrl27Jl0kivLGnj17sH37dsyePRtJSUk6K9X09fWlH1arVq1C165d4ePjg3379nE1WyGgUqmwf/9+tG/fHh07dsSkSZNQqVIlTJkyBX/99ZfUb8yYMejevTv+/vtvFC1aVMaK6VOk/ezdvn07zp49C319fbRs2RIBAQHo1asXli1bhhEjRgAAoqOjceLECbx69UrnGAYGBXjNkbwDR5SfpR+m1Gq14uXLl6JZs2Zi4cKFUvuLFy9Et27dhKOjo7h+/boQQoirV68KDw8PcefOnTyvubB6+PChCAoKEikpKWLhwoWiUaNGYuLEidKpw/RzatKfopo/fz7nUihc2unhuLg4MXjwYPHtt99K27Zt2yacnZ1F9+7dxe3bt3X24ynLgi8sLEx8/vnnYu3atUIIIc6ePSusrKxE06ZNpZ/PT58+FR07dhSOjo6KmjDOcEOZWrZsmZg9e7bO/+wJCQmiXr16Ys2aNUKIt+dwhRDi5cuXolKlSmLixIlS3/TzcSh3PXnyRJQuXVpUq1ZNHDx4UKSmpop58+a9N+BoNBoxbdo0sWLFChmrprx07tw5UaNGDdGsWTNx5swZnW0//vijcHZ2Fr179xY3btyQqULKLVOmTBEVKlSQwurx48eFpaWlsLe3F9WrVxdOTk6iUaNG0s9zpQQcnpaiTCUnJ6Nv377Q19dHUlISAEhzNtKGsA0MDJCcnIwSJUqgXbt2OheDK1KkSN4XXUjdvXsXL168QIkSJfDDDz/gwIEDmD59Ojp37oyTJ09i+vTp0imqxMREjB8/HkuWLEHbtm3lLp3ySNWqVWFmZobz589LV6FNOxX59ddfY8iQIbhz5w6WLl2K5ORkOUulj/TuqeW0z3HIkCGwsbHBgQMHIIRAmzZtcOLECUyaNAkeHh6YOHEiLl68CENDQ6SkpEBfX1+O8nOe3OmK8rczZ86IyZMni4iICCGEELt37xb6+vpi9erVOv26du0qvvnmGy71lsmgQYOEnZ2d6N69u2jZsqU4cOBAhhGc2NhYMWHCBFGsWDFpRRsVHlFRUaJx48aievXq0qnI9N+vO3fuFI8ePZKrPMohW7duFQ8fPhSJiYlCiLefcffu3UWbNm0+uJ9SRmzSFODZQpST0l+9MiUlRZpIdv78eezbtw+Ghobw8vJCz549cffuXXh7eyMkJASVKlWCWq1GYGAgLl26xAnEeUyj0cDIyAjdu3eHVqtF3759sWHDBixbtgwqlQrTp08HAPz6669o0KABnjx5gnPnzqFhw4YyV065Rfx3VeO9e/fw7NkzmJqaoly5crCyssLhw4fRtm1bdO/eHQcOHMBnn30m9e/du7fcpdMnunfvHlasWAFvb2/07NkT7du3R48ePeDr6wtXV1f8+OOP8PDwyHRfxYzY/BdPSxGAt8sEHz9+DCEEDAwM8Msvv+D777/H5MmT4eHhgV9//RWrVq3CP//8g+nTp2Pfvn24ffs2AgMD8ejRI5w7dw61atWS+20UChEREdi/fz8ASPeJaty4MS5evIh79+5h/fr1sLa2xrJly3Do0CFMnz4dbdq0gZGRES5dusRgo2BpQSXtJraDBg1CkyZN0L9/f+zatQulS5eW7vfWs2dP3Lt3j3+QFGDineXeVapUQUhICFasWAF9fX24u7vD3d0du3btgoODA27evAkg83tMKY6s40aUbyQkJIg6deoIZ2dnsWvXLqFSqURAQIC0fc6cOcLOzk5MnTpVREZGSvuk/y/lvvDwcGFhYSFUKpXo2LGj2LVrl7Tq4f/+7//EF198IZ49eyZu3bolunXrJlq1aiV2794ttFqtiI6Olrl6yguXLl0SZmZmYu3ateLp06ciMDBQuLu7C3t7e7F7924hhBDPnj0TlSpVEs2aNZMmklLBkn4F5JMnT8T9+/d1FnJotVpx5coVMWzYMNGyZUuhUqmEoaGhCA0NlaPcPKcS4p3oR4WSEAJ37txBs2bNkJCQgPXr18PT01M67QEAc+fOxcGDB9GxY0d888030k32xH//WqTcFxYWhh49esDQ0BAajQYNGzZEYGAgpk2bhhIlSmDbtm0YOXIkOnTogFu3bsHb2xtFihTBrl27YGJiInf5lIuSk5NhaGiIVatWYe/evTh9+rS07c8//8SiRYuQlJSE7du3o3jx4oiOjkZcXBwqV64sY9X0MdJPI5g9ezaOHTuGa9euoUePHmjfvj369+8v9X3z5g1SUlLg5+eHnTt3omXLlvj222+hUqmU/XNb3mxF+UlYWJgwMDAQJiYmwtXVVWpP/9fAvHnzRMWKFcXcuXMVNwGtoLh7967o1q2bcHNzE/v27RP79+8Xzs7Ows3NTahUKtG0aVPpM7t9+7Y0GZyU59GjR2LRokU6bX5+fqJGjRoiKipKp/3gwYPC0NCQ159SkFmzZglLS0uxb98+cfHiReHs7Czq1q0r1q1bJ/VJfxsFX19fUa1aNWmysZJxzg1JKlSogL/++gtnz55FSEgIvvzySwBvl3WnLQefOXMmJk6ciP79+ytuAlpBUa1aNSxatAgajQbr169HrVq1cOjQIUyZMgWurq7w8vJCkSJFIIRA9erVUb58eblLplyg1WqxY8cObNy4EbNnz5baK1SogKdPn+K3337TmVtRvXp1VK1aVfpepoLt7Nmz2L9/P/bu3YuuXbtCo9Hg4sWLMDc3x/r167Fp0yYA/7tkBwAMHjwYycnJuHbtmpyl5w250xXJJ20Z6O3bt8Xp06d1bqB39uxZUbZsWdGhQwep38qVKzMsASf53L17V7Rv3160b99enD17Vu5ySAZqtVrMmjVLNGjQQEybNk1qHzt2rDA2Nhb+/v7i0aNH4s2bN2LSpEmiSpUq4tmzZzJWTDklMjJSrFixQiQlJYnffvtNWFhYiM2bNwu1Wi2qVKkiqlevLpYsWaKzz9y5c0XJkiWleZNKxnBTyO3du1eYm5uLypUrC0NDQ7F69WrpSpZnz54V5cuXF7Vq1RJff/21MDAwENeuXZO5Ykrv7t274ssvvxQuLi4ZrjxLhYNarRbTpk0TDRo0EFOnTpXax40bJywsLIStra1o1KiRsLS0FMHBwTJWSjktLi5OJCcni27duonp06dLUwW6du0q6tSpI8aMGaNzLaO1a9cWmmtccUJxIST+OwE4LCwMXbt2xbBhw+Di4oIdO3ZgyZIlmDJlCkaMGAELCws8fPgQM2fOhKGhIcaPH4+6devKXT694969exg/fjyio6OxYsUKODg4yF0S5ZK7d+/i1KlTqFKlClq3bi1NIlar1Vi7di3+7//+Dx07doSvry8A4OTJk1Cr1UhKSkKLFi1QqVIled8AZdubN29gbGz83u3JyclwcHBA+/bt4evri6SkJAwYMABdunRBr169oFKpkJqaWuimETDcFFJBQUG4du0a7ty5g9WrV8PQ0BAAsHz5csydOxeTJ0/GsGHDYGVlBQA6q6Yo/7l9+zZmzpyJ5cuXo0KFCnKXQ7ngxYsXqFixIuLj42FqaooaNWqgZs2a6NOnDxo2bAhLS0vMnDkTp0+fhoODA5YuXSp3yfSJfv75Z+zduxdLly6VVqemJ4RAfHw8vLy8EBERgXr16uH69et48eIFrly5Aj09PZ2VVYUJr1BcSB06dAirVq1C9erV8fLlSynETJgwASqVCgsXLkRCQgK8vb1hbW3NYJPP1ahRAzt27OA9vRSsVKlS8Pb2xoYNG/D1119Do9EgMTER/fr1g7m5Odq0aQMrKyt8/vnnOHr0KIyMjDB//ny5y6ZPYGVlhV27dqFo0aKYP38+ypUrp7NdpVLBxMQE3t7eWL16NUJDQ1GqVCkcOXKkUAcbgCM3hdrcuXMxd+5crF27Fp6enihWrJi0bcGCBfjhhx8QHBwMCwsLGaskovS/pCZNmoRff/0Vw4YNw+jRo/Hw4UPcu3cP69atQ1xcnHR9m7Jly+LPP//k928BlXYq6fz582jVqhX69OmDRYsWSQFHvHN9scjISFhbW0vXr0l/G53CiOGmEEj7Jnj8+DHevHkDjUaD2rVrAwDGjh0LPz8/+Pn5oW/fvihatKi034sXL1CqVCm5yiaidNLPm5g0aRJ27doFb29vDBgwABYWFkhOToZKpcKxY8dw584ddOzYETVq1JC5avoUaaH27NmzaNOmTYaAAwBqtRqdO3dG9+7dMWXKFAC8sCrAcKN4af+T79+/HwsWLMDLly9hbW2NMmXKYN++fQCAiRMnYvXq1diwYQN69eoljeDwG4Qof0kfcKZMmYKdO3fC29sb/fv3l04tk7K8G3B69+4NX19flCtXDtHR0ejRowciIiJw+/Ztae4kcc6N4qlUKhw/fhzu7u747rvv4OrqiuPHj2Pw4MHSHWLTLsU9aNAgGBgYSJfuZrAhkkdCQoLOaeI0+vr6UsBZsmQJAOD777+Hnp4e3N3dYWlpmdelUg5JCzFpf1Sm/Tdt7kzz5s0RFBSENm3aQF9fH97e3hgzZgyeP38uBZvCfioqPY7cKJwQAlOnToWenh58fX3x5MkTNGvWDF999RXWrFmj03f69Ono378/atasKVO1RHTv3j3MnDkTbdq0wdChQzPtk34EZ9q0aVizZg0WL16M4cOHF9oJpAVZYmKiNCUgNDQUdnZ2GfqkhZ9z586hffv2SExMRL169XD58mUGm0zwu0Ah3pdRVSoVrl+/DiMjIzx//hwODg5wcXHB6tWrAQA7duzA1q1bAQALFy5ksCGS0fXr1+Hs7IzixYvr3DrhXWkjOACwaNEijB8/Hu3bt2ewKYB2796NWbNmAXg7B7JLly549epVhn5pIzjNmjXDsWPH0Lp1awabD+BXQyFUKhWeP38OPT09WFhY4MCBA0hOTkbPnj3RqlUrBAcHo2HDhujQoQM2bNgAIQQSExNx+vRplClTBklJSVxGTCSjBw8eoGPHjvD09MS8efPe+8sq7S/49Keo5syZk7fFUo5JSkrC8uXLcfbsWdy+fRtnzpxBiRIlMp3zqKenh9TUVDRv3hzHjx8HAAab92DMVwAhBGJiYlCzZk2sWrUKmzdvRrdu3ZCSkgIA+OKLL3Dy5EkUK1YMEyZMAPD2G2rBggU4fPgw3N3dGWyIZLZ37140atQIM2fOlE45PXnyBJcuXcL69etx9epVJCUl6YzOFLarziqJeHv7I/Tv3x/t27fHH3/8gW7duuHzzz8H8P45j+9+5gw2meNXRQFUKhXMzc2xceNG9OnTB6mpqVizZg369u0LIQQcHBzw448/okePHhg+fDhSU1NRunRpnD17FseOHUO1atXkfgtEhd7169eRkJAgzb3Yu3cvdu3ahd9//x0JCQkoW7Ysxo0bh5EjR3KyfwH37qhM8+bN0axZM8yePRuWlpaYNGlSptcn4grWrOPITQGV/ny8RqMBANSrVw/A22+A6OhoREdHS7Pu27Vrh8DAQHTp0gWVK1eGs7Mzzp07hwYNGshSPxG9vW9Q2ghr+/btcfPmTcycORMjR47EiBEjYGVlhR07diAhIQH169dHQEAAEhMTZa6aPlVaQNmyZQsCAgLg4+ODmTNnYsuWLVi6dCmWLVuGFy9eSP3PnDmjsx/9O47cFFB6enqIiIiAVqtFxYoV8csvvyA6OhohISH466+/0KtXL7x58wYTJkyQ/gJo0qQJmjRpInPlRAQAN2/exNixY+Hj44PWrVvD2dkZAwYMwP/93/9BX18fmzdvRtOmTaXl3S4uLli1ahXevHmT6TJxKlg0Gg38/f2RkJCA1NRU9OnTB56enlCpVBgwYACSk5PRvXt3LFmyBI8fP8aVK1cYbrIj1+87TrkiLi5OuLm5iSZNmoi1a9cKlUoldu/eLW3funWrUKlUYsaMGeL58+dCCCGWLl0q9u7dK1fJRPRfqampwsnJSahUKtGwYUNx5swZaVtiYqKIj4/PsM+IESNE9+7dRUJCQl6WSjlEq9VmaHv58qXo0qWLcHR0FFu3bhVJSUlCCCG2b98uLC0tRZ06dUTjxo2ldso6XuemAAsMDMS4ceNw584dLFmyBOPHj0dSUhIMDQ2hUqnw448/YvDgwdJt7/fs2YNLly5leg0FIspbBw8exJo1a/DkyRO8fPkSu3fvhpOTE/T19XXuJRUbGwtfX19s2rQJJ0+elG6dQgXTo0ePUKlSJel5TEwMvv76a0RHR2PkyJHo3bs3DA0N8ddffyExMRF2dnbQ09Pjqqhs4pybAiKza15Ur14dSUlJqFy5Mn799Vc8fPgQRYoUQUpKCoQQ8PDwQEBAAOLj4xEbG4vLly8z2BDlE2m/4NavXw8XFxf06NED58+fBwAp2Pj5+WHQoEHYuXMnfvvtNwabAm7Lli3o3r07AgMDpTZzc3Ns3boVBgYGWLBgAXbv3o3k5GTUrFkTDRs2lJZ/M9hkD0duCpDbt29j27ZtGDZsGCpUqADg7V8Bd+7cwdKlS5Gamgp/f39UrlwZycnJ0n1GUlNTkZKSAiMjIznLJyrUMruWlLe3N/744w8EBQXB3d0dFy9exO7du/HFF19ArVbjp59+QnR0NAYNGoQqVarIVDnllLCwMHTu3BnW1taYPHky2rZtK227cuUKWrdujQoVKuDbb7/Fl19+KWOlBR/DTQGRnJyMZs2a4cqVK6hatSo6deoER0dH9OjRAwDw22+/YeHChdDT08PmzZtRuXJlLF++HCYmJhg6dCivXEoko2vXrmHkyJH46quv0Lx5czRv3hwAEBkZiQEDBmDevHlo2rQpvvzyS1y7dg27d+9G8+bNkZycDCEEr0NVAKU/tZheWFgYunXrBnNzc/j4+KBdu3YAgKCgIGzbtg2WlpZYvHgxr2H0iRhuCpBly5bBwMAAderUwblz5/D999/jyy+/RJs2bTBo0CAcPXoU69evR3BwMFq3bo1t27YhNDRUWiJORHlPq9WiRYsWOH/+PFq2bIng4GAMHz4cDg4O6Nq1K/r06QNjY2P4+/sDALp06YIjR47g1KlTcHR0lLd4+igi3fVotm/fjnv37qFkyZJo0aIFGjZsiIiICHTt2hUlSpRA9+7d0bp1a0yaNAkNGjTA3LlzAejeP4yyj+GmADl58iS6dOmCoKAgNGrUCJGRkdi4cSN8fX3RtGlTfP3111CpVHj27BlCQ0Mxa9YsnqMnygeeP3+OFi1awMLCAp6enjhz5gwePXoEc3NzODs7Y/78+Th06JA0otO7d2/Mnz9fulotFRzpg82kSZOwadMm1KhRAxqNBteuXcOGDRswePBgREREYMyYMQgNDUVycjLKly+PM2fOwNDQkBfrywEMNwXMpEmTEBkZiU2bNsHY2Bh9+vTBn3/+KYWd06dPY8OGDbylAlE+kfaL6tmzZ6hfvz4cHR0xdepUVK1aFVOnTsXDhw8RFBSE8+fPw8HBQe5yKYeEhIRg9uzZmDlzJho3box//vkHa9aswfz587F9+3b06dMHMTExePz4MV6+fAlHR0fo6+tzVVQOYbgpYH7++Wd89913OHv2LIYNG4ZDhw4hKCgItWvXxu3bt3H06FG0a9eOIzZEMvrnn38QFRWF1NRU1K1bV2qPiopCgwYNUKVKFWzfvh0VK1aEWq1GVFQU6tevL2PF9KnSj7YEBATAz88PqampOHLkCMzNzaV+kyZNQkBAAM6fP4+KFSvqHIOnonIOZ5kWMD169IChoSEMDQ1x5MgRHDt2TAoyNWrUwNixYxlsiGR048YNdOjQAa6urujUqROGDRsmbbO2tkZISAgePnyIfv364fbt27CxsUH9+vXBvzMLLq1Wq3MaKS4uDq9evcJff/2FmJgYAG+DCwB07twZAPDy5csMx2GwyTkMNwVI2g+/KVOmoGrVqli7di1/KBLlI3/++SccHBzQokULbNmyBV999RW2bt0KPz8/AG+Xg1tbW+Pq1asICwvDyJEjce3aNQC8b1BB9fvvv+P58+cAgKlTp2L+/Pn45ptvMGHCBFhbW2PMmDF4+PChFFzKli0LfX19KfRQ7uCJvQIk7Yefvb09tFotrl69Cjc3N/5QJMoH7t+/DwcHB0ycOBHz588HAFSsWBE//PADHjx4AADSPLi0gFOxYkX4+Phg//79nCNXAMXGxsLT0xPly5dHnTp1sHv3bukmlwMGDIBGo8HWrVvh4eGB2bNnIyUlBatXr0bp0qWlyeOUOxhuCiBra2vMnj0bw4cPR6dOnXgzTCKZabVabN68GaamptKNagFg586dSE5Oxr1797By5UpYWFigZ8+eUKlUsLa2RlhYGGJjYxlsCigzMzPcvn0b1tbWuH79Og4cOIB69epJc2e++eYb6Ovrw9fXF507d0a7du1Qr1497N27F/r6+pxjk4sYbgqoVq1aoXHjxihbtqzcpRAVenp6evDy8kJCQgJ27twJIyMjxMXFYdmyZZg+fTrs7OywY8cOREREYPr06ahWrRpGjx4NNzc3WFtby10+ZVPaBfqEEIiOjoaenh5MTU0xf/581KhRA+XLl5f6DBkyBHp6evjPf/6DEiVKYPjw4TA2NoZGo+FV43MRV0sVYG/evIGxsbHcZRDRf6nVaixcuBCBgYF48OABjh07htatWwOAtMR3zZo1CA4OxsSJE1GrVi2ZK6ZPcf78eTg5OQEAoqOj4eDgABsbG+zatQvlypXT6bt161Zs2rQJn332GWbPno3PPvtMjpILDU4oLsAYbIjyFxsbG8yYMQMuLi6oVasWQkJCpG1pN7/18vLCxo0bGWwKMCEEzp8/j+bNm8PX1xfPnj1D6dKlcfz4cajVavTr1w+PHj1CUlISevfuje+++w6enp74+uuvERwcjCVLliAlJUXut6FoHLkhIsphaSM4ly9fRteuXTFlyhQA4AXaFGb+/PlYtWoVJk+ejIEDB8LS0hJhYWFo164dEhISYGlpicTERISGhkp/jG7ZsgWtW7fOcI0bylkMN0REuSAt4ISEhKBNmzbSPYOo4El/gb53JwEvXLgQy5Ytw7Rp0zBo0CCULl0aGo0GS5cuhYmJCUaPHg0DAwPOscljDDdERLlErVbDx8cHjx8/xs6dO3VWUlHBs3TpUpQqVQr9+/fXmRawYMECzJs3DwsXLoS7u3uGhR5cFZX3GG6IiHJRVFQUAHBVVAGTfrQm7d/u7u7YvXs3tm7dim7duukEnD59+uDUqVMYPnw4xowZg5IlS8pVOoFLwYmIchVDTcH07NkzJCUl4eXLl7C0tESZMmWwY8cOmJubY/DgwdBqtejevTuKFi0KAChXrhwsLCxw8eJFzJo1S+bqieGGiIgonYCAAKxfvx7379+HWq1G5cqV4eLignXr1mHdunXQarUYOnQohBBo06YNypYtiydPnsDf3x/29vZQqVQ6Iz+U93haioiI6L+2bNmCkSNHYvny5ahRowYMDQ2xefNm/PTTT2jZsiWOHTsGABg1ahT27t2LMmXKIDU1FSkpKbh27RoMDAykC/iRfBhuiIiIAISEhKBnz55YtGgRevXqJbX/888/2L17NyZOnIjOnTvjp59+AgDpqtNv3rzBjBkzYGBgwMnD+QRPSxEREQGIiIiAiYkJWrRoIYUUIQQsLCzQt29fPH36FKtXr8bvv/+O1q1bw93dXWd/Bpv8g+NmREREeDtyo1arYWNjIwWbtHkzJUqUwNdff434+Hg8ffo00/0ZbPIPhhsiIiIANWvWRFxcHH777TcAyDAh+LPPPoONjQ1ev34tR3mUDQw3REREABo1agRDQ0Ns3LgR4eHhUntqaioAIDw8HKVLl8bnn38uV4mURQw3REREeDsy4+fnh0OHDsHHxwfBwcEA3p5uSkhIwJgxY2BmZgZnZ2d5C6V/xdVSRERE/5WSkgJ/f3+MGjUKlpaWqF+/PkqUKIHw8HDExcXh8uXLMDQ05OThfI7hhoiI6B2hoaH44Ycf8Ndff6FChQqoWbMmJkyYAAMDA97dvQBguCEiIsoijtgUDAw3REREmeAtFAouTigmIiLKBINNwcVwQ0RERIrCcENERESKwnBDREREisJwQ0RERIrCcENERESKwnBDREREisJwQ0RERIrCcENERESKwnBDREREisJwQ0RERIrCcENERESK8v/UPc5/GCW+YgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attention                 309,237,645,312      37.39%\n",
      "MLP                       412,317,351,936      49.86%\n",
      "RMS Norm                          179,200       0.00%\n",
      "Output Layer              105,396,568,064      12.75%\n",
      "\n",
      "\n",
      "Total forward FLOPs: 826,951,744,512\n"
     ]
    }
   ],
   "source": [
    "ops_per_matmul = 2  # Multiply + accumulate (MAC)\n",
    "ops_per_activation = 5  # Assuming SiLU\n",
    "ops_per_rms_norm = 7  # y = (x / sqrt(rms[x] + epsilon)) * gamma\n",
    "\n",
    "head_dimensions = embedding_dimensions // num_attention_heads\n",
    "\n",
    "# K, Q, V projections\n",
    "attention = (\n",
    "    ops_per_matmul\n",
    "    * tokens_per_sample\n",
    "    * (embedding_dimensions * 3 * embedding_dimensions)\n",
    ")\n",
    "\n",
    "# Attention logits\n",
    "attention += (\n",
    "    ops_per_matmul * tokens_per_sample * tokens_per_sample * embedding_dimensions\n",
    ")\n",
    "\n",
    "# Reductions\n",
    "attention += (\n",
    "    ops_per_matmul\n",
    "    * num_attention_heads\n",
    "    * (tokens_per_sample * tokens_per_sample * head_dimensions)\n",
    ")\n",
    "\n",
    "# Output projection\n",
    "attention += ops_per_matmul * tokens_per_sample * embedding_dimensions**2\n",
    "\n",
    "attention *= num_hidden_layers\n",
    "\n",
    "# Linear transformations\n",
    "mlp = (\n",
    "    ops_per_matmul\n",
    "    * tokens_per_sample\n",
    "    * (embedding_dimensions * (4 * embedding_dimensions))\n",
    ")\n",
    "mlp += (\n",
    "    ops_per_matmul\n",
    "    * tokens_per_sample\n",
    "    * ((4 * embedding_dimensions) * embedding_dimensions)\n",
    ")\n",
    "\n",
    "# Non-linear activations\n",
    "mlp += ops_per_activation * (4 * embedding_dimensions)\n",
    "\n",
    "mlp *= num_hidden_layers\n",
    "\n",
    "rms_norm = ops_per_rms_norm * embedding_dimensions * (num_hidden_layers + 1)\n",
    "\n",
    "output_layer = (\n",
    "    ops_per_matmul * tokens_per_sample * embedding_dimensions * vocabulary_size\n",
    ")\n",
    "\n",
    "flops = {\n",
    "    \"Attention\": attention,\n",
    "    \"MLP\": mlp,\n",
    "    \"RMS Norm\": rms_norm,\n",
    "    \"Output Layer\": output_layer,\n",
    "}\n",
    "\n",
    "plt.bar(flops.keys(), flops.values())\n",
    "\n",
    "plt.title(\"Model Operations\")\n",
    "plt.ylabel(\"# of FLOPs\")\n",
    "plt.xticks(rotation=45)\n",
    "\n",
    "plt.show()\n",
    "\n",
    "total_forward_flops = sum(flops.values())\n",
    "\n",
    "for name, count in flops.items():\n",
    "    print(f\"{name:20s} {count:20,d} {count / total_forward_flops * 100:10.2f}%\")\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "print(f\"Total forward FLOPs: {total_forward_flops:,}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we'll estimate the number of FLOPs for the backward pass. For this we use a simple heuristic of 2X the forward pass."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total backward FLOPs: 1,653,903,489,024\n"
     ]
    }
   ],
   "source": [
    "total_backward_flops = 2 * total_forward_flops\n",
    "\n",
    "print(f\"Total backward FLOPs: {total_backward_flops:,}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll do the same for the total FLOPs per roundtrip."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total roundtrip FLOPs: 2,480,855,233,536\n"
     ]
    }
   ],
   "source": [
    "total_roundtrip_flops = total_forward_flops + total_backward_flops\n",
    "\n",
    "print(f\"Total roundtrip FLOPs: {total_roundtrip_flops:,}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let's estimate the number of FLOPs using the method in the PaLM paper by Chowdhery, et al. Then, we'll compare the PaLM estimation with our own as a sanity check."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total PaLM FLOPs: 2,481,161,502,720\n"
     ]
    }
   ],
   "source": [
    "palm_flops_per_token = (\n",
    "    6 * total_parameter_count\n",
    "    + 12 * num_hidden_layers * num_attention_heads * head_dimensions * tokens_per_sample\n",
    ")\n",
    "\n",
    "total_palm_flops = palm_flops_per_token * tokens_per_sample\n",
    "\n",
    "print(f\"Total PaLM FLOPs: {total_palm_flops:,}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The two estimates are pretty close so let's proceed."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, let's estimate how long it would take to train over the optimal number of tokens given some common Nvidia Ampere generation GPU hardware configurations. Note that these results shown here are a theoretical scenario and do not factor in additional overhead such as activation checkpointing or network latency."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RTX A2000: 935.43 seconds/epoch, 18.25 days required\n",
      "RTX A4000: 348.64 seconds/epoch, 6.80 days required\n",
      "RTX 3090: 154.75 seconds/epoch, 3.02 days required\n",
      "A100 SXM: 44.01 seconds/epoch, 0.86 days required\n",
      "HGX A100: 6.79 seconds/epoch, 0.13 days required\n"
     ]
    }
   ],
   "source": [
    "from dataclasses import dataclass\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class Device:\n",
    "    name: str\n",
    "    advertised_flops: float\n",
    "    mfu: float\n",
    "\n",
    "    @property\n",
    "    def actual_flops(self) -> float:\n",
    "        return self.mfu * self.advertised_flops\n",
    "\n",
    "\n",
    "devices = [\n",
    "    Device(\"RTX A2000\", 63.9e12, 0.17),\n",
    "    Device(\"RTX A4000\", 153.4e12, 0.19),\n",
    "    Device(\"RTX 3090\", 285.5e12, 0.23),\n",
    "    Device(\"A100 SXM\", 624.0e12, 0.37),\n",
    "    Device(\"HGX A100\", 4992e12, 0.30),\n",
    "]\n",
    "\n",
    "for device in devices:\n",
    "    seconds_per_epoch = samples_per_epoch * total_roundtrip_flops / device.actual_flops\n",
    "\n",
    "    days_required = num_epochs_required * seconds_per_epoch / 60 / 60 / 24\n",
    "\n",
    "    print(\n",
    "        f\"{device.name}: {seconds_per_epoch:.2f} seconds/epoch, {days_required:,.2f} days required\"\n",
    "    )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}