tutorial_day2_reinforcemen_learning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Introduction to Reinforcement Learning with Python.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UJF5zfBmeJgb"
      },
      "source": [
        "# Introduction to Reinforcement Learning with Python\n",
        "\n",
        "In this tutorial we will explore about some essencial notions about reinforcement learning, some useful tools and what is still missing.\n",
        "\n",
        "- The reinforcement learning framework.\n",
        "- Markov decision process.\n",
        "- Q-learning algorithm\n",
        "- Conclusions\n",
        "- Open challenges\n",
        "\n",
        "Created by [Angel Ayala](https://github.com/angel-ayala/) for the Scientific Computing Summer School 2021, 20 - 24 Sept, 2021.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KmN-CE2BNy3o",
        "cellView": "form"
      },
      "source": [
        "#@title Helpers\n",
        "import random\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import ipywidgets as widgets       # interactive display\n",
        "\n",
        "\n",
        "seed_val = 202109\n",
        "\n",
        "\n",
        "def run_ep(agent, env, epsilon=0.99, function_update=None, gamma=0., alpha=0.):\n",
        "    \"\"\"Auxiliary function to run an episode until the end flag is True.\n",
        "\n",
        "    Args:\n",
        "        agent (RL Agent class): an instance of the RL agent.\n",
        "        env (Environment class): an instance of the environment.\n",
        "        epsilon (float): probability of selecting an action randomly.\n",
        "        function_update (function): the policy update function definition.\n",
        "        gamma (float): long-term discounted reward value.\n",
        "        alpha (float): learning rate.\n",
        "    \n",
        "    Returns:\n",
        "        int: the episode total reward.\n",
        "        list: vector with [steps_duration, actions_taken].\n",
        "    \"\"\"\n",
        "    # runtime vars\n",
        "    ep_reward = 0\n",
        "    steps = 0\n",
        "    actions_sequence = []\n",
        "\n",
        "    # initiate the environment\n",
        "    current_state = env.reset()\n",
        "    while not env.end:\n",
        "        # choose an action\n",
        "        action = agent.get_action(current_state, epsilon=epsilon)\n",
        "        # perform an action and observe the next state\n",
        "        next_state, reward = env.perform_action(action)\n",
        "        ep_reward += reward\n",
        "\n",
        "        if function_update:\n",
        "            q_values = function_update(agent, current_state, action,\n",
        "                                       reward, next_state, alpha=alpha,\n",
        "                                       gamma=gamma)\n",
        "\n",
        "        current_state = next_state\n",
        "        steps += 1\n",
        "        actions_sequence.append(action)\n",
        "    \n",
        "    return ep_reward, [steps, actions_sequence]\n",
        "\n",
        "\n",
        "def training_loop(agent, env, episodes, params=dict(), log=False):\n",
        "    \"\"\"Auxiliary function to train an agent for n episodes.\n",
        "\n",
        "    Args:\n",
        "        agent (RL Agent class): an instance of the RL agent.\n",
        "        env (Environment class): an instance of the environment.\n",
        "        episodes (int): the learning duration in number of episodes.\n",
        "        params (dict): the independent function's parameters.\n",
        "        log (bool): if want or not to print the ep end's log.\n",
        "    \n",
        "    Returns:\n",
        "        np.ndarray: the reward values for each episode.\n",
        "        dict: the history of the learning process:\n",
        "            => 'steps': the duration of each episode in number of steps.\n",
        "            => 'actions': the action history in each episode.\n",
        "            => 'best': the most recent best reward as \n",
        "                      (n_episode, Q_values, best_action_hist).\n",
        "            => 'Q_values': the accumulative Q_values of each episode.\n",
        "    \"\"\"\n",
        "    # reproducibility\n",
        "    random.seed(seed_val)\n",
        "\n",
        "    # runtime vars\n",
        "    rewards = []\n",
        "    n_steps = []\n",
        "    actions_hist = []\n",
        "    best_reward = -100000\n",
        "    best_Q = None\n",
        "    best_ep = 0\n",
        "    best_path = []\n",
        "    table_Q = agent.Q_values.copy()\n",
        "\n",
        "    for ep in range(episodes):\n",
        "        ep_reward, hist = run_ep(agent, env, **params)\n",
        "\n",
        "        rewards.append(ep_reward)\n",
        "        n_steps.append(hist[0])\n",
        "        actions_hist.append(hist[1])\n",
        "\n",
        "        table_Q += agent.Q_values\n",
        "\n",
        "        if rewards[-1] > best_reward:  # optimal case -14\n",
        "            best_Q = agent.Q_values\n",
        "            best_ep = ep\n",
        "            best_path = actions_hist[-1]\n",
        "            best_reward = rewards[-1]\n",
        "\n",
        "        if log:\n",
        "            print('EP', ep, 'end, ep_reward', ep_reward, 'steps', n_steps[-1])\n",
        "\n",
        "    return np.array(rewards), dict(steps=np.array(n_steps),\n",
        "                                   actions=np.array(actions_hist),\n",
        "                                   best=(best_ep, best_Q, best_path),\n",
        "                                   Q_values=table_Q)\n",
        "\n",
        "\n",
        "def plot_reward(reward, ylim=None, ax=None):\n",
        "    \"\"\"\n",
        "    Generate plot showing the reward learning curve.\n",
        "    \"\"\"\n",
        "    if ax is None:\n",
        "        fig, ax = plt.subplots()\n",
        "\n",
        "    ax.set_title(\"Learning curve\")\n",
        "    ax.plot(reward, label='Reward')\n",
        "    ax.set_xlabel('Episodes')\n",
        "    ax.set_ylabel('Reward')\n",
        "    if ylim is not None:\n",
        "        ax.set_ylim(ylim)\n",
        "    ax.grid()\n",
        "    ax.legend()\n",
        "\n",
        "\n",
        "def plot_quiver_max_action(env, value, ax=None):\n",
        "    \"\"\"\n",
        "    Generate plot showing action of maximum value or maximum probability at\n",
        "    each state (not for n-armed bandit or cheese_world).\n",
        "    Adapted from the Neuromatch notebook https://colab.research.google.com/drive/1vpMEmotHZvNRuorazdcVfYop560SJOJK\n",
        "    \"\"\"\n",
        "    if ax is None:\n",
        "        fig, ax = plt.subplots()\n",
        "\n",
        "    X = np.tile(np.arange(env.width), [env.height, 1]) + 0.5\n",
        "    Y = np.tile(np.arange(env.height)[::-1][:,np.newaxis], [1, env.width]) + 0.5\n",
        "    which_max = np.reshape(value.argmax(axis=2), (env.height, env.width))\n",
        "    which_max = which_max[::-1, :]\n",
        "    U = np.zeros(X.shape)\n",
        "    V = np.zeros(X.shape)\n",
        "    U[which_max == 0] = 1\n",
        "    V[which_max == 1] = -1\n",
        "    U[which_max == 2] = -1\n",
        "    V[which_max == 3] = 1\n",
        "\n",
        "    ax.quiver(X, Y, U, V)\n",
        "    ax.set(\n",
        "        title='Maximum Q values',\n",
        "        xlim=[-0.5, env.width + 0.5],\n",
        "        ylim=[-0.5, env.height + 0.5],      \n",
        "    )\n",
        "    ax.set_xticks(np.linspace(0.5, env.width - 0.5, num=env.width))\n",
        "    ax.set_xticklabels([\"%d\" % x for x in np.arange(env.width)])\n",
        "    ax.set_xticks(np.arange(env.width + 1), minor=True)\n",
        "    ax.set_yticks(np.linspace(0.5, env.height - 0.5, num=env.height))  \n",
        "    ax.set_yticklabels([\"%d\" % y for y in np.arange(0, env.height * env.width, \n",
        "                                                    env.width)])  \n",
        "    ax.set_yticks(np.arange(env.height + 1), minor=True)\n",
        "    ax.grid(which='minor',linestyle='-')\n",
        "\n",
        "\n",
        "def plot_path(env, action_hist, ax=None):\n",
        "    \"\"\"\n",
        "    Generate plot showing the resultant path for the best results.\n",
        "    \"\"\"\n",
        "    env.reset()\n",
        "\n",
        "    if ax is None:\n",
        "        fig, ax = plt.subplots()\n",
        "\n",
        "    draw_cells = np.zeros((env.width, env.height))\n",
        "\n",
        "    draw_cells[tuple(env.current_cell)] = 1.\n",
        "\n",
        "    for action_ in action_hist:\n",
        "        cell, reward = env.perform_action(action_)\n",
        "\n",
        "        if cell is None and reward == -1:\n",
        "            draw_cells[tuple(env.target_cell)] = 1.\n",
        "        elif cell is None:\n",
        "            continue\n",
        "        else:\n",
        "            draw_cells[tuple(cell)] = 1.\n",
        "\n",
        "    ax.pcolormesh(draw_cells.T[::-1], cmap='cividis' )\n",
        "    ax.set_title(\"2D Agent's path\")\n",
        "    #plt.show()"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cQtN6u8Sl20x"
      },
      "source": [
        "# Reinforcement learning\n",
        "\n",
        "The reinforcement learning (RL) methods were inspired by biological systems [1] in which an agent learns through interaction with its environment. The main idea is that after each action performed in the environment, the associated reward must increase its value as a well-selected action.\n",
        "Learning algorithms have some research since the 1890s with the studies of Pavlov [2], and it conditioned stimulus experiments with a dog. Some following research was conducted by Thorndike and his experiment with a trapped cat, noticing that the cat can escape faster at each trial.\n",
        "\n",
        "The reinforcement learning algorithm began to be of interest from the research of Sutton and Barto [3] as the most natural-like reward system, being confirmed by different biology studies [4] about the correlation between dopamine and reward prediction."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J5DQyEZzO-BN"
      },
      "source": [
        "## The RL framework\n",
        "The Figure 1 shows a basic diagram of the RL framework's main loop, and follows this sequence:\n",
        "\n",
        "- **State observation:** In this case, the agent must sense all the information about its environment to choose the most appropriate action for that given state ($s_t$).\n",
        "- **Action selection:** Different methods can be implemented and requires a strategy to differentiate which action to take, $a \\in A$.\n",
        "- **Action reward and state transition:** After action performs, the environment will transit to a new state ($s_{t+1} \\in S$). Additionally, a reward signal ($r$) is given and represents the quality of the action selected."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "01enbd0JO0zN"
      },
      "source": [
        "<img src=\"https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Fwww.cs.us.es%2F~fsancho%2Fimages%2F2018-12%2Freinforcement-learning-fig1-700.jpg&f=1&nofb=1\">\n",
        "\n",
        "Figure 1: Classic diagram for the RL framework."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rB9KVZXfUkn1"
      },
      "source": [
        "The main idea behind this algorithm is to improve the maximum expected return $G_t$. This can be formally described in a recursive form as:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "esXutqwgUlVf"
      },
      "source": [
        "\\begin{align}\n",
        "G_{t} &= \\sum \\limits_{k = 0}^{\\infty} \\gamma^{k} r_{t+k+1} \\\\\n",
        "&= r_{t+1} + \\gamma G_{t+1},\n",
        "\\end{align}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yuVcErjpU3MJ"
      },
      "source": [
        "where $r_{t+k+1}$ is the $t$_th_ reward value obtained at the moment $k$, and $\\gamma$ is the value to control the impact of future rewards, and $\\gamma \\in [0, 1]$.\n",
        "\n",
        "\n",
        "The RL framework possesses the following elements:\n",
        "\n",
        "- __Policy ($\\pi$)__: defines how to select an action to improve the overall results.\n",
        "- __Reward signal ($r \\in R$)__ : models the problem task through positive and negative values.\n",
        "- __Value function ($V$)__: specify how good is the expected return from a given state or action-state pair.\n",
        "\n",
        "\\begin{align}\n",
        "V_{\\pi}(s_t=s) &= \\mathbb{E} [ G_{t}\\; | \\; s_t=s, a_{t:\\infty}\\sim\\pi] \\\\\n",
        "& = \\mathbb{E} [ r_{t+1} + \\gamma G_{t+1}\\; | \\; s_t=s, a_{t:\\infty}\\sim\\pi]\n",
        "\\end{align}\n",
        "\n",
        "- __Environment's model__ (optional): allows inferring the following state and reward given an action.\n",
        "\n",
        "The learning is accomplished after a determined amount of experiences from the agent's interaction with the environment. Then, the value function is updated inside the learning loop to accurately compute the expected return of all the actions for the maximum number of possible states."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "piQJDm1PhF6n"
      },
      "source": [
        "## Markov decision process (MDP):\n",
        "\n",
        "Formally speaking, RL is a method that solves an MDP in terms of:\n",
        "\n",
        "- __States__: A vector with all the required info to make a decision, _Markov property_.\n",
        "- __Actions__: A value or vector to affect the current agent's state.\n",
        "- __Reward function__: Models the problem task as an action's feedback signal, showing the quality of the performed action.\n",
        "- __Transition function__: Defines the environment dynamics from one state to another given an action $a$."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gjaMY8YoFybz"
      },
      "source": [
        "### The Cliff environment\n",
        "\n",
        "A known toy environment to learn about RL is a grid-world named Cliff Walking. In this problem, the agent must learn how to go from the bottom-left corner (start position) to the bottom-right (goal position) cells of the grid. The idea is that the agent must avoid the cliff zone, which are the grid-cells between the starting and goal position. If the agent reaches any cell in the cliff zone, gets into a terminal state with a negative reward.\n",
        "\n",
        "<img src=\"https://cdn-images-1.medium.com/max/1600/1*FBJfEd_cuVW1CsDK7gCQsg.png\">\n",
        "\n",
        "Figure 2: The cliff walking problem represented in a 2D axis.\n",
        "\n",
        "This problem can be formulated as a Markov process, taking account the following elements from the environment.\n",
        "\n",
        "- **State $s \\in S$**: A vector with the current position of the agent, represented by 2 components $[x, y]$.\n",
        "- **Actions $a \\in A$**: The posibles valid actions that the agent can execute:\n",
        "  - Up $\\uparrow = 0$\n",
        "  - Down $\\downarrow = 1$\n",
        "  - Left $\\leftarrow = 2$\n",
        "  - Right $\\rightarrow = 3$\n",
        "- **Reward function**:\n",
        "$$\n",
        "r = \n",
        "\\begin{cases}\n",
        "    -100 & \\text{si cae en el abismo}\\\\\n",
        "    -1 & \\text{en otros casos}\n",
        "\\end{cases}\n",
        "$$\n",
        "- **Transition function**:\n",
        "$$\n",
        "s_{t+1} = \n",
        "\\begin{cases}\n",
        "    [x, y-1] & \\text{if} & a = 0\\\\\n",
        "    [x, y+1] & \\text{if} & a = 1\\\\\n",
        "    [x-1, y] & \\text{if} & a = 2\\\\\n",
        "    [x+1, y] & \\text{if} & a = 3\n",
        "\\end{cases}\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fhXs527eFegq"
      },
      "source": [
        "#@title The Cliff MDP declaration\n",
        "class CliffWalking(object):\n",
        "    \n",
        "    def __init__(self, width=12, height=4):\n",
        "        # grid size\n",
        "        self.width = width\n",
        "        self.height = height\n",
        "        # actions\n",
        "        self.n_actions = 4\n",
        "        # initial state\n",
        "        self.start_cell = [0, height -1]\n",
        "        self.current_cell = [0, height -1]\n",
        "        # objetive\n",
        "        self.target_cell = [width -1, height -1]\n",
        "        # easy check cliff zone cells\n",
        "        self.cliff = [[x, height -1] for x in range(1, width - 1)]\n",
        "        self.end = False  # episode end's flag\n",
        "\n",
        "\n",
        "        # limited transition funcion\n",
        "        self.transition = [lambda x : [x[0], max(x[1] - 1, 0)],              # up\n",
        "                           lambda x : [x[0], min(x[1] + 1, self.height -1)], # down\n",
        "                           lambda x : [max(x[0] - 1, 0), x[1]],              # left\n",
        "                           lambda x : [min(x[0] + 1, self.width -1), x[1]]]  # right\n",
        "\n",
        "    def reset(self):\n",
        "        # reset environment status\n",
        "        # print('new')\n",
        "        self.current_cell = self.start_cell\n",
        "        self.end = False\n",
        "        return self.current_cell\n",
        "\n",
        "    def compute_reward(self, state):\n",
        "        reward = -1\n",
        "\n",
        "        # check if is any cliff zone cell\n",
        "        if any(state == cliff_cell for cliff_cell in self.cliff):\n",
        "            # print('cliff fall', state)\n",
        "            reward = -100\n",
        "            self.end = True\n",
        "\n",
        "        return reward\n",
        "\n",
        "    def perform_action(self, action):\n",
        "        # check that action is valid\n",
        "        assert action in range(self.n_actions), 'Error! invalid action'\n",
        "\n",
        "        current_state = self.current_cell\n",
        "        next_state = self.transition[action](current_state)\n",
        "        reward = self.compute_reward(next_state)\n",
        "\n",
        "        # reached the target cell\n",
        "        if next_state == self.target_cell:\n",
        "            # print('Target reached!', self.target_cell)\n",
        "            self.end = True\n",
        "\n",
        "        self.current_cell = next_state\n",
        "\n",
        "        if self.end:\n",
        "            self.current_cell = None\n",
        "\n",
        "        return self.current_cell, reward"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8PGkj23_FaFR"
      },
      "source": [
        "### Random agent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mzMAsctuFcTN"
      },
      "source": [
        "class RandomAgent:\n",
        "\n",
        "    def __init__(self, actions, grid_size=(12, 4)):\n",
        "        self.actions = actions\n",
        "        self.Q_values = np.zeros(grid_size + (len(self.actions), ))\n",
        "\n",
        "    def get_random_action(self):\n",
        "        return random.choice(self.actions)\n",
        "\n",
        "    def get_action(self, state, epsilon=0.99):\n",
        "        action = self.get_random_action()\n",
        "        \n",
        "        # store path\n",
        "        action_idx = self.actions.index(action)\n",
        "        self.Q_values[tuple(state) + (action, )] += 1\n",
        "\n",
        "        return action"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OXsQf0kSQZpF",
        "cellView": "form"
      },
      "source": [
        "#@title Testing randomg agent's performance\n",
        "\n",
        "#@title Testing agent's performance\n",
        "\n",
        "#@markdown Make sure you execute this cell to enable the widget!\n",
        "\n",
        "n_episodes = 1000\n",
        "\n",
        "@widgets.interact\n",
        "def plot_tde_by_trial(episodes=widgets.IntSlider(value=10, min=0, max=n_episodes, step=5, description=\"Episodes #\")):\n",
        "    # environment instance\n",
        "    environment = CliffWalking(12, 4)\n",
        "    available_actions = list(range(environment.n_actions))\n",
        "    agent = RandomAgent(actions=available_actions)\n",
        "\n",
        "    # training loop and results\n",
        "    rewards, hist = training_loop(agent, environment, episodes, log=False)\n",
        "    best_ep, best_Q, best_path = hist['best']\n",
        "\n",
        "    print('Last best path, ep', best_ep)\n",
        "    fig, axes = plt.subplots(1, 3, figsize=(18, 6))\n",
        "    plot_reward(rewards, ax=axes[0])\n",
        "    plot_quiver_max_action(environment, hist['Q_values'], ax=axes[1])\n",
        "    plot_path(environment, action_hist=best_path, ax=axes[2])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "njS0zdWcpCqd"
      },
      "source": [
        "## TD-learning method\n",
        "\n",
        "The temporal-difference learning algorithm intends to reduce the error in the prediction of the maximum expected return $G_t$ [5]. It aims to improve the policy $\\pi$'s action selection. The error's calculation depends on gamma to control the future policy values impact, formulated as follows:\n",
        "\n",
        "\\begin{align}\n",
        "\\delta_{t} = r_{t+1} + \\gamma V(s_{t+1}) - V(s_{t})\n",
        "\\end{align}\n",
        "\n",
        "The error reductions after a few policy update iterations depend on the learning rate parameter $ \\alpha$. This parameter specifies how much affect the TD-error in the update of the policy values.\n",
        "\n",
        "\\begin{align}\n",
        "V(s_{t}) \\leftarrow V(s_{t}) + \\alpha \\delta_{t}\n",
        "\\end{align}\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c544RMx0ZpZ1"
      },
      "source": [
        "## Action selection strategy\n",
        "How do you know which action to take at a given state?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xiFG4UGn6nIb"
      },
      "source": [
        "![imagen.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZEAAAGvCAYAAACaUeiKAAAABHNCSVQICAgIfAhkiAAAIABJREFUeF7tfQn8F9P6/xFRcalEpRKitFBJomxxW9xbWixFJUlaFC2y3Gspflx/USktQnsSKRUha9xLSkRRKKJNtjakhfk/78edz/18P8v3O5/5nJk5M/M8r9f07TNzlue8z8w8c86zHWARKSFjEDjggAOM4UUYEQQEAUGgKAQOKqqAXPcfAZHr/mMuPQoCgkDuCOCjt1ju1aSGICAICAKCgCDwJwIiROROEAQEAUFAEHCNgAgR19BJRUFAEBAEBAERInIPCAKCgCAgCLhGQISIa+ikoiAgCAgCgoAIEbkHBAFBQBAQBFwjIELENXRSURAQBAQBQUCEiNwDOSFw4oknKtiGd+vWLWu94cOHcxkc7777btZy9erV4zKXXXYZl3n22WcL/M5aMeXCUUcdxfW2bt3qtEpauX79+nEbY8aMSbsmJ/QicMoppzDWn3zyid6GQ9baSy+9pP7+97+ratWqqcMOO0zVrVtXXXfddWrjxo2hGokIkVBNV/DMnn/++czEO++8k5WZ1157LXFt0aJFGcv98ssvatWqVXztvPPOy1hGTvqPwMiRI9VDDz2kfv/9d/87j1GPgwcPVhdddJFauHCh2rBhgzrkkEPUxx9/rB577DFVo0YN9fbbb4cGDREioZkqMxi1hcjnn3+ufvjhhzSm9u3bpxYvXpw4n02ILFu2LPGisttMa8zhiRtuuEHdeuut6tBDD3VYQ4plQ+C2225TN910k9q/f3+2InI+TwQ+/PBDFtS4X5988kn166+/qh9//FGtW7dO/fWvf+XfXbp0Ubt3786zJ3+qixDxB+fI9JL8wl+yZEnauHAOq4yTTz5ZHXjggeq9995TO3bsSCtnb3MdeeSRqnbt2mnXczlxxx13qH/961+8JSAkCJiOwNixYxVCG2EL9YorrlAHHfRn9KkTTjhBzZ49Wx1xxBHq66+/Vh988IHpQ2H+RIiEYprMYbJy5cq8hwvKtKX16quv8rW2bduqRo0a8WrjjTfeSBuALYDOPfdc3h+POuHLHsJVSBBYu3Ytg9CyZcs0MCBAoDMC2du9aYUMOyFCxLAJCQM7helFbH0IluXNmzfn4WTa0rKFSLatrD179qghQ4aok046SZUoUUJVrVpVderUiZf8qVSxYsWsivX58+ez4v64445TFSpUUK1bt1bjxo3jL8FstHnzZjYcQLslS5ZUNWvW5O2ynTt3ZquSdt5W1EOo3nfffapcuXK8Uipfvrxq166dsoUtlKi9evVSMDLA9Tp16qiHH344TScBviFsX3jhhbS+cAJt4vq8efMKXMcqcOjQoer0009XpUuXVjBCgHAHBth6tAkYof5vv/3Gp4B5LsYKf/zxhxoxYgS/GDFWzBvm65VXXinAT+oP3AfQDZQtW1b95S9/UQ0aNGC9ANrLRMCtY8eOrDfAC7dJkyaqf//+vB2Ujb788kvVo0cPbht9AOuuXbuyDiIbYUu2TZs2CquDUqVKqerVq/N4Pv3002xV1Isvvsi8Yezo59RTT2XsU3nDKh0fWfbHWHKDuC/XrFnDp0KzskYoeCFzEKB7xxxmsnAybdo0vIEtergsehElStFL1qKluUUvIIv2c63//Oc/XI4elgIt0UPN53GsWLEicY2W8nyOLFasc845h/9fqVIli7a7LNoa499HH3209e233xZoj4QDX0s+Tysga+DAgYl+SBhY9KJK/G7fvr31888/J9rp27cvXxswYIBFL3qLXqAWvagsEj6JOk2bNrXQrhOy27vgggu4Pr0QLLK+YczwG+OZNGkSjwf/xxjpBZ/o64EHHijQTatWrfja888/n7F7einx9eeeey5xnXRWFr0E+TzGA5xIkCT6uPTSSxNlSXBZnTt3tooVK8bXaZuFf2/fvj1jf8kn0c+FF16YaLdMmTKJcaK9//u//yvQBglKLovzuF8OPvhgxoaET6KNu+66K61fEuQ8Dhu/5PLHHHOM9e9//zutDu6pww8/PNEu8LbbIGW2NX78+LQ6ZF2YKI/5otU384l+UWf58uUF6uCeIEGWaBf3P23TJto49thjLbJES+sn04m5c+dyPWDy3XffZSpi1Dnwav4byyjIvGcGk2I6kTVJ4gF5//33E+wuWLCAz9MqhM9BwNDXIp+jFUSi3IwZM/gcXjbJL2VbiOAaXuT0NZioQ1+AfA7XSP9RAKJMQsQWdPRFaJHpsLV3716uQzoai1Y13M4//vGPRDv2Sx/nzzjjDOubb75JXHvmmWcSL1fS5RToO9uP5PZ69+5t0Rc+F4Wgo5VNAj/6Kra++uorvgYe6QuZr+Hln0xuhEifPn24LVrtWZgzm8i01IJQxTWyAirQD16AOG/zW+Bilh/du3fnOhC6pDTmUrSSsKZPn24VL16cr9FqNFHbFiI4f80111ikSOZrGP+gQYO4PF7eyfcGrSj5PNobNWpU4gMA2NkC7Pjjj7doBZvoZ8uWLRYpr7ke+rFfyj/99JNFxgN8HgIc95ZNqINzEBpTpkxJ8LBt2zbrkksu4TroL5meeOIJPk8rVxbitHXJl8n4hJ8FXKPVRwHeCjTw3x8QTngmUD71IyJTeRPOgVfz31gmIOUjD5iUMBBWF+AVD7RNN954I5+7//77E+dom4XPJX/x0VYPn6PtggJDTRYiyV/UdiG0i3pkuVKgXqoQwYsELxSUffrpp9PgpG0Hvpb8orZf+ni50nZWWh3apuE6eGE4Ibu90047LW31QltJ3BZeVqtXry7QHIQXruELPnmV50aI1K9fn9uiLaU0lvESPfPMMy0IyGTKVYiAf4wDX860NZfWzy233MI84CVuky1EwB+ETTJBcNgvfvvDA2WwUkM7mV6uwMm+H0ePHp1oDsIbdf72t7+l8YUTV111FV+HcLCJtgP5HG2TpdWhbSbGrEWLFolrpOdi4YH5ShXIKITr9mpw6tSpaW3aJ0i/mFglXn/99VnLmXaB71X6R0gQyBkBW5eR7EyYrA+xG8ykF7H1Idn8Q2gLi/ejUwn706DkvfzUMvj90UcfKfpCVbSNoOgFkVYE+poHH3xQ0cOaphuBbgG6kFRy2ndqPTiT0QumwGnod0DYG8eRTFWqVGGrNugE8vXVwL48aObMmQo6pmQaNmwYO4LSllaB87n+oNUn8wmdCuYtlXr27MlY2/dB8nV6yacZVQArGx97nqGjgmMiTGJpdZXaBVs30QcMn0/Wwdj/v/nmm9Pq4ARMmVPr2JhBqU2r7AL1oIcBZnAStAllaPWiSCCqs88+O60f6FNggQV6/fXX067jBHQg0CXR1qGCuToJwozlTD0pmQ1NnRnD+YIQoa/yhIUWbdOwNQkUpHigbGrWrBn/Fw8QXjZ4MZAehM9lEyJQYmai1JdxpjI498UXX/AlWLlkqoOXDm2bZKyOF0UmytROpnKp52DCnEp2W1D0ZyIotHUQ6YTYaW3ixIkKBgb0Bc0vOiijbQugfPuxsYayOhPRijBvrG1rJiiss/kCwdsbZBtewBpu/fr1fC4bb7Vq1WIBBIMJ+DzBIIB0cWx4ANN02tZUtPJQpAvjv7hfSb9SYJifffYZ/6btrqxRHOBTBYKCPxNBcIAHfDjBqCJsJEIkbDNmCL/2SgT27PhStM148cAlv3BhgYKveDxAtoMhBAksa7I93BBE+ZD98si0oiiq3Xz7Lqr95Ou6hEW2PvFSghn2Pffcw0KcdFF8gDAn8JrGSiEfPvzA2g4Dkk3oYjykWOdx0XYg/0UIHAgSWJnhXstEWPHBWm7Tpk1cD0IEQgVYwRmQdCK88rBX2/Aqx8oSKysIRxC8zUG4v7MJCbvvbCbeZBDARUi3ZBcN1V/ZzgrVdJnDbLK/CB4y22QVW0WplLylZW9l4Ysv29d9Pi819I0XAyiTR30qb6m/8+07tT2/fmfzbsYXNMyCYWaKbZjbb79dNWzYkF942E6CGXU+5AfW9jYZVrvZyI6bZn84gC8IBJgsZ3J2RTvYMvz++++5yeQPDmxBwYEVKyCsNCZMmMDmvTD3njNnDq9USEnP9WzeIIxJX1Hokbo9xg0Q2XOHlVYYSYRIGGfNEJ7t1QiZ8qpM+hCbTVuIYI+6KH2IjqHZ22GF2fRjDx/+CWSpo6NL39rAiyoT2Vsm9jXoQKBHIMU3n8KLEdtZWJUsXbpUkYKaz+MFmQ/ZWGcLpghhBZyvvfZa190g6CcIL3WEBMlEts+HzQ8EiK1byeYPAgFBFmHsj2ELEawsMBbbtwPtwceELM1Yz4YVDwQPmeIyG/aL3/btyMQb+MbzgfqZCNu7OOzVTaYyJp8TIWLy7BjOmy1EnnrqKV7WQ5FtP/DJrJOvBCuLIUDsuFp2XS+GiG0y6CLwYkWAu1TC9gHCS9g6nNTrJv7GSxFkbx8l84jtl9QXFL7AoffAvr/91ZxchyyW+GdRRgrJdTL9n8xd+TS2yeyv+uRyiA2FFVCqYj9TW9nO4Wsfuiry61Fk5ZdWDNtWcHQE2fwk/x9GBJnIPp9cB+3A4fPOO+9MqwJnzcaNG/N5Gzc4cZJZLt/X+JhKJZSDoyhW6KlzZJeFPgcHtsvCSCJEwjhrhvBsCwJYp4BsJXoqe9iThpISDzteNLCASVa+p5bP9zfat18CZMbJgsS2dFq5cqUic1PuAl7LYSEIAxD245P33qEAvvrqq9OGAcztOvC+t3UFKAiBbxsWZBPmhW0dJXeGlyos2rBCwIozefWHr/V7772Xi2Me3BK2GMm8m6sjQCQEiS2UoC+BngIKfnzJY4vOJnJY5BUYLMhwHspvEJTYiEBAzp78cYO4azbZQmLy5MkcHNG+b/AXv8nZk3VItlEIFO3Y+gJhxYV0BrbXP3SFsH7DKgXx4bJhDYxwpHq2J5gy/T+m2R3HnR+6X0IFgW2fD77JlDQr7/BARhkc9LBlLGf7iSR7UicXtK9TaIkC9VP9RHARjmu2HwD6hO+BXQ6/6WWRcEBEeduv45FHHsnIm309k4dzpgp2eQqtnnYZzn7gwXbKTC0ARzdcT3b4IwOGhMc9fDLgpGh705Nuif0aUCfZvwYe3HZbuAZvbXh22x7b8KSGc10y2Y6YcOyE0yUJ/VT20n7DkZG+yBPzC09y28kU/cInJZlsPxFaCaa1hRP2dfhlJBPasXmH02HyfMKrHL4WqUSrZI4WAD5QF+MHXvgNn5hHH320QBX4pMCvBNdxAGvgbPuu4By98AvUgV8SfJeS64Afux9gmexsm8qjXS91vKnlTPwN3mUlQigIuUfA/rrC1xm2rbJRsp9ANtPebHXdnKeXDFvX4Ovx4osv5i0HfL3iSxNmlG+99ZZCmbAQtgphwIDYTFAawxcGW1v4EiYnNv4KTiWY8kIfQOFL2B8F20HYXoFiHfG8sCpLtXhCUi7s8+Or3bY8Sm039TeMLGAFRmFM2BwWK07oGTDniCeVbTsptZ2ifqOdl19+ORELDdZOZ511lqJQNYwH/p9KHTp0YH0DVp/Y5oSSHasCrIxgLYgkUMmE+5gEMa92YJSAbVEo7bGlBp8jbIVSpIMCdUjQ8BxgWxcxsaA3QT/o75///Ccr520/o1T+ovD7AEi3KAwkKmPATSxTEpXZ9G4csOjBFhW2cPASExIEgkAA7ysRIkEgX0ifIkQKAUcuCQKCgFEI4H0l21lGTYkwIwgIAoJAuBAQIRKu+RJuBQFBQBAwCgERIkZNhzAjCAgCgkC4EBAhEq75Em4FAUFAEDAKAREiRk2HMCMICAKCQLgQECESrvkSbgUBQUAQMAoBESJGTYcwIwgIAoJAuBAQIRKu+RJuBQFBQBAwCgERIkZNhzAjCAgCgkC4EBAhEq75Em4FAUFAEDAKAREiRk2HMCMICAKCQLgQkBzr4Zov4TZmCCACLbITIicFosEi/wXSrO7atYsPJEXCb+RQwYEIvch9gSROOJBTBAmPhAQBrxCQAIxeIeuyXQnA6BK4iFSDsFi0aJFCtkIcCGOPXOAI5Q6hULNmTUV5MBJCAyHXEeLdFiqI7guhA4GDA3nskZQJYfpxIHEY2hISBHQgIFF8daCouQ0RIpoBDUFzyN2BfBTTpk3j0O7IR2G/9LG6yJeQyc8WSsi2RwmuOJ8GJVJSSPkqJAi4RUCEiFvkPKwnQsRDcA1r+pNPPuF0t/PmzUu81Bs0aOA5l9j+soUWUtsOHjyYVzhCgkCuCIgQyRUxH8qLEPEB5IC7+PDDD9Xdd9/NecGRB/yyyy4LjKNZs2apoUOHshBBXnrRnwQ2FaHsWISIgdMmQsTASdHIEtKxQm+BVKtIpWoKzZkzR+HA9tm4ceNMYUv4MBwBvK/ExNfwSRL2ooEA8r3jgYP1FHQfJgkQINy+fXs1ffp0XokcdNBBzKOQIOAEAbHOcoKSj2VkJeIj2D51BQurE044QY0cOdKnHvPrBibCgwYNYgX83Llz82tMakcaAVmJRHp6ZXBBI7B06VIFE9wePXqERoAAM/iiQOB17dpVHXHEEWr58uVBQyn9G4yArEQMmxxZiRg2IS7ZwfbVxIkT1cKFC9Whhx7qspXgq8E8uFWrVqpnz56qU6dOwTMkHBiFgCjWjZqOP5kRIWLgpOTIEsx2V61apSZPnpxjTXOLd+7cWcH8eMCAAeYy6YIzPG9xI8uytA1ZtrO0QSkNCQJ/IjBmzBi1devWSAkQjAxK9w0bNqgJEyZEbqrxUo3L4cXkyXaWF6jm0aasRPIAL+CqECBwIBw7dmzAnHjXfa9evVS9evUU/kaB4va86R6vrESi8BTIGIxAAFtY69evj7QAAdDjx4/nmFxhsTQz4uaIOBPiJxLxCZbheY8AlOjQgQwbNsz7zgzoYcSIERw5eMaMGQZwIywEjYCEgg96BqT/UCMAM15YYS1evDjU48iVeehImjRpwuFSTjvttFyrS/kIISA6EcMmU/eepWHDixw78AOBIj3MZrxuJ2XHjh2qatWqavv27W6bCLxe3J433eMVnUjgt7AwEGYE4Ik+c+bMWAoQzBscESdNmsQhU4Tii4DoROI79zLyPBCAHgShTFq3bp1HK+GvilDylStXFv1I+KfS9QhkO8s1dN5U1L3c9IZLaVXm6X/3wP79+1XJkiXVvn37QndjxG0edY9XtrNCd8sLwyYggHDuUXS6c4stov6OGjVK9enTx20TUi/ECMh2VognT1j3HwEklEI+EARVFPofAr1792YF+8qVKwWWmCEgQiRmEy7DzQ8BZCREQimhdASgYAc+QvFCQIRIvOZbRpsHAghpgpS2piWUymNIWqteeumlvBKBR7tQfBAQIRKfuZaR5okAQpsgJ7pQdgSAT1w897OjEK8rYp1l2Hzrtp4wbHihZWfbtm2qWrVq6qeffgrtGPxi/PDDD1ebNm3ifO2mU9yeN93jFess0+9w4c8YBKZOnaquuuoqY/gxmRHgBLyE4oGAbGfFY55llHkiMG3aNNWlS5c8W4lHdeAEvITigYAIkXjMs4wyDwTWrFnD3unI7CdUNAKNGjViL/a1a9cWXVhKhB4BESKhn0IZgNcILFq0SFWsWNHrbiLVfvny5RVwE4o+AiJEoj/HMsI8EXj99dfVBRdckGcr8aoOvIBbnCjM0YzzmScRIvmgJ3VjgYAXQmTdunXs9V67dm2OAlylShXVvHlz9dJLL0UC0ygLEcwbrJKQQwZWaG3atFGlSpVSTzzxRCTmLtdBiBDJFTEpHysEVqxYoRDyXae56pIlS1h4PP744+y8WLZsWfXjjz+qV155RV100UUKudrDTmXKlFEtW7bkjI9RJZh9n3/++Wr+/PmqRIkSPI9xJBEicZx1GbNjBFavXq3++OMPx+WdFOzevbvas2eP6t+/P/udbNiwQe3cuVPdd999XH3w4MGhjIibOnZE9wV+USXM05FHHqkQyQDz2K1bt6gOtdBxiRApFB65GHcEYJl18skna4MB2QA//fRTVaxYMXX//fcrZEYEIRLubbfdpi688EJVt25dFixeEiynHnnkES+7YNyAX1Rp8+bN6rnnnlO1atXyfIh+zJfbQYgQcYuc1IsFAogDVaNGDW1jxd45BAZWN5l8KV599VX17rvvskmxTffcc486/vjj1bfffquNj9GjR6s77rhDW3uZGgJuUY6jhRhqFSpUSBt6WOcrbSAOT4gQcQiUFIsnAgceeKCqWbOmtsEXL15c9evXj9uDgrZ69eqqb9++6sknn1QbN27M2A+2StavX69+//33jNdzPQndy6OPPpprtZzLYyUCBXSQhLF6RcmCPrmPsM6XW5xEiLhFTurFAoH333+flaY66aGHHlLTp09XZ5xxBjvkQZHeqVMndeyxx6qzzjpLvfnmmzq7S7TVuXNn3mKCFRh0Ml4TVl3Lly/3uptC20dkYWRdRPBM3XT00UfrbrJAe37Pl9vBiBBxi5zUiwUCu3bt0mqZBdDwdQ6h8d577/EW1axZs3g1UrVqVQXLrWTzWKxcRo4cyVjDC/yUU05xjfuvv/6qTjrpJNWqVSvtY8rEFAIxwmAgSHr44Yd5+xDRhSHUdAoT6LVSKczzlToWx78tIaMQoIkzip+4M0OKb4sEiTYYyJTXIrNX6+uvv05rk6yZrNatW1u4Bzp27MjXp0yZYtHKgc8NHz7cIkVuWj2coIyL1q233mpRnvOM11NP1qlTxypdunTqaa2/SYBYZBqttU03jZUrV47xw0GCxKKViUXh6q29e/fyuVzp2muv5XpkmJBW1fT5cjPetEEmnUB76aKUzgoJAoKAUrC+Of300xMWVDow+fe//63oBc7+IKkE/UuLFi34NAkD/ouIuLb1z+WXX86ObZkIZqaw9tKlN8nUR67n4FuD8PlYeQV5/PDDD2wNB8JqDOmNb7/9dk9ywod5vnKdX7u8CBG3yEm9SCLw+eefqyFDhrB+AgEXoRP5+eeftY21YcOGClseMPOFSW9y20uXLk1st8CJLeyErUB45tOHa6AHrUQSvj7Y0oKOCxZUUXDqNOEeESFiwiwID4EiAK/xoUOHquOOO44Fx7333st+GviSxoGXoS5CIEe8wEBYORxxxBEssOC0hui3sMKCp3fv3r11dRlYO9CHQC8SJE2ePFn99ttvrA+B8MAHAlYicBQ8+OCDg2QtMn0fFJmRyEAEARcInHrqqRx6BN7VOGzCCwYeyHgJQYjojOJ7yy23qHr16qkRI0awH8WWLVsULH1wDt7sHTp0UNjaKoxIB6I++OCDRBH4loAmTZrEKx2b2rdvrxCCJAgyQYggKgC2BiG4ITiCojDMl1tsRIi4RU7qRQIBbCmRMpTDlidbE+FFfMUVV3BIC3y56iboPmz9h5u2Ea8JX9WplLqCOfPMMwMTIsAt6BwsTz/9NJs066THHntM4ciFwjBfuYwnuaxsZ7lFTupFAgEICkTOhUIaX852oEWsPLC9hfMmxn+CyWqyrgF+JyBs3SSfR6DHoAh6n6BJtwBxO54wzJfbsclKxC1yUi9SCMB6BwIDPgVYhWBbCWRS6A4ySQ0V5rpDxoRq8MRs2ObLLb6yEnGLnNSLDAIwP8XXOwRJ06ZN+S/5afD4TBAiUAqDbr75ZjV+/PjQ4B5XIRLW+XJ7Y4kQcYuc1IsEArYAsQdz9dVXc2gQbGWB4KORyTPZz8GDp2bNmqmFCxeqmTNn+tl1Xn1hVedHhNu8mPSgcljnyy0UB8D50G1lqacfgdSXmv4epEUbAadYwwwXJr9Bm6uGaeYQhBAhVpBsy2Ryeg+YPIZceNM9XrQnK5FcZkDKRgaBXB6mKKd69WpCvUgp7BWv0m5+CIgQyQ8/qR1CBHIRIBieCJHcJ1mESO6YhbWGCJGwzpzw7QqBXAUIOoE+AnG0hJwjgOjEppjXOudaSrpBQISIG9SkTigRcCNAMFAo2inqLsfREioaAYSzhxc+gi8KRR8BESLRn2MZISHgVoDY4CE669SpUwVLBwgAJ+AlFA8ExDrLsHnO92Vn2HCMYEcHptu3b+c859u2bTNiTCYzAa9/rEQoF4vJbDJvOu4N4weZxKDu8aI9WYmE6Q4QXnNGQNdDQwmcVNu2bTkLoVB2BGbMmMExx8IgQLKPQq7kgoCsRHJBy4eyul56PrBqfBe6sUQMrUsuuYRzgQhlRqB69erqhRdeYB+RMJDue8T0Meser6xETJ9x4c81ArofFjBSs2ZNPubMmeOaryhXRMTc+vXrh0aARHku/BybbGf5ibb05QsCXggQm/E777xThEiWWaT87wr4CMULAREi8ZrvyI/WSwEC8OrWrcvh4sMUCNGPSX/kkUc4O2OQoef9GKf0kY6A6ETSMQn0jNcvwUAH53HnfmKH4IJ79uwpMgOhx0M2onnggPhiyGUSNvLznjEBG93jFZ2ICbMqPGhBQPfDURRTSEM7aNCgoorF4vpNN92kJk6cGIuxyiDTEZDtrHRM5EzIEPBbgACeLl26sBc79ABxptmzZyuEOLnyyivjDEOsxy7bWYZNfxAvRMMgyImdoPGKc5j4sIR7L+yGCvr+KYw3L67pHq9sZ3kxS9KmbwjofiDcMI5ota1atXJTNfR1MO7XXnst9OOQAeSHgGxn5Yef1A4IARMECIbeoEED1bNnT9W5c+eAkAimW6QPvvHGG1W9evWCYUB6NQYBESLGTIUw4hQBUwSIzW+nTp1YmAwcONDpEEJd7oYbblBNmjRRHTp0CPU4hHk9CIgQ0YOjtOITAqYJEHvYAwYM4JDxvXr18gmJYLrp0aMH+8r069cvGAakV+MQEMW6YVNi6kvSBJjCgA2cED/77DM1YsQIEyDTygNWIBAg3bt319pukI2F4Z7SiY/u8aI9WYnonCFpyzMEdN/8XjGKlUjVqlUjpyOBDgRBFaMkQOx7APdWXA4v7ntZiXiBah5thuVlmccQc64aRkwQEn3s2LFq4cKF7M0dVoIZb+vWrRVWIaIDCesseseBu1eFAAAgAElEQVQ3C1+LyLsupOVcEQjjCzPXMeZSPsx4fPDBB+qCCy5Q8G5v165dLsM2oiwcCWF5BjNescIyYkqMY0KEiHFTEr9Ma4VNQZgFSPK42rdvrypVqsR6EsTcMp0QCwuhTOCJ/swzz5jOrvAXIAJ4RkUnEuAESNfZEYiKAMEIkX+kUaNGqmTJkmrcuHHZB23AFUTjxfZb48aNRYAYMB9hYEG2swybpSi9PN1CG2UM+vTpo5CvHauTSy+91C1E2ushoRTigCGc++jRo7W3Lw1GEwHZzjJwXqP8AnUCdxzGv3LlSnX33Xcr/B0yZIiC5VNQNH36dOYFGQmRUErygQQ1E+HsV7azwjlvkeU6DgIEk3fKKafwVtG8efNYaX344Yervn37qvfee8+XuV2yZInCigjJtRYvXsw50WfNmiUCxBf0o9eJbGcZNqdxeZGmwh7XcQOHXbt2qalTp6pp06apypUrq/Lly7NVF44yZcqkQpXzb5jpIlAkDijLt2zZoq666ioOZ3/YYYfl3J5UEARsBGQ7y8B7IY4v0ziOOdutt3btWrVo0aLES79ly5Zq//79HFKlRo0a/LdUqVK8esGB1QSE0M6dO/nYvXu3+vTTT9lrHgeswV5++eWEUGrevLmqVq1atu7lvCCQEwIiRHKCy5/CcXuhxm28ud5Fq1atUqtXr1Zr1qxhoQC8li9fnhAaCPyI37ZQwW8QBA6OWrVqyTZVrqBLeccIiBBxDJV/BeP0Uo3TWP27g6QnQcA/BPAMi5+If3h70tPGjRvZqxh+CLDvL1euHHsX33XXXer777/3pE8djYoA0YGitCEIBI+AKNaDn4MCHOTycoVVzRVXXMF74tj7hkL2l19+YT8EEPbP3333XVW6dGmjRpnLGI1iXJgRBASBAgjISiTENwRCnl133XUsQHr37q1+/PFHhVXJtm3bFGI2HXfccbyPjvhHJpEIEJNmQ3gRBPJHQLaz8scwkBa++eYb9d133yU8jKFYtQmOY9jiAkExawqJADFlJoQPQUAfAiJE9GHpa0vIWbFv3z71ww8/qAMPPDCtb6xIQL///nvata+//lohpeuxxx7Lx5VXXqkglLwirJpEgHiFrrQrCASLgPkhRYPFJxS9w3ns448/Vl999ZX68ssv2eTzzTffzMg7Qm1ceOGF6ueff+a/oLlz57Ln9BtvvMEmoToJAgRCTjIO6ERV2hIEzEFAhIg5c5EzJxAUgwcPVu+//36iLhTsiH909tlnq7feeiutzTvuuIMV73BAa9q0KV+HALnooos4jhMC8ekiW4D88ccfupqUdgQBQcAwBGQ7y7AJccrOsmXLVIsWLViAIBosYh998cUX7LG8YsUKdfnll6c1BWe1+fPnq1atWiUECAphRQLP6GeffVatW7curZ6bEyJA3KAmdQSB8CEgQiR8c8YcIy/F3r172UILwfwgNE488cRE0iOEykilV199lbeV/v73v6de4nNYMaBMviQCJF8Epb4gEB4ERIiEZ64KcArLLJAd5iJ1GJmEAVYqICjlU8k+9/nnn6deyum3CJCc4JLCgkDoERAhEtIprFOnDnM+ZcoUtWPHjsQoNm/ezPkpnn/+eT4H/xGb4FMCQuKhVLLPIYifWxIB4hY5qScIhBcBESIhnbv+/furo446Sr3zzjscPvyMM85QJ510kqpSpYp66aWXOEc2CFtdCCkO4VC8eHE+B3PbbFSsmLtbQgRINkTlvCAQbQTcvTGijUkoRlehQgUF5TryQkCYINIr4mZBuCAU+P3336969OjBIU8++eQT1ndUrFiRx2b7kCQPFDknQJUqVcp5/CJAcoZMKggCkUFATHxDPJXQY2A7KxtNmDBB4bDJFiIIj5JK9rljjjkm9VKhv0WAFAqPXBQEIo+ArEQiP8X/G+B5553HP7DdlUr2uXPOOSf1UtbfIkCyQiMXBIHYICBCJDZT/WeiovPPP18999xz6qOPPkqMHAEbEREY/iJIZOSERIA4QUnKCALRR0BCwRs2x17HmEJIFDgpQoEO3xL4kyDSL/pFWlYEbyyKRIAUhZBcFwTigQDeGyJEDJtrr4UIhgvF+913382WXaDGjRtzyBPkHymKRIAUhZBcFwTig4AIEQPn2g8h4nbYIkDcIue+HkLYJOdYRzBLhLqBzw+O008/nX//5S9/4aNhw4YcuTk5x3rdunXdMyA1BYFCEBAhUgg4QV0yVYiIAPHnjkAiMWwrvv7663xcfPHFbJ6NVSIEQ82aNVWJEiUSQuOwww7jiMy2UEHsNAgdxEnDgW3LBQsWsK8QjmbNmjlacfozWukl7AiIEDFwBk0UIiJAvL1R4LczdepUNW3aNHX88ccrmFnbL32sLvIlOJraQgkRDZBPBv5FXbp0MS51cr5jlfr+IiBCxF+8HfVmmhARAeJo2lwVghPogw8+qObNm5d4qWeLheaqgyyVsP1lC6127dpxOgGscIQEgVwRECGSK2I+lDdJiIgA8WbCP/zwQzZsQEDMu+66S1122WXedOSgVaQQGDp0KAuRO++8U4n+xAFoUiSBgAgRA28GU4SICBBvbg6E7ofe4pJLLlFt27b1phMXrc6ZM0fhwPYZ0gwICQJOEMD7SpwNnSAVwzKwApKMhPomHuFp8MDBegq6D5MECEbZvn17NX36dF6JIDsmeBQSBJwgIH4iTlDysYwpKxEfhxz5rmBhdcIJJ6iRI0eGYqwwER40aBAr4OfOnRsKnoXJYBCQlUgwuBvbK7aw3IaCN3ZQATK2dOlSBRNcRFMOiwABXFiFgt+uXbuqI444QiHKgZAgkA0BWYlkQyag80GtREQHonfCsX01ceJEtXDhQnXooYfqbdzH1mAe3KpVK9WzZ0/VqVMnH3uWrsKAgCjWDZylIISICBC9NwLMdletWqUmT56st+EAW+vcuTOnYh4wYECAXOjvGs9b3AjPuy6S7SxdSIa4HREgeidvzJgxauvWrZESIEAISvcNGzYUyE+jF7ngWsMzEJfDC5RlO8sLVPNo08+ViAiQPCYqQ1UIEDgQjh07NsPVaJzq1auXqlevnsLfKJCfz5sJeOker6xETJjVgHgQAaIXeGxhrV+/PtICBIiNHz+eY3KFyVBA70xLa6kIiJ9IKiIx+A3/D/ED0TfRUKJDBzJs2DB9jRrc0ogRIzhy8IwZMwzmUljzCwHJse4X0ob0AwFSvHhxcSTUNB8w44UV1uLFizW1GI5moCNp0qQJh0s57bTTwsG0cOkJAqIT8QRW943q3rNM5sQWIHAmE9KDAPxAoEgPsxmvWyR27NihqlatqrZv3+62icDrefm8BT64DAzoHq/oRDKAHNVTIkD0zyw80WfOnBlLAQI04Yg4adIkDpkiFF8ERCcSg7kXAaJ/kqEHQSiT1q1b6288RC0ilHzlypVFPxKiOdPNqmxn6UY0z/Z0LzdFgOQ5IVmq656nLN2E4vT+/ftVyZIl1b59+0LBbzKTcZtH3eOV7azQ3fK5MSwCJDe8nJZGOPcJEyY4LR75coj6O2rUKNWnT5/Ij1UGmI6AbGelYxKJMyJAvJlGJJRCPhAEVRT6HwK9e/dmBfvKlSsFlpghIEIkghMuAsS7SUVGQiSUEkpHAAp24CMULwREiERsvkWAeDehCGmClLamJZTybsS5tXzppZfySgQe7ULxQUCESITmWgSIt5OJ0CbIiS6UHQHgExfP/ewoxOuKWGcZNt9urSdEgHg7kdu2bVPVqlVTP/30k7cdRaD1ww8/XG3atInztZtObp8308eVjT/d4xXrrGxIh+y8CBDvJ2zq1Knqqquu8r6jCPQAnICXUDwQkO2skM+zCBB/JnDatGmqS5cu/nQW8l6AE/ASigcCIkRCPM8iQPyZvDVr1rB3OjL7CRWNQKNGjdiLfe3atUUXlhKhR0CESEinUASIfxO3aNEiVbFiRf86jEBP5cuXV8BNKPoIiBAJ4RyLAPF30l5//XV1wQUX+NtpyHsDXsAtThTmaMb5zJMIkXzQC6CuCBD/QfdCiKxbt4693mvXrs1RgKtUqaKaN2+uXnrpJf8H6EGPURYimDdYJSGHDKzQ2rRpo0qVKqWeeOIJD5A0v0kRIubPUYJD0wTIxo0bVc+ePRX2wBEWvFy5cpx/G74C33//fYiQzc7qihUrFEK+6zRXXbJkCQuPxx9/nJ0Xy5Ytq3788Uf1yiuvqIsuukghV3vYqUyZMqply5ac8TGqBLPv888/X82fP1+VKFGC5zGOJEIkJLNumgB54YUXVK1atTgQ4QcffMAvWSS7+uijjzj0xbnnnhvqZEX2bbF69WrtWSC7d++u9uzZo/r3789+Jxs2bFA7d+5U9913H3c7ePDgUEbETX2UEN0X+EWVME9HHnmkQiQDzGO3bt2iOtRCxyVCpFB4zLhomgCxLEshku2uXbsUAu/hKxqrEnyZQaAcd9xxChZNs2fPNgPAPLjAOE4++eQ8WihYFdkAP/30U1WsWDF1//33K2RGBCES7m233aYuvPBCVbduXRYsXhIspx555BEvu2DcgF9UafPmzeq5557jjymvyY/5cjsGESJukfOpnmkCBMP+5ptv1HfffcdfYaNHj1bwULapfv36vMUFKmorAw5pEDomE+JA1ahRQxuL2DuHwMC8ZvKlePXVV9W7777LJsU23XPPPer4449X3377rTY+MG933HGHtvYyNQTcohxHCzHUKlSokDb0sM5X2kAcnhAh4hCoIIqZKECAA/JqIwHRDz/8oA488MA0aLAiARWWyx06E2zrvP3222n1TTqB8dWsWVMbS8WLF1f9+vXj9qCgrV69uurbt6968skneTWXibBVsn79+kLxzFQv2znoXh599NFsl7Wdx0oECuggCWP1ipIFfXIfYZ0vtzgd5Lai1PMWAVMFSOqot2zZoj7++GP11VdfqS+//FItX75cvfnmm6nFCvzGlg62w7Bnbjq9//77rDTVSQ899BA7LiKR07Jly1i5DmU6XrgwUvjXv/7FClvd1LlzZ4Xx2KsDZCP0krDqwv0QJCGy8N69exVWBzfddJNWVo4++mit7aU25vd8pfbv9LesRJwi5WO5MAgQCIqGDRuqY445hq1woBsZMWIE60fOPvvsjGgtXLiQ/S3gzYy95DAQ9D46LbMwZgiLTp06qffee4+3qGbNmsWrEazwYLmVbB6LlcvIkSMZKuB2yimnuIbt119/VSeddJJq1aqV9jFlYgrbnDAYCJIefvhh3j6ExSCEGiIx6yLotVIpzPOVOhbHv0lJKmQQAjRxFt2cBnGUzsrSpUutgw8+2AKv9KVn0UvQoq9pi7a4uDApbPkavRgLVH7mmWcseoHxQV/iXIZekOkd+HSGFP8WbSNZZO9vkeLSopdsWs+k+LZIkKSdd3uChKxFuiLr66+/TmuCVmZW69atGZeOHTvy9SlTpljkP8Lnhg8fbpHwTauHE5Rx0br11lsTc5CxUNLJOnXqWKVLly6qWF7XSYBYJIDzakNHZTI9Z/xwkCCxaAVmUbh6i1YofC5Xuvbaa7ke7vNUMn2+3Iw3dYzJvxnXwgrINX8RIB2Cq5vaXy4ti0wZmU/aksrYNQQDrqcKkeTCeIGjTJBCBEINLxT6YuaXHX2xWuT4Z5ES2yITZeupp56yzjvvvIxjdHty3rx5PG6y6MnYhC2AKXti4jqZAnMd0plkrIOT06dP5zK//fZb1jLJF/wQIuiP/IaYr6APfJgl83DIIYdYtjBwBFhSocKECIqZPF/AQCehvfT1GJ0V8h8BewvL/55z7xGWWaBsAQlhYRQGwn75+PHjebsD21bQ0fzyyy+s38FWE3Q92NP/+eeftQ0HW4DY8oCZL0x6k9umFV5iu8ULnYi2QThsCJjCM59eWoEecILF8wXClhZ0XNCRRMGp0+FUeFpMhIin8DprPAw6kOSR0Fcs/6Slu4KS3CbYzdM2jHr++ef5FPQjphNyX+CAh7VNUDjDkezee+9l3QFehroIgRzxAgPBTwSe/sceeyybS0OpDissW8ekq8+g2oE+JNn8Owg+Jk+erGh1lhAeQ4YMUbt37+b5pS3ZIFiKXJ9inRXwlMIMFjdzYeawAbOY1j08rSdOnKjeeecdVvbCBBZmvbDOwksXVjBQYNJ2ESuOoUQP+mWSOggYBuAFA0EIJbctKDAXsIqxX/S2ENEZxfeWW27h8DAwRIClFCzcYOmDczB77tChQ0bT6eQxkA6kgI8NfEtAkyZN4pWOTe3bty8gIFNx8PK3CUIE9yrM0TGfEBxBURjmyzU2OvfHpK3cEIAilfwQClSiicytkYBK0xezRV/wrD+A8vnMM8+0Bg4caFFAOgvjIh8ICwpNejlaJGDSuAxCJ0LbVBZZ6VhkBWXRdpFFQoT5Iq9q1otAJ9KuXbsCvJKAsSh+Vhr/fp7ItMdOX9SO9AxQ4mciP3QitBVokUDO1L1v515++eVC+/LieTN5vnSPF+3JdpZr8ZtfRaw8SLkXCl+JTCOFOSq+4rH6wFc8voTh/wCTXzjoIaYWHAq3bt2qyAooUxO+ncOKAzqGpk2bJqKvvvHGG6pr167MA7bhMB+NGzdWc+bMKcAXzpsY/wkmq/R2TBykWGe+sXWTfB6BHoMi6H2CJkRGNoHCMF9ucRIh4ha5POqFXYDkMXTfqmK76uqrr04IjaFDh7LCHA8zBGAyYcsNCm+E9k4lk0J3wGkuTKQ7ZEyYxg5ewzZfbvEVIeIWOZf1RIC4BM5BNSiloThFAEgIDaw88FUOPQGZ6mZt4YorrlBYmWQiE4QILIpAN998M1uThYXiKkTCOl9u7ysRIm6Rc1FPBIgL0BxUKWq7ykETWYsgQmsmz+SsFTy4gBVVs2bNFDz+Z86c6UEP3jQJ02k/Itx6w737VsM6X25HfAC0Tm4rSz3nCDgVIAiJEYcpefbZZxX8NBDS48Ybb3QO5H9LJltX4aHFUdhqI+cOkirADBeh2U2zMMtnTF7XRRBChFgx3cw7Ls+bPd+6x4v2xMTX66eJ2ncqQHxgxZguyCM7Z2GJ7SqsOnAgNDoEB/7vNdmxrBD6W8gZAl6kFHbWs5TyGwHZzvIYcREg+QPs5XaVE+6inC/cyfjdlBEh4ga1cNYRIeLhvIkAcQ9uLtZV7ntxVhP6CJgBCzlHABZvppjXOudaSrpBQLaz3KDmoI4IEAcgZSkC6yo/t6uysJE4jeRKFHWXc3GcfvrpRRWP/XWEs4cXfrVq1WKPRRwAECHiwSyLAMkPVPhrpPpy5Ndi/rURXwvpfEWIFI0lcAJeQvFAQKyzNM9zvgJEt/WE5uHFtrnt27fz6shO/RtbIBwMHPHGsBKhcDgOSgdbJG7Pm+7xoj3RiWi8h/MVIBpZkaY0I4DQLbDOQhZCoewIzJgxQ8F5MwwCJPso5EouCMhKJBe0CimrS4Do/lIohGW5lCMCiKEF02QTYkLlyLpvxatXr65eeOEF9hEJA8XtedM9XlmJaLrLdQkQTewY0Ywd4BDe3tdcc4166623jOArHyYQ8h5HapDGfNqMUt2nn35a1a9fPzQCJErYBzkW2c7KE30RIP8DMDnAIfI3XHjhhZxRDrlHKN1snkibUf3OO+8UIZJlKpA3BvgIxQuBwK2zPvroI94eoJwOnKAHYcRhSonw4jhgDYPfUNbhwG+8mBAYDwdCXZ966qmBzJoIEE5+kvAiR2gQeJG//fbbnK0vilS3bl2+DxEIsVevXlEcoqsxUW54zs4YZOh5V4xLpbwR8F0nAkFBiWIUPFpxtGnThsOCwBYfQgHbBciBbAsNKOiQh9oWKkhtib1ptIMD2yULFixQ8CrGAQcntOM1eSVAdO9ZeoUDtqvgST5t2jQWHDiistpwghmCC+7Zs6fIDIRO2gp7GeCA+GLIZRI2CsvzpgtX3eNFe74IEZhFwnYcLxyYSSJxkf3Sh7DIl5CG0xZK8CyGYxjs1Lt06eJJQiSvBAhw0D3J+WKbXB/bVRAcOOBIBsERV38A3MuUuY8DSMad+vXrp8466yx15ZVXhg4Kk583L8DUPV60h+0IzwipOelFY5UpU8aiSK0WbUt51ldyw8uWLbPoxrbILJP7p+0ybf1mSmmrrXFqiG4cnc3l3RZtHVqk07BolcGpcClPh0VCOu92o9AAmfxac+fOjcJQXI/hmWeesSgas+v6QVc07XnzGg/d40V7nryxKCm9hQeM9kctstjwGpdC23/qqacs2iKz2rdvb5H+pdCyRV30WoCgf92TXNSYsl2nlR3nUKcvDatbt24WeZFnKxrr88jNvmPHjlhiQGHerbJly4Z67KY8b36BqHu8aE/7dtZ1112noLeAPb1JobNhlklfjewENW7cOBp7buTlFlYyJ7qXm7mMUrarckHrz7LY0howYEAkTJhzHT1y0o8dO1bVq1cv16rGlA/yeQsCBN3jRXvaTHynTJnC+/nIVY39YpMECCaLViLMF6xroBTF/52SXwLEKT86y2HxY6ePhUkurONgXQUdU1z1Hbng26BBA9WzZ0/VuXPnXKqFvmzHjh05mViYBUjoJ8GQAWhZiVx88cWsaB0xYoQhwyqcDQiFQYMGsQIeq5PCyG8BovtLIdvY4m5dlQ0Xt+dx78PEefjw4W6bCE29G264gR0KoVAPO/n1vJmCk+7xor28hAgpsFXTpk0573Pr1q1NwckxH3CO6tq1q8IL9bTTTkur57cAAQO6Jzl5ULJdlTbFWk9MmDBBffDBB+xDElXq0aOHOvPMM1X37t0jMUQvnzcTAdI93ryECLav4Im8cOFCdeihh5qIlyOeYB7cqlUr3pLo1KlTok4QAgSd655kbFfZZrm2MyBMc6PqDOho0j0sBAEC/6WwrMpzgQIrEGwHR0WAePG85YJnEGV1v19cC5EHH3xQkfkuv5yiQtjTxv42lKRBCRCdN7VsVwV3Z8J3BFEWpk+fHhwTmnuGDqRJkyaR2MJKhgYvwbgRPix1kSshMmbMGLV+/Xo1bNgwXXwY087AgQPZc75Pnz6KzHkD4SufLwXZrgpkyjJ2ipDosFzCSh3e3GGln376ibeqsQrp0KFDWIchfHuEQM5CBAIEca7wN6qEPV+sSIKKi5SrEJHtKnPvROhHEJkB1m/t2rUzl9EsnM2ePZu3eV977TWxwsqCUdxP5yREsIW1devWSK5AUm8EbGkhPWv//v1TL3n+26kQke0qz6dCWwcwL69UqRLrSWBebjohFtZNN92kvv32W0Ue6aazK/wFiADeV478RKBEhw4kiltYmfDHw449bWxJmERxCLVuEt66eIGja6NGjVTJkiVdObrq4sNJO4jGi+03OBKKAHGCmJQp0sQXZrz4KqGwF7FDC4pEPFRItOMXpa5EZLvKL+T96Qf6NuRrx+qEYk7506mDXpBQCibvCOc+evRoBzWkiCDwpzVpkbGzyHzXolDsfoV2Maofetgt+irzlSe6Mbk/iV3lK+y+dvbxxx9z0EJKWWCRj5Wvfad2RpEbLHIctC6//HILAVOFBIFcEMD7qlAhQlYZ1vz583NpM3JlaSvCIqWoL+P68ssvOQAj+XBY5MRp0TaiL/1KJ8EgQInYrGuvvdaidAjW9ddfby1ZssQXRt59912rd+/eFsWR4/4///xzX/qVTqKHAN5XWbezoAdZsWJFJJ2mcl2GIrwDvHSTnRFzbSNbebqtCjgDQu+BcCziDJgNseidR8I1O99O5cqVVfny5RP5diiNQt4DhpmunW8HyvItW7Yk8u0gIKmQIOAWgUKts1L35t12EoV68BmBUnTfvn3ahpPNukpw1wZxKBtau3atWrRoUeKl37JlS/ZZsjN/4m+pUqUUhaDnA0ndIIQQeQEHImjDDN/O/AlrMGQSTc78iTh3QoKADgSyChGEc0c0XvhMCP2JAMLHr1y5kh3I3JITZ0ARIm7RjWY9WEUiHTRtfbFgwP2B8PO20IBPE37bQgW/QUgRjaNWrVqS9zyat4YRo8L9mGa0ji0sfM2IACk4R7SHzOk/8VDXqVPH8QSmblchbhVCrct2lWMIY10Q91ou91uswZLBB4JAmp8IpT/lhFJC6QjALBP4OCFsVyFCsJ2f45577lGkOFd33nmnCBAnAEoZQUAQCAUCBRTrn3zyCcfHwde2UGYEsCc9b9483ipIJSfbVal1Un/LdlYqIvJbEBAETEUA76sCKxGENhkyZIip/BrBF/ABTjZhu0oyAxoxNcKEICAIBIBAYiWybds2zk4Ic0ChwhGAEvOpp55Ss2bN4jS70HPgOPfccwuv6OCqrEQcgCRFBAFBwAgECijWYacuObWdzUuXLl0Uwsb/4x//UPCnERIEBAFBIK4IJLaz8EWNl6NQ0QhA2JYuXVqEbtFQSQlBQBCIOAIsRGB/fsIJJ3AeDaGiEUBEVngWr1u3rujCUkIQEAQEgQgjwEIEHq0VK1aM8DD1Dw2hKYCbkCAgCAgCcUaAhQji6iAsgpBzBIAXcBMSBAQBQSDOCLB1FqyNNm3axHF4gqDff/+dE0A9/vjj6osvvlAUel4df/zxCnGDbr75ZlWuXLkg2Cq0T6+s2cQ6q1DY5aIgIAgYhABbZ3300Ufq4osvDkyAwM8CyXmQEAeEqKWUw4QdHhGrClnhkKsags4kAp8QcnDQrF27tkmsCS+CgCAgCPiGQDFE/Pzjjz986zC1o9mzZ7MAOeaYY9R7773HfioIV005DlT16tVZeT1q1KjUakb8RnRV4CckCAgCgkBcESiG6KAI5REUUYIc7rpbt27qjDPOSLBx4oknsvc88nj8+uuvQbFXaL/ADfgJCQKCgCAQVwSKwbw3UxwovwCx9TAvvvii2rp1a4Fur7jiCgUhc9999yXOI3cC9CW33nqrXyxm7Qe4AT8hQUAQEATiikAxIlWzZs3Axn/NNdew4x70Hscdd5xq1aqVeuCBBzhcOtRMzVAAACAASURBVLaLUglbb+vXr1c//vhj6iXff2MlAsWSkCAgCAgCcUWgGBLalChRIrDxV61alRXolOuZFeovvPCCuuWWWzgOFXwxBg0axNZaJhIyzAE/IUFAEBAE4opAMWwPBWXaa4MO7+/HHntMfffdd2rJkiVq2LBh6qKLLuK0n8OHD1etW7dWsOK69957edUCgjkwVgHPPvtsYHMHizFkmBMSBAQBQSCuCBxw2GGHWVu2bFH0NxAMoJiGnwgiCKeuiJYuXaoQYgSEcnv27FH/+c9/VJ8+fXilgq2w888/X2E1EwRByFWqVEmrIBE/kSBmUvoUBAQBNwiwn8jpp58emAAB00g5++GHH6qnn35aXXbZZQXGAWuto446Sn3//fdq37596tRTT2WBASEC819kDgySsIKTeGNBzkD0+0a66uQc68iU+f777/MqHQeeX/zGvYijYcOG/FGWnGO9bt260QdKRhgYAgfhBoTOIaiVSOPGjVmI3H777apKlSps0gvCA4LtKwiQo48+WtWqVSswkLJ1DB5FJ5INHTnvBgGsuBctWsQhdXDAERjGJDDiaNeuHRvBYMVuCw08t3h+baGye/duFjqwGlywYAHrGPEXYXpwNGvWLFCTfjeYSB2zETgINyNuwKCECHKWI90snAvPOussfjjKli2rNm/ezKuPgw8+mJM/wYrMNII+xDRP+iAw2rhxo0IOeXw14yVYvHhxjnLcpk0b1bdvX15NCmVHACF0kM8H6Rhgvg7HW/hN4bcTfSWeXRx2ENXUlQfuU1soLV68WH399decxgCpH2wdY3bu5IogUAQCtC1k0VcLQmgFRj/88INFFlkW3fwWhROx6MGxaOvKom0ri/KWF+Br+/btFg3JImuuwPi1OyZvdYu+DLXygbGFiZ5//nmeL/B90EEHWaQjsujFxL9x0Be0RS/JMA3JN14ptI9FGTH5nr/xxhst2hXwpe9ly5ZZ/fr143kiYWXhPhYSBNwgwM95p06dLPqCdFM/kDomCRHayrLoa04rDmESIrTNYtFXMwuL3r17Wzt27EhgQX4/Fvn98DWyvNOKUdgbAzZt27a1KOaaRbrAQIdDaZ75Q6h9+/aheg8ECpp0nkAAz3cxKOGwhyqUOwKIm0Vo5l4xIjW++eYbNss+8sgj1ejRowts7dWvX1/17NmTR4pgmkJ/InDdddex2TqMQoBLqjGJ3zh16NCB47/RxySb1tPHgN8sSH8hR6BYWEN37N27N3Dogw4ZEzQAsJSD3oq2IxWshlIJe/0gfKikEvbmEdYGMdKOOOIIjps2YcKEQIOBpvKo8/eUKVPYrwnWU9B10EpEZ/N5t0UrETV9+nQFfQptSzKPQoKAEwRCJ0QOOeQQVrLDggVKeYRiD4riLkSScYevETI9jh8/nnPAXHjhherBBx/MODW0haKaN2+uSJ/C1kaXXHIJR27GygVfxFEjWFjBAhGr1h49ehg9vF69erE/FqwOYQ0mJAgUiQDlE7GgFwkTkemvRVYsFhwlyYQxMNbpS9qCclQn0YTpbM7ztt544w2LfBUSinTwDwU7jCTIIZTPk4VWgg8K9W9RuBiLLLYs+gBInP/ll18sMkHl8nPnzvWcbz86oNQGFoXysebPn+9Hd9r7wDyQ9aFvCn/tA5AGPUcAzyu/sXCjJCtFPe85Ah1QAEi2bqGtGq2jCZMQoYgCFplg84ufEotZZIptUWZKi7a4GJNHHnkkTYhAkYvyFJk5Dbd///vffA1WeWGnyZMnsxAlH45QDwXvhXPOOceira5Qj0OY9wYB/mikf9gJCXbkpu3TgjdTCXgh6jB8Ipo0acLhV+zDRJ8WL3AcN26cgm4KyuJHH300rYtMUZjhswDcknPH2BXhEwRCYrIwE7bxoDSH3ifsBD+ot956S3Xu3JmNKAYMGBD2IRXgP45RuEmcaJ1D9uCzhYjWliPeGIQIXp5QGsOzHsLE/ou4XnfeeScL5iCzRno9BXipgLKFfnn11VfTWIBOgHx/WGeSStCVgCCUw0pjxozhvDi0EgnrEDLyDaX7hg0b2PghaoSXalwOL+aOhQiUnFCMCjlHAIrgFi1acAVa7qs77rhDvfbaa7ESKnXq1OHxw/KItj0S4CHaQMeOHVlxDioq9wse4LvuuotNX6Fo7969e6KtMP0HAgSGHjCVjSJhfpD3B8YTQoJAAgF7pwzKUXiyChWNAGVbtCjGV9EF/1uCtgOsu+++mxXHtNXFe8wkdCwSOmk6FZoYx+0GXZA+PFhBDp5h5EDmqxaZ7PIYyWzXuummm/gaFO1NmzbNqHejqMwWbW1xOYrYbG3atCnoYbnqnwQHjzcO1L9/f2vEiBGRGGqYnjcdgOseL9pLvLFGjRrFoRCEikYA3tljx44tumCWEoUJFd2TnIUFbacpy6RFcZgS1nIQrgMHDmRhQDoRi7avrHLlylkURLNA+BOEQkHoGowXggj3H8qHkaBEJ+fBMLLummdYdEZB2R625831hP23ou7xor0D0Db9R1E4EQ7+ZjuI4ZxQZgQQFE9nDhakAn7zzTf5gB4F22NRVtQjACD0Rtj2IoHDEZydBBrMPBvBnkXOm8GDB0dCiZ4rktBdIVLBaaedlmtVY8rHLX+P7vGivYQQwawiyRP2+REKQSgzAjNmzOAXhlcKRkwKrGFsoYK/UbL+goc7dCnQKUFnAoEZZkL0XCjSkdo5bgQ9GKIW4AM0rKT7pWo6DrrHy9ZtycsjL6LS5rv8Mq3+SSedZFHYes/Yopswre3Ctr90+6mkda75xJNPPslbWPQFq7ll/5ujtM2hdSTUhdacOXMs8mzX1Zzv7WR63nxnwscOdY8X7RVYiUCKIgQFQk8glo5QQQSQfRE53ZHfxCty8qWQvP0VtpUKhT5nay7kjkHMrEyE1TApbzNdMuYcxoBQJiNHjjSGp6AYueGGGziNdRhD1jh53oLC1Yt+dY83bTsLTH/88cfqgQce4GBsQgURQCrff/7zn4pCeHsGjZtJDpNQgS4E/BZGpHBXFD6+sCKBX3MzT4Ez7REDcCotWbIkB+MMG8VtHnWPN6MQwU2AcNCI5olgbEJ/IkAhPDjlKBSJXpKOSc4kVMjENqGsRx9C7hGAkymi8ZoeTNH9CHOviegFK1euVGS1mHvlAGvoeN4CZD/nrnWPN6sQAWcIB41onplCfOfMecgrAAdsvfz222+ej0T3JINhCBUKlJhQ1qdaf4lQcT6t2MKC052ESk/HDCv12267TZ1yyinpFw0948XzZuhQmS3d4y1UiGA7i9J1yp4vAU/+M6px48ac/8Jr0j3JmfhNtf4SoZIJpcznEB4dCaUkzlw6PrNnz2Z94TPPPJN+0dAzfjxvJg1d93gLFSIYuDwwSvn9YOieZCc3sAgVJygpDmkC83fJ1Jgdr5NPPlnNmzdPIdldGCiI5y1IXHSPt0ghgsFiGweB1xDNM26EaLJk0ltk7CeduOieZDe8iVDJjFq3bt3U3/72t8BT2mbmzoyzM2fO5Bhyjz/+uBkMFcGFCc9bESxqvax7vI6ECAKuwdwSL5a4EZz8oCiEkYFfpHuSdfCdSahEIcx5LtggkkO1atVCH6Y+lzG7LYsPTgp7E4ooBCY+b25xd1JP93gdCREwBi/tF198MVZmv5Rkib84/fbe1z3JTm6sXMtAqMBUN0708MMPcwh78QspetYpkyVHY77++uuLLhxwiTA8bzoh0j1ex0IEg6ConbytBcuUqBOcpypUqMCJlijAIFs0+EW6J9kvvqPeD0W55vshW+6UqI8/l/FRWmB14403qiVLluRSLZCycXvedI8X7XE+ESeEjGZQmkXddwQ+ANi++sc//sGmsTBxxheokHMEsFKh0Pec7AxZHs877zzOFwIzY4rw4LwhQ0quWbNGnXDCCSJAHM4HvNcrV66s1q5d67CGFAszAo6FCAaJF2y9evUilyLTnkA7fIOdFOnYY4/lzITIwocXoJAzBFIzO95zzz0sjO2/YRMqixYtUhUrVnQ2eCnFCJQvX14BN6HoI5CTEAEcWIkgcidyLkeJkIkPlliZsup9+eWX/AKcNGlSlIbs21jCLlQQnh+rKiHnCMQx5XaYoxk7n9kMJd0GkERCGnLAswg4t00YUY9St1oUDNCi/N5F8kMmnhblTi+yXD4FaIryqR7KumTpZQ0dOpSzH9Ieq0UrFYu2vyxa/RkxHsp1Yu3cuVMrL7TVw0m5atWqZZUqVcqi7R+rWbNmFhmwaO0nqMbIPN4qU6ZMUN077tfN82YnU6Pgp9bGjRutiy++2KLYYdaDDz7ouN+gCroZb2G8or283lhk/stpUBEOOoxEnrVW2bJlrRUrVjhmHy87ZPLzinRPsld8etkuHs4hQ4ZYlJiLw8YHKVQozImFLH46CemVDznkEB5b8eLFWYDgJcQPJB0Up01nd4G1RREeLIqnFVj/Tjp287zZQmTu3LmcDhptQGBOnDjRSZeBlnEz3sIY5nu2sAJOryGfAFLrUhRPp1UCLUcxsCwyQ7TIjNcVHxQG3KItGld1i6qke5KL6i8M14MUKsh/gpehTsLqA/OMXOW7du3ipvHs3HfffXweAmXv3r06uwykLTKRtyh9QiB9O+3UzfNmCxEIEDIisCiSgdPuAi/nZryFMY32ctaJUKU0opUI5xNAOGhE8zSZEI0XXviIheU2xg+tRFhHAt0QAWzycCPBW6oinlaDbHZt/0UqYVq5cIBJN/TRRx9lrQbLLFgl6iJkA6Tkb2y1dv/99ytkRgQh4CmCF8KIA9aBMKf3kmA5hWfBSwJuwC+qhPTOzz33nKKPAs+H6Md8uR5EYVLGzTUKI29RNE8LW0UmEQWG4y9KrEB0EeUK5z18UrzrapK/RIVyQyDflcrRRx9t3XzzzRk7JWdTR/qyjJUznMQKgwQGzzPlTMlQIv0UmUtbxx13nLVly5b0iy7PkCWiVbp0aZe1nVWD3lT3VqCznp2XcvO82SsRvOcykcnz5Wa8mcZon0N7WlYiyRIMYULwRYVonvgSIYW1awGnoyKiEVevXp0zEiKhlM58IGICrGOG8m8j35UK5nH8+PEKUQpSCabJ8L7WRaQD4ajQIOQjwb0JD2/aNlOkpM3YDWK4wemVUiFnvJ7ryVdeeYUdJ70mPP9+OupmGg/G6hXBdygThXW+Mo3F0bnCpEy+12gpyxYosG6hEAgWebDm26Sj+lBcYkVEWwUWPaie5kS3GYJlkQ7FGk2aozFKIecIFLVSIcdSXlHifqFcGAUappe8RcnInHfmoCT5Hln4Sj/jjDO4X8w5Dvz/zDPPTLNKg+4E12EJlA9hVUDRdRP9eb0SoW07iwRwPiznXZfieFklSpSwhg0blrEtN8+bvRIZNWpUxjZNni834804yP+e5Hu3sAK6rsE8EhYnUEJRDnerT58+FoVYt2AGqINgpovtMwgOKPnxINKKKKG01NGHkzZ0mADrnmQnfMetTKpQOfXUU/lDB9hjqwkvHtt0nZwMLdr79gyirVu3WthqxTYrtqxsYUKRcLlPe+uLH1Y66tSp45oXPButWrXiA+P1WohA6FWqVMk1vzoqkm8XfxzAjBoGC6nCxM3zZguRTFZ0ps+Xm/EWNg++CZFkJmAfP2bMGBYmMIuDngJWHHfccQd/nVEiLAtfMLgBbdt8/MVvnF++fLlFWeWs22+/neuhPsx0YWkFwbFu3brCxuz5tXxNgHVPsucDjkAH+OiwTW7tl/XBBx9swYQdLyDbgkrHUPHBQ/lILOjTUolylVutW7dmYUHOr3wZloDNmzfncxS3ziJFbmo1/g1T5FtvvdWxhSSEkddCBM8thFXQVK5cucTqK1mYQD/l5nkrTIiYPl9uxlvY/AUiRFIZwgMFM0Aoo7DcJk94XgLjCwY3IHwF8Be/cb5Lly5chqyjuB7qm0b5mADrnmTTsDGNn4EDB/LHjC08IEwOPfRQPnAN959OooRN3BfMfDMRvm5xHR9ZNjnZHsEHGOrBfN0J+SFEwAeFSUpga2McxF+yhivAB+bZFgZO8EouU5gQQTmT50v3+wXtHUT/BEq1a9dWOKJEMAGmrQk2AfY7CnCUcPR6LAjvDoX6nj17FJTrZ599tmrZsqWiCAaKfAC4e/qAUT///HPCFDdfnho2bKigXIeZLwxQYOxhm/kuXbpUkdczdwGz5bATreAU7QwEbgZ/1FFHqR9++IHhpJUIx8NDgFBEGg5L8iyT74XAhYjJ4OTDG+JF2VGA8SAdf/zx+TQndT1AgL72FSnSWWjAxykTQYjgZWi/6DOVyeUcAjnCx4i2nthP5IEHHlC0yla//PJLIuEVBBnp93Jp1siytJ0VeEbUyZMnK1qdJYQH/IkGDx5sJF5hZUqEiIczZ5sAwxTwiSeeUGTB5WFv0nSuCCBcOY7CyBYiOqP43nLLLRwNGzl6yPJLkf+HIl8VPocAoEiEBtPiwoh0IApZR20ii0T+L4KEYqVjU/v27RVt1xXWlGfXTBAiyMpK0QBYcAcpPMIwX25vBBEibpHLoR6iACOqKba2kKdbKDwIYPtp9+7d2hlu0aKFwuGW5s+fz176qZS6giGjgcCECHALOokX6U0VGSakwpTXb3ISVThyoTDMVy7jSS6r3dnQLSNRr4dw4tjeQnImofAgAAe/1atXG8cw7iPS4SYOONWCsHWTfD5IfSP0PkGTbgHidjxhmC+3YxMh4hY5F/XIGZG3Kbp27eqitlQJAgFyzuMtJxOITFJNYMMxD8AN+MWVwjZfbudJhIhb5FzWo3wkHGQPoTqEzEfABCECiyIQxfdia7KwUFyFSFjny+19JULELXJ51JMowHmA53NVRGhFxN0g6eqrr1aUsEotXLhQzZw5M0hWcuobkYn9iHCbE1M+FA7rfLmF5gA4x7itLPXyQ+Cbb75hf5JkE2AErJMpyQ9X3bWROgCh2Skciu6mI9seghAi3TR56Bs9xrg9b7rHi/aC/cQy+vbynjmJAuw9xjp6iGO+8Hxxk7z0+SIYnvpi4mvAXCWbABvAjrCQgoAtRNq2bSvYOERAhIhDoCJQTFYihkyibQJsCDvCRhIC0Ecgi52QcwS+/fZb7f4ZznuXkn4iIELET7SL6AsmwCAxAS4CKJ8vI7kSRd1VFGHa557D2R3lDWIv/GrVqoVzAMJ1TgiIEMkJLn8KiwmwPzjn0gss6qZOnZpLldiWBU7ASygeCIh1lmHzbFtPvPXWW4rC3ksUYEPmh5JUcRDNbdu2GcKRuWwg3hhWIrqCVno5Ut3WSl7yqqNt3eMV6ywds+JRG8lRgL/66iuPepFmnSJACZwUFOuUhdBplViWmzFjhqJEcaEQILGcIA8GLSsRD0DNp8lMXwoSBTgfRPXVRQwthI83ISaUvlHpbYly0qsXXniBfUTCQJmetzDw7ZZH3eOVlYjbmfC5HkyAEcoaYb6FgkOAMmsqHHPmzAmOCYN7RsTc+vXrh0aAGAxlqFgTxXpIpkuiAJsxUYh9JkIk81xQ/ncFfITihYAIkRDNt0QBDn6y6tatyylzwxQI0Q/UKDe8OvLIIyOX6toP7MLeh+hEDJtBJ3uWMKFEpsTFixdr5X7jxo28bbZixQq1Zs0azpCHzH9t2rRRffv2VchVLfQnAgguiNzsRWUgjANewAHxxZDLJGzk5HkL25gK41f3eNGeCJHCEA/gmtNJ1m0CDGUorGqQTxwvyPLly3Peb5i2guBwhxSssFISUmratGlq+fLlauTIkbGHo1+/fpyn/sorrwwdFk6ft9ANLAvDuscrivUsQIfhtE4TYEQNvu6661iAIL0qIq9iVQKfCOTxRqRhrExmz54dBmh84RE+PPBihx4gzoR7AiFOwihA4jxvOscuKxGdaGpoy82XQr4mwHgZnnjiibwlsXXr1rQtmvvvv1/ddttt6sYbb5Qv75Q5jnOY+LCEey/ssXTzvBXWnunXdI9XViKmz7hD/vI1Aa5atarat2+f+uGHH9IECFiwvbSRbzyZsIJ59NFHeSujTJkyqmLFipw86bXXXnPIefiLwWquVatW4R+IixFg3HGaaxcQxaKKhIKPyDTjZXbNNddwmJShQ4e6HhXCVXz88ccKXvIQTtj3f/PNNzO216tXLzVhwgSFvCitW7dWf/zxh1qwYIH661//ynGmsOUTdWrQoIHq2bOn6ty5s5o+fXrUh5sYX8eOHXllWq9evdiMWQaaGQHZzsqMS2Bn811u3n333ZwpccqUKTmNAYJi8ODBBSLVQsFeu3Zt3uaCIh8WWqNHj+Z2IWjwAjnnnHPUiy++qOy80sjWCDNY1P3+++9z4iHMhUeMGMHZD4cPHx7mYTji/YYbbmCHQijUw075Pm9hG7/u8cp2VtjuAAf8wtkr1yjAy5YtUy1atGABcumll3J8qC+++ELt3r2bzX0vv/zytJ7feecdTuMLwWMLEBTCqgRJnLA1BoESFxowYABbsGF1FmXq0aMHfyREQYBEeZ78HJs4G/qJtk99IQw3/D2g63CSr33cuHFq7969bKH1zDPPsNCAoh2rCdD+/fvTOMcqBA5mZ5xxRto1KOcPPfRQNhOOEwE/4AKBEkXCCuTMM89U3bt3j+LwZEwuERAh4hI406vlYgL83Xff8XCwv5+JXn311bTTeJlcf/316uijj2ZBBbNgBCa85ZZbFFYpeKEecsghafWifgIrEQhv6EiiRNCBYAsrigIEWzJxOTy5J+kFIGQQAjTJ2rmhPBgWKd6ztksvfgv9Nm7c2CLnwkS5TZs2WR06dOBrOMgZMWMb5CeQKINynTp1ssiSK2PZuJwkJXsanmEcO30c8DieeuqpMLIvPHuMAL8bPO5Dms8RAS+ECFho2rSpRbG3MnJDFlkWhTRhQUCJhKyGDRtatJ1lFStWzCKlunXTTTfxNdre4nZ27NhRoB0Kd2HNnTvXeuyxxyz6YrXoq846//zzLdKpZOwvLifJso3xo4CNoRwybW1aZcuWtT788MNQ8i9Me48A3gtinUUomES6rSeSxwYT4CpVqmQ0AYbDIZTyb7/9NltV1alTR9EXqBo0aBDrNuDJToJCkWBRn332WaHhT0joqIceeojNf6GIjTu1b99eVapUScGCy9YzmYwJYmFhDuGJDh2ZkCCQDQG8r2Ql4r2wzqkHmqycyudamHxILFK851otrTy2Nyi/Sdp5nKDAkLxyIUVsxutxPEmxtnglN3bsWKOHTybcFumyrCeffNJoPoU5MxDAcy6K9WwiNqLn3ZgAZ4ICUYS7deumEPoilWz/EKx6hP5EAIp2RAVYuXIlx5kyLQ4ZEkqBL6wyEY0XwTiFBAFHCJghz4QLGwGaNF/AwGqBfDos8jJ31d/DDz+cWG0kt7Fz507WqUCfAp2AUDoC5KhpkT+OVaNGDWvmzJnpBXw8gxUSWV1ZZNZtrVq1yseepasoIID3lT9vrCig5dMY/BIiGA7pQVgJTuFNch4dlOlkEsyChJzPLDLptSjMiVWhQgU+d/vtt+fcZtwqUGRk69prr7UoyZVF5tLWkiVLfIGAQvpbpONiIwr0//nnn/vSr3QSPQTwrIti3dF6zb9CXirWs43CbRRg5Bu59957eWsGIT+Q2a5WrVpq4MCBqmXLltm6k/MpCCAEP2KNIUcJkoDBkAFe/zgQ2DJfwpYjYqvhgLIc8dHgkIrYZiRI8m1e6scYAfavgWyMMQbGDT0IIQIQ8MLCSwV6DqHgEFi7dq1atGhR4qUPYYyIAQipQttf/BdhZg4//HA+kKoXQoi2EflAqBo4fUK3gQPWYC+//HJCKDVv3lxVq1YtuAFKz5FCQISIgdMZlBABFIWZABsIVSxYIj2FWr16NScFg1DA/YHIyrbQQJQB/LaFih11AAIHB1aGCKIpJAh4gYAIES9QzbPNIIUIWHcbBTjPYUt1QUAQCCECeF+JiW8IJ85LlnWZAHvJo7QtCAgC5iAgSanMmQtjOIHSFXnVEUgQSa7wtSEkCAgCgkAmBESxngmVAM8FvZ2VPHTkA4EwQZIrCuIYICrStSAgCJiIgGxnmTgrBvGEBFNIeYskV2+88YZBnAkrgoAgYAoCsp1lykwYzAdyrcMEGFtbYgJs8EQJa4JAAAiIYj0A0MPYJRzVEOH3rrvuCiP7wrMgIAh4hIAIEY+AjWKzlI9EHXjggapr165RHJ6MSRAQBFwgIELEBWhxriImwHGefRm7IJCOgOhE0jGRM0UgICbARQAklwWBGCEgJr6GTbZJJr5FQSMmwEUhJNcFgWgjICa+0Z5fz0cnJsCeQywdCALGIyDbWcZPkfkMigmw+XMkHAoCXiEginWvkI1Zu2ICHLMJl+EKAv9FQISI3AraEBATYG1QSkOCQGgQECESmqkKB6NiAhyOeRIuBQFdCIhORBeS0k4CATEBlptBEIgPAmLia9hch8nEtyjoxAS4KITkuiAQbgTExDfc82c892ICbPwUCYOCQN4IyHZW3hBKA0UhICbARSGU/fqKFSsK5FhH7LL3339f7dq1i4/TTz+df//lL3/ho2HDhur333/n/Op2jvW6detm70CuCAJ5IiDbWXkCqLt6lLazUrG55pprVJUqVdTQoUNTL8nv/yKwZs0atWjRIgWTaRwXX3wx53Q5+eSTWSjUrFlTlShRIiE0DjvsMPXzzz8nhMru3btZ6Hz22Wd8FCtWTC1YsIBD+eNo1qwZtyUkCOhAAO8rESI6kNTYRpSFCGC6++67OVPilClTNKIW7qa2bdumpk6dqqZNm8YZJI855pjESx+ri3xp586dCaG0efNm9fXXXysYP3Tp0kWVLl063+alfowRECFi4ORHXYgAcrwwn3jiCbV48WIDZ8A/lj755BP14IMPqnnz5iVe6g0aNPCcAWx/2UKrXbt2avDgwbzCERIEckVAhEiuiPlQPg5CBDC+9dZb/CWMbIkYc5zomGF3aQAAEJZJREFUww8/5BXZF198wUm+LrvsssCGP2vWLN5ehBCBj4/oTwKbilB2LELEwGmLixAB9HE0Ab7uuusU9BaXXHKJatu2rTF34Jw5cxQObJ+NGzfOGL6EEbMRwPtKPNbNnqNIcxcnE2DogPDAwXoKug+TBAhusvbt26vp06fzSuSggw5iHoUEAScIiGLdCUo+lonTSiQZVlgOYXurW7duPqLtT1ewsDrhhBPUyJEj/ekwz15gIjxo0CBWwM+dOzfP1qR6lBGQlUiUZzdkY4tiFOClS5cqmOD26NEjNAIEtw18USDwunbtqo444gi1fPnykN1Nwq6fCMh2lp9oS1+FIhClKMDYvoLV09atW1Xr1q0LHbepF7HltmHDBjVgwAA1Y8YMU9kUvgJGQLazAp6A1O7jup2VjEPYTYBhtrtq1So1efLk1OkN7e/OnTsrmB9DoESJ4mYZiLmzLEvbFIp1ljYo9TUkQuRPLMNqAjxmzBg2Wx42bJi+m8KQlgYOHMje7rAwiwrF7XnTPV4RIgY+Cbon2cAhOmYpbCbAECBwIBw7dqzjMYatYK9evVS9evUU/kaB4va86R4v2hOdSBSehIiOIUwmwNjCwgokygIEt9n48eM5JldYLM0i+mgYNSwRIkZNhzCTCQFEAb7nnnvUpEmTMl0O/ByU6NCBRHELKxO4I0aM4MjBomzPhE78zoli3bA5173cNGx4ebFjYhRgmPHCCiuOccCaNGmiRo8erU477bS85jXIynF73nSPV3QiQd69WfrWPclZugntadOiAMMPBGa8hx56aGgxdcv4jh07VNWqVdX27dvdNhF4vbg9b7rHKzqRwG9hYSBXBBAk8MILL1TnnXderlW1l4cn+syZM2MpQAAmHBGxxYiQKULxRUB0IvGd+9COHLkwoCPBV7BOm/dcAIEeBKFMwupImMtYCyuLUPKVK1cW/UhhIEX8muhEDJtg3ctNw4anlZ0gTYBlnv43lfv371clS5ZU+/bt0zq/fjQWt3nUPV7ZzvLjLpU+PEMgKBNgONtNmDDBs3GFrWFE/R01apTq06dP2FgXfjUgINtZGkCUJoJFwE8TYCSUQj4QBFUU+h8CvXv3ZgX7ypUrBZaYISBCJGYTHtXh+hUFGNZhSCgllI4AFOzARyheCIgQidd8R3q0XkcBRkgTpLQ1LaGUKZN66aWX8koEHu1C8UFAhEh85joWI/XSBBihTZATXSg7AsAnLp772VGI1xWxzjJsvnVbTxg2PN/Y0R0FeNu2bapatWrqp59+8m0MYe3o8MMPV5s2beJ87aZT3J433eMV6yzT73DhzzUC5557rnr77bc5S99XX33luh27InKcwD9FqGgEgBPwEooHArKdFY95juUodZoAT5s2jXPACxWNAHACXkLxQECESDzmOdajzNcEeM2aNeydjsx+QkUj0KhRI/ZiX7t2bdGFpUToERAhEvoplAE4QSAfE+BFixapihUrOulGyvwXgfLlyyvgJhR9BESIRH+OZYT/RcCtCTAE0AUXXCA45oAA8AJucaIwRzPOZ55EiOSDntQNHQJuTIC9ECLr1q1jr/fatWtzFOAqVaqo5s2bq5deeil0mGZiOMpCBPMGqyTkkIEVWps2bVSpUqXUE088kQmKyJ8TIRL5KZYBpiJQWBTgDh06cJ50m1asWKEQ8l2nueqSJUtYeDz++OPsvFi2bFn1448/qldeeUVddNFFCrnaw05lypRRLVu25IyPUSWYfZ9//vlq/vz5qkSJEjyPcSQRInGcdRmzymQCjJfCnDlz1AMPPJBAaPXq1eqPP/7Qilj37t3Vnj17VP/+/dnvZMOGDWrnzp3qvvvu436QKTGMEXFTQUJ0X+AXVcI8HXnkkfzRgXns1q1bVIda6LhEiBQKj1yMMgKpJsCvvfYahzRP9nGAZdbJJ5+sDQZkA/z0009VsWLF1P3336+QGRGESLi33XYbJ9yqW7cuCxYvCZZTjzzyiJddMG7AL6q0efNm9dxzz6latWp5PkQ/5svtIESIuEVO6kUGAdsEGNtIu3btYkEyduxYHh/iQNWoUUPbWLF3DoGB1U0mX4pXX31Vvfvuu2xSbBMScB1//PHq22+/1cYHcqPfcccd2trL1BBwi3IcLcRQq1ChQtrQwzpfaQNxeEKEiEOgpFh0EViwYIH69ddf1bJly3iQCPX+//7f/+P/w+O9Zs2a2gZfvHhx1a9fP24PCtrq1aurvn37qieffFJt3LgxYz/YKlm/fr36/fffM17P9SR0L48++miu1XIuj5UIFNBBEsbqFSUL+uQ+wjpfbnESIeIWOakXCQSQJ/zyyy9X7733nvrll18SY8K203/+8x/1/vvvs9JUJz300ENq+vTp6owzzmCHPKyAOnXqpLC9dtZZZ6k333xTZ3eJtjp37sxbTLACg07Ga8Kqa/ny5V53U2j7iCyMlSWCZ+qmo48+WneTBdrze77cDkaEiFvkpF4kELj33ntVixYtFIIGQqDghQOCEIGCHdtbOi2z0Da+ziE0ILiwRTVr1ixejSBnPCy3ks1jsXIZOXIk8wQv8FNOOYX/74aw2jrppJNUq1attI8pEz/AFAYDQdLDDz/M24eILgyhplOYQK+VSmGer9SxOP5tCRmFAE2cUfzEiRkyR7XopWM1bdrUOuSQQyza77ZI8W2RINEGA5nyWujn66+/TmuTrJms1q1bW7gHOnbsyNenTJli0cqBzw0fPtwiRW5aPZygjIvWrbfeapFVV8brqSfr1KljlS5dOvW01t8kQCwSwFrbdNNYuXLlGD8cJEgs+lCwKFy9tXfvXj6XK1177bVcjwwT0qqaPl9uxps2yKQTjGthBeSa/wjonmT/RxCdHskh0CI/AK0DmjdvHr+AyKInY7t4MeE6ZU9MXCdTYD5HOpOMdXCStse4zG+//Za1TPIFP4QI+qtXr17iBc4vnIAOWjUU6BsfCbYwcARYUqHChAiKmTxfut8vaC99PUZnhQQBQUCxhRR0Ij///LM2OBo2bKiw5QEzX5j0Jre9dOnSxHYLnNjCTtgKhGc+vVcDPWglkvD1wZYWdFywoIqCU6cJ94gIERNmQXgwFgHoQ/Ay1EUI5IgXGAh+ItDDQKEOpzVEv4UVFjy9e/furavLwNqBPgR6kSBp8uTJilZnrA+B8BgyZAhb38FR8OCDDw6Stcj0fVBkRiIDEQQ8QMAWIjqj+N5yyy2KtnnUiBEj2I9iy5YtCpY+OAdvdoRegWlxYUQ6EPXBBx8kisC3BDRp0iRe6djUvn17hRAkQZAJQgRRAeD9D8ENwREUhWG+3GIjQsQtclIvFghg+wlfrroJFmE43BLiNeGrOpVSVzBnnnlmYEIEuAWdg+Xpp59mk2ad9NhjjykcuVAY5iuX8SSXle0st8hJvVggAAc/E+M/wWQ1WdcAvxMQtm6SzyPQY1AEvU/QpFuAuB1PGObL7dhEiLhFTurFAgGTQneQSWqoMNcdMiZUgydmwzZfbvEVIeIWOakXCwRMECJQCoNuvvlmNX78+NDgHlchEtb5cntjiRBxi5zUiwUCiNCayTPZz8FfffXVqlmzZmrhwoVq5syZfnadV1/wFPcjwm1eTHpQOazz5RaKA+Ac47ay1NOPAEJiyJToxzWfFmGGi9DsQZur5jMGv+siCCFCrCDZlskUt+dN93jRnqxETL7DhTcjEIhyqlevAPYipbBXvEq7+SEgQiQ//KR2DBAQIZL7JIsQyR2zsNYQIRLWmRO+fUMA+ghksRNyjgCiE5tiXuucaynpBgERIm5QkzqxQgA5OCjqLsfREioaAYSzhxd+tWrVii4sJUKPgAiR0E+hDMAPBK666qoCudf96DOsfSBHPfASigcCYp1l2Dzrtp4wbHihZWf79u2c53zbtm2hHYNfjCPeGFYilIvFry5d9xO35033eMU6y/WtJxXjhgAlcFJt27blLIRC2RGYMWOGuuKKK0IhQLKPQq7kgoCsRHJBy4eyur8UfGA5Nl0ghhYli+JcIEKZEahevbp64YUX2EckDBS35033eGUlEoa7XHg0BoGaNWsqHHPmzDGGJ5MYQcTc+vXrh0aAmIRdmHkRxXqYZ0949x2BO++8U4RIFtQp/7sCPkLxQkCESLzmW0abJwJ169ZVUByHKRBinkN2VJ1yw3N2xiBDzztiVAppR0B0Itohza9B3XuW+XEjtbMhgOCCe/bsKTIDYbb6UToPHBBfDLlMwkZxe950j1d0ImG744VfYxBAGtpBgwYZw0+QjNx0001q4sSJQbIgfQeIgGxnBQi+dB1eBLp06cJe7NADxJlmz56tEOLkyiuvjDMMsR67bGcZNv26l5uGDS9y7MQ5THxYwr0XdtPF7XnTPV7Zzirs7pJrgoADBBCttlWrVg5KRq8Ixv3aa69Fb2AyopwQkO2snOCSwoJAQQQaNGigevbsqTp37hwraDp27KhuvPFGVa9evViNWwabjoAIkXRM5IwgkBMCnTp1UhAmAwcOzKleWAvfcMMNqkmTJqpDhw5hHYLwrREBESIawZSm4ovAgAEDFELG9+rVK9Ig9OjRQ8FXpl+/fpEepwzOOQKiWHeOlS8ldSu+fGFaOkkgACfEzz77TI0YMSJyqGAFAgHSvXv3yIwtbs+b7vGKYj0yj4IMxBQEsBKpWrVq5HQk0IEgqGKUBIh9z+BFGJfDi+dEViJeoJpHm7q/FPJgRarmgQBCoo8dO1YtXLiQvbnDSjDjbd26tcIqRHQgYZ1F7/hm4WsRedeFtJwrAiJEckXM3PIffPCBuuCCCxS829u1a2cuo1k4gyMhLM9gxitWWFlAivlpESIG3gAiRAyclDxZat++vapUqRLrSRBzy3RCLCyEMoEn+jPPPGM6u8JfgAiITiRA8KXr+CCA/CONGjVSJUuWVOPGjTN64IjGi+23xo0biwAxeqbMYU62s8yZC+ZEViKGTYhmdvr06aOQrx2rk0svvVRz6+6bQ0IpxAFDOPfRo0e7b0hqxgoB2c4ycLpFiBg4KZpZWrlypbr77rsV/g4ZMkTB8ikomj59OvOCjIRIKCX5QIKaiXD2K9tZ4Zw34TrkCJxyyim8VTRv3jxWWh9++OGqb9++6r333vNlZEuWLFFYESG51uLFizkn+qxZs0SA+IJ+9DqR7SzD5lRWIoZNiA/s7Nq1S02dOlVNmzZNVa5cWZUvX56tunCUKVMmbw5gpotAkTigLN+yZYu66qqrFMLZH3bYYXm3Lw3EFwHZzjJw7kWIGDgpPrK0du1atWjRosRLv2XLlmr//v0cUqVGjRr8t1SpUrx6wYHVBITQzp07+di9e7f69NNP2WseB6zBXn755YRQat68uapWrZqPI5KuooyACBEDZ1eEiIGTEiBLq1atUqtXr1Zr1qxhoYD7Y/ny5QmhgcCP+G0LFfwGQeDgqFWrlmxTBTh/Ue9ahIiBMyxCxMBJEZYEAUEgIwJ4X0kU34zQyElBQBAQBAQBJwiIEHGCkpQRBAQBQUAQyIiACJGMsMhJQUAQEAQEAScIiBBxgpKUEQQEAUFAEMiIgAiRjLDISUFAEBAEBAEnCIgQcYKSlBEEBAFBQBDIiIAIkYywyElBQBAQBAQBJwiIEHGCkpQRBAQBQUAQyIiA+RlyMrId7ZNw4BESBAQBQSAMCPx/DpiGn+i+oXMAAAAASUVORK5CYII=)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "85icnDUx8vtG"
      },
      "source": [
        "As mentioned before, the agent must iteratively decide what action must take. In this regard, a policy $\\pi$ is a process that decides how to take an action.\n",
        "In this tutorial, we will discuss the $\\epsilon$-greedy policy.\n",
        "In this policy, the e parameters balance between exploration and exploitation of the actions.\n",
        "\n",
        "In the exploration stage, selects a random action with an $\\epsilon$ probability of happening. By counterpart, with a 1 - $\\epsilon$ probability, the exploitation stage selects an action using the information of the future expected reward. Being formulated as follows:\n",
        "\n",
        "\\begin{align}\n",
        "a = \n",
        "\\begin{cases}\n",
        "    \\text{random} & \\text{with } \\epsilon \\text{ probability}\\\\\n",
        "    \\max V(s_t) & \\text{with } 1 - \\epsilon \\text{ probability}\\\\\n",
        "\\end{cases}\n",
        "\\end{align}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b4UR-1daHWyg"
      },
      "source": [
        "class TDAgent(RandomAgent):\n",
        "\n",
        "    def __init__(self, actions, grid_size=(12, 4)):\n",
        "        super().__init__(actions, grid_size)\n",
        "\n",
        "    def get_epsilon_greedy_action(self, state, epsilon=0.99):\n",
        "        # explotation\n",
        "        if random.random() > epsilon:\n",
        "            return np.argmax(self.Q_values[tuple(state)])\n",
        "        # exploration\n",
        "        return self.get_random_action()\n",
        "\n",
        "    def get_action(self, state, epsilon=0.99):\n",
        "        return self.get_epsilon_greedy_action(state, epsilon)    "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "koCa0QgPaO89"
      },
      "source": [
        "## $Q$-learning algorithm\n",
        "One widely studied policy update methodology is the $Q$-learning algorithm [6].\n",
        "This algorithm implements an off-policy update scheme, improving the value function prediction.\n",
        "For this algorithm, the value function is now a $Q$ values estimator for each state or state-action pair ($Q(s_t, a_t)$). The following formulation converge in an optimal policy $q*$:\n",
        "\n",
        "One widely studied policy update methodology is the $Q$-learning algorithm [6].\n",
        "This algorithm implements an off-policy update scheme, improving the value function prediction.\n",
        "The value function is now a $Q$ values estimator for each state or state-action pair for this algorithm. The following formulation converges in an optimal policy q*:\n",
        "\n",
        "\\begin{align}\n",
        "Q(s_t, a_t) = Q(s_t, a_t) + \\alpha [r_{t+1} + \\gamma \\max_a Q(s_{t+1}, a) - Q(s_t, a_t)]\n",
        "\\end{align}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EWrG4ogQLFfH"
      },
      "source": [
        "def q_learning_update(agent, state, action, reward, next_state,\n",
        "                      alpha=0.5, gamma=0.99):\n",
        "    \"\"\"This function intends to update the agent.Q_values with the Q-learning \n",
        "    algorithm.\n",
        "\n",
        "    Args:\n",
        "        agent (RL Agent class): an instance of the RL agent.\n",
        "        state (list): a cell position from the environment.\n",
        "        action (int): the selected action for the state.\n",
        "        reward (int): value which represent the reward.\n",
        "        next_state (list): a cell position from the environment after perform\n",
        "                           the action.\n",
        "        alpha (float): learning rate.\n",
        "        gamma (float): long-term discounted reward value.\n",
        "    \n",
        "    Returns:\n",
        "        np.ndarray: the updated agent.Q_values.\n",
        "\n",
        "    This method is for you to implement following the Q-learning equation in\n",
        "    the upper cell.\n",
        "    \"\"\"\n",
        "\n",
        "    q_idx = tuple(state) + (action, )  # state-action pair index\n",
        "    # obtain the Q values from the agent.Q_values\n",
        "    current_q = ...\n",
        "    # get the best Q value for the next state\n",
        "    if next_state is None:\n",
        "        best_q = 0\n",
        "    else:\n",
        "        best_q = ...\n",
        "    \n",
        "    # put back the td-error terms\n",
        "    agent.Q_values[q_idx] += alpha * (...)\n",
        "    \n",
        "    return agent.Q_values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "H6Z3FpAzNWqM"
      },
      "source": [
        "#@title Testing Q-learning agent's performance\n",
        "\n",
        "#@markdown Make sure you execute this cell to enable the widget!\n",
        "\n",
        "n_episodes = 1000\n",
        "\n",
        "@widgets.interact\n",
        "def plot_tde_by_trial(episodes=widgets.IntSlider(value=10, min=0, max=n_episodes, step=5, description=\"Episodes #\"),\n",
        "                      epsilon=widgets.FloatSlider(value=0.5, min=0.05, max=1., step=0.05, description=\"Exploration rate: \"),\n",
        "                      alpha=widgets.FloatSlider(value=0.5, min=0.05, max=1., step=0.05, description=\"Learning rate: \"),\n",
        "                      gamma=widgets.FloatSlider(value=0.99, min=0.01, max=1., step=0.01, description=\"Long-term discount factor: \")):\n",
        "    # environment instance\n",
        "    environment = CliffWalking(12, 4)\n",
        "    available_actions = list(range(environment.n_actions))\n",
        "    agent = TDAgent(actions=available_actions)\n",
        "    params = dict(epsilon=epsilon,\n",
        "                alpha=alpha,\n",
        "                gamma=gamma,\n",
        "                function_update=q_learning_update)\n",
        "\n",
        "    # training loop and results\n",
        "    rewards, hist = training_loop(agent, environment, episodes, params=params,\n",
        "                                log=False)\n",
        "    best_ep, best_Q, best_path = hist['best']\n",
        "\n",
        "    print('last best path, ep', best_ep)\n",
        "    fig, axes = plt.subplots(1, 3, figsize=(18, 6))\n",
        "    plot_reward(rewards, ax=axes[0])\n",
        "    plot_quiver_max_action(environment, hist['Q_values'], ax=axes[1])\n",
        "    plot_path(environment, action_hist=best_path, ax=axes[2])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "35CB5sSRO1tS"
      },
      "source": [
        "## Discussion\n",
        "\n",
        "One advantage is that there is no need to have a labeled dataset for its training, relying only on the reward signal to optimize the agent overcome.\n",
        "Additionally, RL algorithms can model complex robotics behaviors, which classic machine learning paradigms can not achieve. Different studies in diverse areas are researching more about these methods, being the most approximated methods to the natural learning process.\n",
        "\n",
        "In the counterpart, RL methods still require too many samples to learn how to solve a given task. Some transfer-learning and human-interaction approaches [7-8] are current forms to reduce the experiences required in highly complex tasks. Another challenging gap is the real-world domain shifting, where the algorithm is more exposed to noise signals and other differences from a simulated environment."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ci87Vamez3vH"
      },
      "source": [
        "### Some useful tools\n",
        "\n",
        "- [OpenAI Gym](https://gym.openai.com/) [(GitHub)](https://github.com/openai/gym/): Gym is a toolkit for developing and comparing reinforcement learning algorithms. It supports teaching agents everything from walking to playing games like Pong or Pinball.\n",
        "- [MushroomRL](https://mushroomrl.readthedocs.io/en/latest/) [(GitHub)](https://github.com/MushroomRL/mushroom-rl): MushroomRL is a Reinforcement Learning (RL) library developed to be a simple, yet powerful way to make RL and deep RL experiments. \n",
        "- DeepMind RL Course [(YouTube)](https://www.youtube.com/watch?v=2pWv7GOvuf0&list=PLqYmG7hTraZDM-OYHWgPebj2MfCFzFObQ).\n",
        "- A Webots simulator scene with OpenAI Gym [(GitHub repository)](https://github.com/angel-ayala/webots-fire-scene).\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0jsQcrjrH3aP"
      },
      "source": [
        "## References\n",
        "\n",
        "[1] Neftci, E.O., Averbeck, B.B. Reinforcement learning in artificial and biological systems. Nat Mach Intell 1, 133–143, (2019). https://doi.org/10.1038/s42256-019-0025-4\n",
        "\n",
        "[2] Pavlov, P. I. Conditioned reflexes: an investigation of the physiological activity of the cerebral cortex. Annals of neurosciences, 17(3), 136, (2010).\n",
        "\n",
        "[3] Sutton, R. S., and Barto, A. G. Reinforcement learning: An introduction. MIT press, (2018). http://incompleteideas.net/book/the-book-2nd.html \n",
        "\n",
        "[4] Glimcher, P. W. Understanding dopamine and reinforcement learning: The dopamine reward prediction error hypothesis. Proceedings of the National Academy of Sciences, 108 (Supplement 3) 15647-15654, (2011). http://doi.org/10.1073/pnas.1014269108\n",
        "\n",
        "[5] Barto, A. G. Temporal difference learning. Scholarpedia, 2(11), 1604, (2007).\n",
        "\n",
        "[6] Watkins, C. J., and Dayan, P. Q-learning. Machine learning, 8(3-4), 279-292, (1992).\n",
        "\n",
        "[7] Cruz, F., Magg, S., Weber, C., and Wermter, S. Training agents with interactive reinforcement learning and contextual affordances. IEEE Transactions on Cognitive and Developmental Systems, 8(4), 271-284, (2016).\n",
        "\n",
        "[8] Moreira, I., Rivas, J., Cruz, F., Dazeley, R., Ayala, A., and Fernandes, B. Deep Reinforcement Learning with Interactive Feedback in a Human–Robot Environment. Applied Sciences, 10(16), 5574, (2020).\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PSJhjNqA7PQm"
      },
      "source": [
        "# Are you interested in learn something more?\n",
        "If you want to learn something extra about RL, you can start developing your own agent to solve the following gym environment named Cartpole-v1.\n",
        "In this activity, you must consider that different data or numerical domains can represent the action and state in a problem.\n",
        "Here you must replace the table-based Q-learning algorithm for an approximation function to process the state in the continuous domain. Same thing for the image array state, which is commonly processed by a convolutional neural network."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oe7UsyZB_I9J"
      },
      "source": [
        "The following image illustrates the Cartpole problem.\n",
        "\n",
        "<img src=\"https://d3i71xaburhd42.cloudfront.net/3dd67d8565480ddb5f3c0b4ea6be7058e77b4172/2-Figure1-1.png\">\n",
        "\n",
        "This problem belongs to the continuous domain. The main idea is to keep the pole vertically as long as possible, considering the problem is solved after 495 steps.\n",
        "This problem can be formulated as an MDP as follows:\n",
        "\n",
        "- __States__: represented as a vector with 4 components $< x, x', \\theta, \\theta' >$ corresponding to a) the car position respect the lane center; b) the car velocity; c) the pole angle of inclination; d) the pole angular velocity;\n",
        "- __Actions__: represented as a binary value to specify the direction of the force $F$.\n",
        "- __Transition function__: defined in the works of [Brownlee, 2005](https://researchbank.swinburne.edu.au/file/62a8df69-4a2c-407f-8040-5ac533fc2787/1/PDF%20(12%20pages).pdf) comprising:\n",
        "  \n",
        "  Diferential equation of pole movement:\n",
        "\\begin{align}\n",
        "\\ddot{\\theta}_t = \\frac{g \\sin\\theta_t + \\cos\\theta_t [\\frac{- F_t - m_pl \\dot{\\theta}^2_t \\sin\\theta}{m_c + m_p}]}{l[\\frac{4}{3} - \\frac{m_p \\cos^2\\theta_t}{m_c + m_p}]}\n",
        "\\end{align}\n",
        "\n",
        "  Diferential equation of care movement:\n",
        "\\begin{align}\n",
        "\\ddot{x}_t = \\frac{F_t + m_p l [\\dot{\\theta}^2_t \\sin\\theta_t - \\ddot{\\theta}_t \\cos\\theta_t]}{m_c + m_p}\n",
        "\\end{align}\n",
        "\n",
        "  where $m_c$ is the car mass, $m_p$ is the pole mass, $l$ is the pole length, $F$ the applied force, and $g$ gravitacional acceleration.\n",
        "\n",
        "- __Reward function__: defined by\n",
        "\\begin{align}\n",
        "r= \n",
        "\\begin{cases}\n",
        "1 & \\text{pole inside the threshold angle.}\\\\\n",
        "0 & \\text{if surpass the threshold angle.}\n",
        "\\end{cases}\n",
        "\\end{align}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "zNUKSCgkDl1v"
      },
      "source": [
        "#@title Libraries and helpers\n",
        "!apt-get install -y xvfb python-opengl > /dev/null 2>&1\n",
        "!pip install pyvirtualdisplay > /dev/null 2>&1\n",
        "\n",
        "import gym\n",
        "import math\n",
        "import random\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt # grafico\n",
        "from gym.wrappers.monitor import load_results # resultados monitor\n",
        "# visualización de entorno\n",
        "from IPython import display as ipythondisplay\n",
        "from pyvirtualdisplay import Display\n",
        "\n",
        "%matplotlib inline\n",
        "display = Display(visible=0, size=(400, 300))\n",
        "display.start()\n",
        "\n",
        "def show_render(env, step=0, info=\"\"):\n",
        "    plt.figure(3)\n",
        "    plt.clf()\n",
        "    plt.imshow(env.render(mode='rgb_array'))\n",
        "    plt.title(\"%s | Step: %d %s\" % (env.spec.id,step, info))\n",
        "    plt.axis('off')\n",
        "\n",
        "    ipythondisplay.clear_output(wait=True)\n",
        "    ipythondisplay.display(plt.gcf())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4ii5mRgf--Zs"
      },
      "source": [
        "#@title Cartpole environment\n",
        "environment = gym.make(\"CartPole-v1\")\n",
        "# MDP information\n",
        "print(environment.action_space)  # action space config\n",
        "print(environment.action_space.n)  # number of actions\n",
        "print(environment.observation_space)  # observation space or state config\n",
        "print(environment.observation_space.low)  # min values in the state vector\n",
        "print(environment.observation_space.high)  # max values in the state vector"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WcAkoJ9HEVIv"
      },
      "source": [
        "episodes = 10\n",
        "timesteps = 0\n",
        "\n",
        "for e in range(episodios):\n",
        "    state = environment.reset()\n",
        "    ep_reward = 0\n",
        "    end = False\n",
        "    while not end:\n",
        "        # render environment\n",
        "        show_render(environment, timesteps, \"EP {}\".format(e))\n",
        "        # random action selection\n",
        "        action = ...\n",
        "        # policy action selection\n",
        "        # action = ...\n",
        "        # ejecutar accion y observar nuevo estado\n",
        "        next_state, reward, end, info = environment.step(action) \n",
        "        ep_reward += reward\n",
        "        state = next_state\n",
        "        timesteps += 1\n",
        "    print('EP end', e, 'reward', ep_reward)\n",
        "\n",
        "environment.close()"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}