dqn_working.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "scratchpad",
      "provenance": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "lIYdn1woOS1n",
        "outputId": "b98f40e7-4b90-41b3-e3ea-a6190c4b2063",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# Reference: https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.tensorboard import SummaryWriter\n",
        "\n",
        "import argparse\n",
        "from distutils.util import strtobool\n",
        "import collections\n",
        "import numpy as np\n",
        "import gym\n",
        "from gym.wrappers import TimeLimit, Monitor\n",
        "from gym.spaces import Discrete, Box, MultiBinary, MultiDiscrete, Space\n",
        "import time\n",
        "import random\n",
        "import os\n",
        "\n",
        "# TRY NOT TO MODIFY: seeding\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "env = gym.make('CartPole-v1')\n",
        "seed = 1234\n",
        "random.seed(seed)\n",
        "np.random.seed(seed)\n",
        "torch.manual_seed(seed)\n",
        "env.seed(seed)\n",
        "\n",
        "# modified from https://github.com/seungeunrho/minimalRL/blob/master/dqn.py#\n",
        "class ReplayBuffer():\n",
        "    def __init__(self, buffer_limit):\n",
        "        self.buffer = collections.deque(maxlen=buffer_limit)\n",
        "    \n",
        "    def put(self, transition):\n",
        "        self.buffer.append(transition)\n",
        "    \n",
        "    def sample(self, n):\n",
        "        mini_batch = random.sample(self.buffer, n)\n",
        "        s_lst, a_lst, r_lst, s_prime_lst, done_mask_lst = [], [], [], [], []\n",
        "        \n",
        "        for transition in mini_batch:\n",
        "            s, a, r, s_prime, done_mask = transition\n",
        "            s_lst.append(s)\n",
        "            a_lst.append(a)\n",
        "            r_lst.append(r)\n",
        "            s_prime_lst.append(s_prime)\n",
        "            done_mask_lst.append(done_mask)\n",
        "\n",
        "        return np.array(s_lst), np.array(a_lst), \\\n",
        "               np.array(r_lst), np.array(s_prime_lst), \\\n",
        "               np.array(done_mask_lst)\n",
        "\n",
        "# ALGO LOGIC: initialize agent here:\n",
        "class QNetwork(nn.Module):\n",
        "    def __init__(self, env):\n",
        "        super(QNetwork, self).__init__()\n",
        "        self.fc1 = nn.Linear(np.array(env.observation_space.shape).prod(), 64)\n",
        "        self.fc2 = nn.Linear(64, 64)\n",
        "        self.fc3 = nn.Linear(64, env.action_space.n)\n",
        "\n",
        "    def forward(self, x, device):\n",
        "        x = torch.Tensor(x).to(device)\n",
        "        x = torch.tanh(self.fc1(x))\n",
        "        x = torch.tanh(self.fc2(x))\n",
        "        x = self.fc3(x)\n",
        "        return x\n",
        "\n",
        "def linear_schedule(start_e: float, end_e: float, duration: int, t: int):\n",
        "    slope =  (end_e - start_e) / duration\n",
        "    return max(slope * t + start_e, end_e)\n",
        "\n",
        "buffer_size = 10_000\n",
        "lr = 1e-5\n",
        "\n",
        "rb = ReplayBuffer(buffer_size)\n",
        "q_network = QNetwork(env).to(device)\n",
        "target_network = QNetwork(env).to(device)\n",
        "target_network.load_state_dict(q_network.state_dict())\n",
        "optimizer = optim.Adam(q_network.parameters())\n",
        "loss_fn = nn.MSELoss()\n",
        "print(device.__repr__())\n",
        "print(q_network)\n",
        "\n",
        "# TRY NOT TO MODIFY: start the game\n",
        "obs = env.reset()\n",
        "episode_reward = 0\n",
        "total_timesteps = 100_000\n",
        "start_e = 1.0\n",
        "end_e = 0.05\n",
        "exploration_fraction = 0.5\n",
        "exploration_steps = int(exploration_fraction * total_timesteps)\n",
        "train_frequency = 10\n",
        "learning_starts = buffer_size\n",
        "batch_size = 64\n",
        "max_grad_norm = 1.0\n",
        "target_network_frequency = 100\n",
        "gamma = 0.99\n",
        "\n",
        "episode = 0\n",
        "episode_rewards = []\n",
        "step_rewards = []\n",
        "\n",
        "for global_step in range(total_timesteps):\n",
        "\n",
        "    # ALGO LOGIC: put action logic here\n",
        "    epsilon = linear_schedule(start_e, end_e, exploration_fraction*total_timesteps, global_step)\n",
        "    \n",
        "    if random.random() < epsilon:\n",
        "        action = env.action_space.sample()\n",
        "    else:\n",
        "        logits = q_network.forward(obs.reshape((1,)+obs.shape), device)\n",
        "        action = torch.argmax(logits, dim=1).tolist()[0]\n",
        "\n",
        "    # TRY NOT TO MODIFY: execute the game and log data.\n",
        "    next_obs, reward, done, _ = env.step(action)\n",
        "    episode_reward += reward\n",
        "    step_rewards.append(reward)\n",
        "\n",
        "    # ALGO LOGIC: training.\n",
        "    rb.put((obs, action, reward, next_obs, done))\n",
        "    if global_step > learning_starts and global_step % train_frequency == 0:\n",
        "        s_obs, s_actions, s_rewards, s_next_obses, s_dones = rb.sample(batch_size)\n",
        "        with torch.no_grad():\n",
        "            target_max = torch.max(q_network.forward(s_next_obses, device), dim=1)[0]\n",
        "            td_target = torch.Tensor(s_rewards).to(device) + gamma * target_max * (1 - torch.Tensor(s_dones).to(device))\n",
        "        old_val = q_network.forward(s_obs, device).gather(1, torch.LongTensor(s_actions).view(-1,1).to(device)).squeeze()\n",
        "        loss = loss_fn(td_target, old_val)\n",
        "\n",
        "        # optimize the midel\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        nn.utils.clip_grad_norm_(list(q_network.parameters()), max_grad_norm)\n",
        "        optimizer.step()\n",
        "\n",
        "    # update the target network\n",
        "    if global_step % target_network_frequency == 0:\n",
        "        target_network.load_state_dict(q_network.state_dict())\n",
        "\n",
        "    # TRY NOT TO MODIFY: CRUCIAL step easy to overlook \n",
        "    obs = next_obs\n",
        "\n",
        "    if done:\n",
        "        episode += 1\n",
        "        # TRY NOT TO MODIFY: record rewards for plotting purposes\n",
        "        print(f\"episode={episode}, global_step={global_step}, episode_reward={episode_reward}, epsilon={epsilon}\")\n",
        "        episode_rewards.append(episode_reward)\n",
        "        obs, episode_reward = env.reset(), 0"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "device(type='cuda')\n",
            "QNetwork(\n",
            "  (fc1): Linear(in_features=4, out_features=64, bias=True)\n",
            "  (fc2): Linear(in_features=64, out_features=64, bias=True)\n",
            "  (fc3): Linear(in_features=64, out_features=2, bias=True)\n",
            ")\n",
            "episode=1, global_step=39, episode_reward=40.0, epsilon=0.999259\n",
            "episode=2, global_step=61, episode_reward=22.0, epsilon=0.998841\n",
            "episode=3, global_step=88, episode_reward=27.0, epsilon=0.998328\n",
            "episode=4, global_step=97, episode_reward=9.0, epsilon=0.998157\n",
            "episode=5, global_step=117, episode_reward=20.0, epsilon=0.997777\n",
            "episode=6, global_step=127, episode_reward=10.0, epsilon=0.997587\n",
            "episode=7, global_step=163, episode_reward=36.0, epsilon=0.996903\n",
            "episode=8, global_step=192, episode_reward=29.0, epsilon=0.996352\n",
            "episode=9, global_step=218, episode_reward=26.0, epsilon=0.995858\n",
            "episode=10, global_step=238, episode_reward=20.0, epsilon=0.995478\n",
            "episode=11, global_step=247, episode_reward=9.0, epsilon=0.995307\n",
            "episode=12, global_step=257, episode_reward=10.0, epsilon=0.995117\n",
            "episode=13, global_step=296, episode_reward=39.0, epsilon=0.994376\n",
            "episode=14, global_step=309, episode_reward=13.0, epsilon=0.994129\n",
            "episode=15, global_step=317, episode_reward=8.0, epsilon=0.993977\n",
            "episode=16, global_step=370, episode_reward=53.0, epsilon=0.99297\n",
            "episode=17, global_step=380, episode_reward=10.0, epsilon=0.99278\n",
            "episode=18, global_step=397, episode_reward=17.0, epsilon=0.992457\n",
            "episode=19, global_step=410, episode_reward=13.0, epsilon=0.99221\n",
            "episode=20, global_step=433, episode_reward=23.0, epsilon=0.991773\n",
            "episode=21, global_step=457, episode_reward=24.0, epsilon=0.991317\n",
            "episode=22, global_step=475, episode_reward=18.0, epsilon=0.990975\n",
            "episode=23, global_step=485, episode_reward=10.0, epsilon=0.990785\n",
            "episode=24, global_step=508, episode_reward=23.0, epsilon=0.990348\n",
            "episode=25, global_step=520, episode_reward=12.0, epsilon=0.99012\n",
            "episode=26, global_step=533, episode_reward=13.0, epsilon=0.989873\n",
            "episode=27, global_step=549, episode_reward=16.0, epsilon=0.989569\n",
            "episode=28, global_step=565, episode_reward=16.0, epsilon=0.989265\n",
            "episode=29, global_step=575, episode_reward=10.0, epsilon=0.989075\n",
            "episode=30, global_step=595, episode_reward=20.0, epsilon=0.988695\n",
            "episode=31, global_step=619, episode_reward=24.0, epsilon=0.988239\n",
            "episode=32, global_step=677, episode_reward=58.0, epsilon=0.987137\n",
            "episode=33, global_step=687, episode_reward=10.0, epsilon=0.986947\n",
            "episode=34, global_step=708, episode_reward=21.0, epsilon=0.986548\n",
            "episode=35, global_step=738, episode_reward=30.0, epsilon=0.985978\n",
            "episode=36, global_step=751, episode_reward=13.0, epsilon=0.985731\n",
            "episode=37, global_step=775, episode_reward=24.0, epsilon=0.985275\n",
            "episode=38, global_step=794, episode_reward=19.0, epsilon=0.984914\n",
            "episode=39, global_step=808, episode_reward=14.0, epsilon=0.984648\n",
            "episode=40, global_step=824, episode_reward=16.0, epsilon=0.984344\n",
            "episode=41, global_step=870, episode_reward=46.0, epsilon=0.98347\n",
            "episode=42, global_step=915, episode_reward=45.0, epsilon=0.982615\n",
            "episode=43, global_step=931, episode_reward=16.0, epsilon=0.982311\n",
            "episode=44, global_step=943, episode_reward=12.0, epsilon=0.982083\n",
            "episode=45, global_step=962, episode_reward=19.0, epsilon=0.981722\n",
            "episode=46, global_step=981, episode_reward=19.0, epsilon=0.981361\n",
            "episode=47, global_step=996, episode_reward=15.0, epsilon=0.9810760000000001\n",
            "episode=48, global_step=1058, episode_reward=62.0, epsilon=0.979898\n",
            "episode=49, global_step=1103, episode_reward=45.0, epsilon=0.979043\n",
            "episode=50, global_step=1134, episode_reward=31.0, epsilon=0.978454\n",
            "episode=51, global_step=1149, episode_reward=15.0, epsilon=0.978169\n",
            "episode=52, global_step=1163, episode_reward=14.0, epsilon=0.977903\n",
            "episode=53, global_step=1177, episode_reward=14.0, epsilon=0.977637\n",
            "episode=54, global_step=1193, episode_reward=16.0, epsilon=0.977333\n",
            "episode=55, global_step=1214, episode_reward=21.0, epsilon=0.976934\n",
            "episode=56, global_step=1250, episode_reward=36.0, epsilon=0.97625\n",
            "episode=57, global_step=1270, episode_reward=20.0, epsilon=0.97587\n",
            "episode=58, global_step=1292, episode_reward=22.0, epsilon=0.975452\n",
            "episode=59, global_step=1316, episode_reward=24.0, epsilon=0.974996\n",
            "episode=60, global_step=1334, episode_reward=18.0, epsilon=0.974654\n",
            "episode=61, global_step=1345, episode_reward=11.0, epsilon=0.974445\n",
            "episode=62, global_step=1371, episode_reward=26.0, epsilon=0.973951\n",
            "episode=63, global_step=1396, episode_reward=25.0, epsilon=0.973476\n",
            "episode=64, global_step=1408, episode_reward=12.0, epsilon=0.973248\n",
            "episode=65, global_step=1418, episode_reward=10.0, epsilon=0.973058\n",
            "episode=66, global_step=1428, episode_reward=10.0, epsilon=0.972868\n",
            "episode=67, global_step=1437, episode_reward=9.0, epsilon=0.972697\n",
            "episode=68, global_step=1516, episode_reward=79.0, epsilon=0.971196\n",
            "episode=69, global_step=1541, episode_reward=25.0, epsilon=0.9707210000000001\n",
            "episode=70, global_step=1606, episode_reward=65.0, epsilon=0.969486\n",
            "episode=71, global_step=1624, episode_reward=18.0, epsilon=0.969144\n",
            "episode=72, global_step=1636, episode_reward=12.0, epsilon=0.968916\n",
            "episode=73, global_step=1655, episode_reward=19.0, epsilon=0.968555\n",
            "episode=74, global_step=1674, episode_reward=19.0, epsilon=0.968194\n",
            "episode=75, global_step=1693, episode_reward=19.0, epsilon=0.967833\n",
            "episode=76, global_step=1708, episode_reward=15.0, epsilon=0.967548\n",
            "episode=77, global_step=1723, episode_reward=15.0, epsilon=0.967263\n",
            "episode=78, global_step=1738, episode_reward=15.0, epsilon=0.966978\n",
            "episode=79, global_step=1752, episode_reward=14.0, epsilon=0.966712\n",
            "episode=80, global_step=1772, episode_reward=20.0, epsilon=0.966332\n",
            "episode=81, global_step=1788, episode_reward=16.0, epsilon=0.966028\n",
            "episode=82, global_step=1815, episode_reward=27.0, epsilon=0.965515\n",
            "episode=83, global_step=1828, episode_reward=13.0, epsilon=0.965268\n",
            "episode=84, global_step=1850, episode_reward=22.0, epsilon=0.96485\n",
            "episode=85, global_step=1868, episode_reward=18.0, epsilon=0.964508\n",
            "episode=86, global_step=1921, episode_reward=53.0, epsilon=0.963501\n",
            "episode=87, global_step=1935, episode_reward=14.0, epsilon=0.9632350000000001\n",
            "episode=88, global_step=1947, episode_reward=12.0, epsilon=0.963007\n",
            "episode=89, global_step=1969, episode_reward=22.0, epsilon=0.962589\n",
            "episode=90, global_step=1994, episode_reward=25.0, epsilon=0.962114\n",
            "episode=91, global_step=2034, episode_reward=40.0, epsilon=0.961354\n",
            "episode=92, global_step=2050, episode_reward=16.0, epsilon=0.96105\n",
            "episode=93, global_step=2073, episode_reward=23.0, epsilon=0.960613\n",
            "episode=94, global_step=2096, episode_reward=23.0, epsilon=0.960176\n",
            "episode=95, global_step=2105, episode_reward=9.0, epsilon=0.960005\n",
            "episode=96, global_step=2120, episode_reward=15.0, epsilon=0.95972\n",
            "episode=97, global_step=2132, episode_reward=12.0, epsilon=0.959492\n",
            "episode=98, global_step=2157, episode_reward=25.0, epsilon=0.959017\n",
            "episode=99, global_step=2197, episode_reward=40.0, epsilon=0.958257\n",
            "episode=100, global_step=2210, episode_reward=13.0, epsilon=0.95801\n",
            "episode=101, global_step=2221, episode_reward=11.0, epsilon=0.957801\n",
            "episode=102, global_step=2236, episode_reward=15.0, epsilon=0.957516\n",
            "episode=103, global_step=2253, episode_reward=17.0, epsilon=0.957193\n",
            "episode=104, global_step=2304, episode_reward=51.0, epsilon=0.956224\n",
            "episode=105, global_step=2317, episode_reward=13.0, epsilon=0.955977\n",
            "episode=106, global_step=2329, episode_reward=12.0, epsilon=0.955749\n",
            "episode=107, global_step=2346, episode_reward=17.0, epsilon=0.955426\n",
            "episode=108, global_step=2362, episode_reward=16.0, epsilon=0.955122\n",
            "episode=109, global_step=2383, episode_reward=21.0, epsilon=0.954723\n",
            "episode=110, global_step=2394, episode_reward=11.0, epsilon=0.954514\n",
            "episode=111, global_step=2411, episode_reward=17.0, epsilon=0.954191\n",
            "episode=112, global_step=2427, episode_reward=16.0, epsilon=0.953887\n",
            "episode=113, global_step=2442, episode_reward=15.0, epsilon=0.9536020000000001\n",
            "episode=114, global_step=2459, episode_reward=17.0, epsilon=0.953279\n",
            "episode=115, global_step=2480, episode_reward=21.0, epsilon=0.95288\n",
            "episode=116, global_step=2492, episode_reward=12.0, epsilon=0.952652\n",
            "episode=117, global_step=2524, episode_reward=32.0, epsilon=0.952044\n",
            "episode=118, global_step=2553, episode_reward=29.0, epsilon=0.951493\n",
            "episode=119, global_step=2583, episode_reward=30.0, epsilon=0.950923\n",
            "episode=120, global_step=2604, episode_reward=21.0, epsilon=0.950524\n",
            "episode=121, global_step=2619, episode_reward=15.0, epsilon=0.9502390000000001\n",
            "episode=122, global_step=2634, episode_reward=15.0, epsilon=0.949954\n",
            "episode=123, global_step=2651, episode_reward=17.0, epsilon=0.949631\n",
            "episode=124, global_step=2668, episode_reward=17.0, epsilon=0.949308\n",
            "episode=125, global_step=2691, episode_reward=23.0, epsilon=0.948871\n",
            "episode=126, global_step=2722, episode_reward=31.0, epsilon=0.948282\n",
            "episode=127, global_step=2741, episode_reward=19.0, epsilon=0.947921\n",
            "episode=128, global_step=2757, episode_reward=16.0, epsilon=0.947617\n",
            "episode=129, global_step=2786, episode_reward=29.0, epsilon=0.947066\n",
            "episode=130, global_step=2805, episode_reward=19.0, epsilon=0.946705\n",
            "episode=131, global_step=2837, episode_reward=32.0, epsilon=0.946097\n",
            "episode=132, global_step=2849, episode_reward=12.0, epsilon=0.945869\n",
            "episode=133, global_step=2864, episode_reward=15.0, epsilon=0.945584\n",
            "episode=134, global_step=2912, episode_reward=48.0, epsilon=0.944672\n",
            "episode=135, global_step=2926, episode_reward=14.0, epsilon=0.944406\n",
            "episode=136, global_step=2940, episode_reward=14.0, epsilon=0.94414\n",
            "episode=137, global_step=2954, episode_reward=14.0, epsilon=0.943874\n",
            "episode=138, global_step=2986, episode_reward=32.0, epsilon=0.943266\n",
            "episode=139, global_step=3028, episode_reward=42.0, epsilon=0.942468\n",
            "episode=140, global_step=3045, episode_reward=17.0, epsilon=0.942145\n",
            "episode=141, global_step=3064, episode_reward=19.0, epsilon=0.941784\n",
            "episode=142, global_step=3082, episode_reward=18.0, epsilon=0.941442\n",
            "episode=143, global_step=3097, episode_reward=15.0, epsilon=0.941157\n",
            "episode=144, global_step=3125, episode_reward=28.0, epsilon=0.940625\n",
            "episode=145, global_step=3143, episode_reward=18.0, epsilon=0.940283\n",
            "episode=146, global_step=3156, episode_reward=13.0, epsilon=0.940036\n",
            "episode=147, global_step=3174, episode_reward=18.0, epsilon=0.939694\n",
            "episode=148, global_step=3193, episode_reward=19.0, epsilon=0.939333\n",
            "episode=149, global_step=3211, episode_reward=18.0, epsilon=0.938991\n",
            "episode=150, global_step=3221, episode_reward=10.0, epsilon=0.938801\n",
            "episode=151, global_step=3232, episode_reward=11.0, epsilon=0.938592\n",
            "episode=152, global_step=3259, episode_reward=27.0, epsilon=0.938079\n",
            "episode=153, global_step=3280, episode_reward=21.0, epsilon=0.93768\n",
            "episode=154, global_step=3294, episode_reward=14.0, epsilon=0.937414\n",
            "episode=155, global_step=3307, episode_reward=13.0, epsilon=0.937167\n",
            "episode=156, global_step=3351, episode_reward=44.0, epsilon=0.936331\n",
            "episode=157, global_step=3361, episode_reward=10.0, epsilon=0.936141\n",
            "episode=158, global_step=3378, episode_reward=17.0, epsilon=0.935818\n",
            "episode=159, global_step=3410, episode_reward=32.0, epsilon=0.93521\n",
            "episode=160, global_step=3425, episode_reward=15.0, epsilon=0.934925\n",
            "episode=161, global_step=3438, episode_reward=13.0, epsilon=0.934678\n",
            "episode=162, global_step=3451, episode_reward=13.0, epsilon=0.934431\n",
            "episode=163, global_step=3472, episode_reward=21.0, epsilon=0.934032\n",
            "episode=164, global_step=3509, episode_reward=37.0, epsilon=0.933329\n",
            "episode=165, global_step=3532, episode_reward=23.0, epsilon=0.932892\n",
            "episode=166, global_step=3545, episode_reward=13.0, epsilon=0.932645\n",
            "episode=167, global_step=3557, episode_reward=12.0, epsilon=0.932417\n",
            "episode=168, global_step=3570, episode_reward=13.0, epsilon=0.93217\n",
            "episode=169, global_step=3605, episode_reward=35.0, epsilon=0.931505\n",
            "episode=170, global_step=3635, episode_reward=30.0, epsilon=0.9309350000000001\n",
            "episode=171, global_step=3655, episode_reward=20.0, epsilon=0.930555\n",
            "episode=172, global_step=3669, episode_reward=14.0, epsilon=0.930289\n",
            "episode=173, global_step=3678, episode_reward=9.0, epsilon=0.930118\n",
            "episode=174, global_step=3689, episode_reward=11.0, epsilon=0.929909\n",
            "episode=175, global_step=3712, episode_reward=23.0, epsilon=0.929472\n",
            "episode=176, global_step=3729, episode_reward=17.0, epsilon=0.929149\n",
            "episode=177, global_step=3744, episode_reward=15.0, epsilon=0.928864\n",
            "episode=178, global_step=3778, episode_reward=34.0, epsilon=0.928218\n",
            "episode=179, global_step=3816, episode_reward=38.0, epsilon=0.927496\n",
            "episode=180, global_step=3831, episode_reward=15.0, epsilon=0.927211\n",
            "episode=181, global_step=3862, episode_reward=31.0, epsilon=0.9266220000000001\n",
            "episode=182, global_step=3872, episode_reward=10.0, epsilon=0.926432\n",
            "episode=183, global_step=3883, episode_reward=11.0, epsilon=0.926223\n",
            "episode=184, global_step=3894, episode_reward=11.0, epsilon=0.926014\n",
            "episode=185, global_step=3916, episode_reward=22.0, epsilon=0.925596\n",
            "episode=186, global_step=3936, episode_reward=20.0, epsilon=0.925216\n",
            "episode=187, global_step=3963, episode_reward=27.0, epsilon=0.924703\n",
            "episode=188, global_step=3991, episode_reward=28.0, epsilon=0.924171\n",
            "episode=189, global_step=4008, episode_reward=17.0, epsilon=0.923848\n",
            "episode=190, global_step=4051, episode_reward=43.0, epsilon=0.923031\n",
            "episode=191, global_step=4075, episode_reward=24.0, epsilon=0.922575\n",
            "episode=192, global_step=4094, episode_reward=19.0, epsilon=0.922214\n",
            "episode=193, global_step=4113, episode_reward=19.0, epsilon=0.921853\n",
            "episode=194, global_step=4123, episode_reward=10.0, epsilon=0.921663\n",
            "episode=195, global_step=4133, episode_reward=10.0, epsilon=0.921473\n",
            "episode=196, global_step=4158, episode_reward=25.0, epsilon=0.920998\n",
            "episode=197, global_step=4188, episode_reward=30.0, epsilon=0.920428\n",
            "episode=198, global_step=4198, episode_reward=10.0, epsilon=0.920238\n",
            "episode=199, global_step=4234, episode_reward=36.0, epsilon=0.919554\n",
            "episode=200, global_step=4250, episode_reward=16.0, epsilon=0.91925\n",
            "episode=201, global_step=4282, episode_reward=32.0, epsilon=0.918642\n",
            "episode=202, global_step=4294, episode_reward=12.0, epsilon=0.9184140000000001\n",
            "episode=203, global_step=4307, episode_reward=13.0, epsilon=0.918167\n",
            "episode=204, global_step=4328, episode_reward=21.0, epsilon=0.917768\n",
            "episode=205, global_step=4342, episode_reward=14.0, epsilon=0.917502\n",
            "episode=206, global_step=4366, episode_reward=24.0, epsilon=0.917046\n",
            "episode=207, global_step=4386, episode_reward=20.0, epsilon=0.916666\n",
            "episode=208, global_step=4409, episode_reward=23.0, epsilon=0.916229\n",
            "episode=209, global_step=4439, episode_reward=30.0, epsilon=0.915659\n",
            "episode=210, global_step=4454, episode_reward=15.0, epsilon=0.915374\n",
            "episode=211, global_step=4501, episode_reward=47.0, epsilon=0.914481\n",
            "episode=212, global_step=4514, episode_reward=13.0, epsilon=0.914234\n",
            "episode=213, global_step=4536, episode_reward=22.0, epsilon=0.913816\n",
            "episode=214, global_step=4549, episode_reward=13.0, epsilon=0.913569\n",
            "episode=215, global_step=4574, episode_reward=25.0, epsilon=0.9130940000000001\n",
            "episode=216, global_step=4585, episode_reward=11.0, epsilon=0.9128850000000001\n",
            "episode=217, global_step=4595, episode_reward=10.0, epsilon=0.912695\n",
            "episode=218, global_step=4621, episode_reward=26.0, epsilon=0.912201\n",
            "episode=219, global_step=4636, episode_reward=15.0, epsilon=0.911916\n",
            "episode=220, global_step=4647, episode_reward=11.0, epsilon=0.911707\n",
            "episode=221, global_step=4663, episode_reward=16.0, epsilon=0.911403\n",
            "episode=222, global_step=4709, episode_reward=46.0, epsilon=0.910529\n",
            "episode=223, global_step=4729, episode_reward=20.0, epsilon=0.910149\n",
            "episode=224, global_step=4752, episode_reward=23.0, epsilon=0.909712\n",
            "episode=225, global_step=4774, episode_reward=22.0, epsilon=0.909294\n",
            "episode=226, global_step=4807, episode_reward=33.0, epsilon=0.908667\n",
            "episode=227, global_step=4824, episode_reward=17.0, epsilon=0.908344\n",
            "episode=228, global_step=4843, episode_reward=19.0, epsilon=0.907983\n",
            "episode=229, global_step=4863, episode_reward=20.0, epsilon=0.907603\n",
            "episode=230, global_step=4877, episode_reward=14.0, epsilon=0.9073370000000001\n",
            "episode=231, global_step=4889, episode_reward=12.0, epsilon=0.907109\n",
            "episode=232, global_step=4905, episode_reward=16.0, epsilon=0.906805\n",
            "episode=233, global_step=4916, episode_reward=11.0, epsilon=0.906596\n",
            "episode=234, global_step=4929, episode_reward=13.0, epsilon=0.9063490000000001\n",
            "episode=235, global_step=4952, episode_reward=23.0, epsilon=0.905912\n",
            "episode=236, global_step=4971, episode_reward=19.0, epsilon=0.905551\n",
            "episode=237, global_step=4981, episode_reward=10.0, epsilon=0.905361\n",
            "episode=238, global_step=5025, episode_reward=44.0, epsilon=0.904525\n",
            "episode=239, global_step=5103, episode_reward=78.0, epsilon=0.903043\n",
            "episode=240, global_step=5120, episode_reward=17.0, epsilon=0.90272\n",
            "episode=241, global_step=5154, episode_reward=34.0, epsilon=0.902074\n",
            "episode=242, global_step=5172, episode_reward=18.0, epsilon=0.901732\n",
            "episode=243, global_step=5206, episode_reward=34.0, epsilon=0.901086\n",
            "episode=244, global_step=5228, episode_reward=22.0, epsilon=0.900668\n",
            "episode=245, global_step=5243, episode_reward=15.0, epsilon=0.900383\n",
            "episode=246, global_step=5298, episode_reward=55.0, epsilon=0.899338\n",
            "episode=247, global_step=5346, episode_reward=48.0, epsilon=0.8984260000000001\n",
            "episode=248, global_step=5359, episode_reward=13.0, epsilon=0.8981790000000001\n",
            "episode=249, global_step=5403, episode_reward=44.0, epsilon=0.897343\n",
            "episode=250, global_step=5456, episode_reward=53.0, epsilon=0.896336\n",
            "episode=251, global_step=5471, episode_reward=15.0, epsilon=0.896051\n",
            "episode=252, global_step=5523, episode_reward=52.0, epsilon=0.895063\n",
            "episode=253, global_step=5551, episode_reward=28.0, epsilon=0.894531\n",
            "episode=254, global_step=5569, episode_reward=18.0, epsilon=0.894189\n",
            "episode=255, global_step=5590, episode_reward=21.0, epsilon=0.89379\n",
            "episode=256, global_step=5607, episode_reward=17.0, epsilon=0.893467\n",
            "episode=257, global_step=5620, episode_reward=13.0, epsilon=0.89322\n",
            "episode=258, global_step=5647, episode_reward=27.0, epsilon=0.892707\n",
            "episode=259, global_step=5673, episode_reward=26.0, epsilon=0.892213\n",
            "episode=260, global_step=5686, episode_reward=13.0, epsilon=0.891966\n",
            "episode=261, global_step=5696, episode_reward=10.0, epsilon=0.891776\n",
            "episode=262, global_step=5750, episode_reward=54.0, epsilon=0.89075\n",
            "episode=263, global_step=5796, episode_reward=46.0, epsilon=0.889876\n",
            "episode=264, global_step=5817, episode_reward=21.0, epsilon=0.8894770000000001\n",
            "episode=265, global_step=5827, episode_reward=10.0, epsilon=0.889287\n",
            "episode=266, global_step=5844, episode_reward=17.0, epsilon=0.888964\n",
            "episode=267, global_step=5891, episode_reward=47.0, epsilon=0.888071\n",
            "episode=268, global_step=5903, episode_reward=12.0, epsilon=0.887843\n",
            "episode=269, global_step=5935, episode_reward=32.0, epsilon=0.887235\n",
            "episode=270, global_step=5957, episode_reward=22.0, epsilon=0.886817\n",
            "episode=271, global_step=5984, episode_reward=27.0, epsilon=0.886304\n",
            "episode=272, global_step=5998, episode_reward=14.0, epsilon=0.886038\n",
            "episode=273, global_step=6026, episode_reward=28.0, epsilon=0.885506\n",
            "episode=274, global_step=6040, episode_reward=14.0, epsilon=0.88524\n",
            "episode=275, global_step=6074, episode_reward=34.0, epsilon=0.884594\n",
            "episode=276, global_step=6085, episode_reward=11.0, epsilon=0.884385\n",
            "episode=277, global_step=6098, episode_reward=13.0, epsilon=0.884138\n",
            "episode=278, global_step=6114, episode_reward=16.0, epsilon=0.883834\n",
            "episode=279, global_step=6140, episode_reward=26.0, epsilon=0.88334\n",
            "episode=280, global_step=6156, episode_reward=16.0, epsilon=0.883036\n",
            "episode=281, global_step=6168, episode_reward=12.0, epsilon=0.882808\n",
            "episode=282, global_step=6177, episode_reward=9.0, epsilon=0.882637\n",
            "episode=283, global_step=6188, episode_reward=11.0, epsilon=0.882428\n",
            "episode=284, global_step=6207, episode_reward=19.0, epsilon=0.882067\n",
            "episode=285, global_step=6227, episode_reward=20.0, epsilon=0.881687\n",
            "episode=286, global_step=6250, episode_reward=23.0, epsilon=0.88125\n",
            "episode=287, global_step=6267, episode_reward=17.0, epsilon=0.880927\n",
            "episode=288, global_step=6285, episode_reward=18.0, epsilon=0.8805850000000001\n",
            "episode=289, global_step=6298, episode_reward=13.0, epsilon=0.8803380000000001\n",
            "episode=290, global_step=6341, episode_reward=43.0, epsilon=0.879521\n",
            "episode=291, global_step=6352, episode_reward=11.0, epsilon=0.879312\n",
            "episode=292, global_step=6366, episode_reward=14.0, epsilon=0.879046\n",
            "episode=293, global_step=6380, episode_reward=14.0, epsilon=0.87878\n",
            "episode=294, global_step=6436, episode_reward=56.0, epsilon=0.877716\n",
            "episode=295, global_step=6471, episode_reward=35.0, epsilon=0.877051\n",
            "episode=296, global_step=6488, episode_reward=17.0, epsilon=0.8767280000000001\n",
            "episode=297, global_step=6502, episode_reward=14.0, epsilon=0.8764620000000001\n",
            "episode=298, global_step=6515, episode_reward=13.0, epsilon=0.876215\n",
            "episode=299, global_step=6533, episode_reward=18.0, epsilon=0.875873\n",
            "episode=300, global_step=6547, episode_reward=14.0, epsilon=0.875607\n",
            "episode=301, global_step=6556, episode_reward=9.0, epsilon=0.875436\n",
            "episode=302, global_step=6574, episode_reward=18.0, epsilon=0.875094\n",
            "episode=303, global_step=6585, episode_reward=11.0, epsilon=0.874885\n",
            "episode=304, global_step=6596, episode_reward=11.0, epsilon=0.874676\n",
            "episode=305, global_step=6613, episode_reward=17.0, epsilon=0.874353\n",
            "episode=306, global_step=6647, episode_reward=34.0, epsilon=0.873707\n",
            "episode=307, global_step=6661, episode_reward=14.0, epsilon=0.873441\n",
            "episode=308, global_step=6706, episode_reward=45.0, epsilon=0.8725860000000001\n",
            "episode=309, global_step=6727, episode_reward=21.0, epsilon=0.872187\n",
            "episode=310, global_step=6771, episode_reward=44.0, epsilon=0.871351\n",
            "episode=311, global_step=6782, episode_reward=11.0, epsilon=0.8711420000000001\n",
            "episode=312, global_step=6799, episode_reward=17.0, epsilon=0.870819\n",
            "episode=313, global_step=6812, episode_reward=13.0, epsilon=0.870572\n",
            "episode=314, global_step=6822, episode_reward=10.0, epsilon=0.870382\n",
            "episode=315, global_step=6837, episode_reward=15.0, epsilon=0.870097\n",
            "episode=316, global_step=6856, episode_reward=19.0, epsilon=0.8697360000000001\n",
            "episode=317, global_step=6865, episode_reward=9.0, epsilon=0.869565\n",
            "episode=318, global_step=6879, episode_reward=14.0, epsilon=0.869299\n",
            "episode=319, global_step=6893, episode_reward=14.0, epsilon=0.869033\n",
            "episode=320, global_step=6910, episode_reward=17.0, epsilon=0.86871\n",
            "episode=321, global_step=6924, episode_reward=14.0, epsilon=0.868444\n",
            "episode=322, global_step=6951, episode_reward=27.0, epsilon=0.867931\n",
            "episode=323, global_step=7011, episode_reward=60.0, epsilon=0.866791\n",
            "episode=324, global_step=7022, episode_reward=11.0, epsilon=0.866582\n",
            "episode=325, global_step=7039, episode_reward=17.0, epsilon=0.866259\n",
            "episode=326, global_step=7061, episode_reward=22.0, epsilon=0.8658410000000001\n",
            "episode=327, global_step=7075, episode_reward=14.0, epsilon=0.865575\n",
            "episode=328, global_step=7096, episode_reward=21.0, epsilon=0.8651760000000001\n",
            "episode=329, global_step=7116, episode_reward=20.0, epsilon=0.864796\n",
            "episode=330, global_step=7134, episode_reward=18.0, epsilon=0.8644540000000001\n",
            "episode=331, global_step=7148, episode_reward=14.0, epsilon=0.864188\n",
            "episode=332, global_step=7162, episode_reward=14.0, epsilon=0.8639220000000001\n",
            "episode=333, global_step=7174, episode_reward=12.0, epsilon=0.863694\n",
            "episode=334, global_step=7219, episode_reward=45.0, epsilon=0.862839\n",
            "episode=335, global_step=7237, episode_reward=18.0, epsilon=0.8624970000000001\n",
            "episode=336, global_step=7257, episode_reward=20.0, epsilon=0.862117\n",
            "episode=337, global_step=7269, episode_reward=12.0, epsilon=0.861889\n",
            "episode=338, global_step=7286, episode_reward=17.0, epsilon=0.861566\n",
            "episode=339, global_step=7301, episode_reward=15.0, epsilon=0.861281\n",
            "episode=340, global_step=7317, episode_reward=16.0, epsilon=0.860977\n",
            "episode=341, global_step=7340, episode_reward=23.0, epsilon=0.8605400000000001\n",
            "episode=342, global_step=7365, episode_reward=25.0, epsilon=0.8600650000000001\n",
            "episode=343, global_step=7376, episode_reward=11.0, epsilon=0.859856\n",
            "episode=344, global_step=7398, episode_reward=22.0, epsilon=0.859438\n",
            "episode=345, global_step=7411, episode_reward=13.0, epsilon=0.859191\n",
            "episode=346, global_step=7427, episode_reward=16.0, epsilon=0.858887\n",
            "episode=347, global_step=7453, episode_reward=26.0, epsilon=0.858393\n",
            "episode=348, global_step=7470, episode_reward=17.0, epsilon=0.85807\n",
            "episode=349, global_step=7526, episode_reward=56.0, epsilon=0.857006\n",
            "episode=350, global_step=7560, episode_reward=34.0, epsilon=0.85636\n",
            "episode=351, global_step=7589, episode_reward=29.0, epsilon=0.855809\n",
            "episode=352, global_step=7611, episode_reward=22.0, epsilon=0.855391\n",
            "episode=353, global_step=7622, episode_reward=11.0, epsilon=0.855182\n",
            "episode=354, global_step=7646, episode_reward=24.0, epsilon=0.854726\n",
            "episode=355, global_step=7671, episode_reward=25.0, epsilon=0.854251\n",
            "episode=356, global_step=7696, episode_reward=25.0, epsilon=0.853776\n",
            "episode=357, global_step=7716, episode_reward=20.0, epsilon=0.853396\n",
            "episode=358, global_step=7737, episode_reward=21.0, epsilon=0.852997\n",
            "episode=359, global_step=7760, episode_reward=23.0, epsilon=0.85256\n",
            "episode=360, global_step=7779, episode_reward=19.0, epsilon=0.852199\n",
            "episode=361, global_step=7791, episode_reward=12.0, epsilon=0.851971\n",
            "episode=362, global_step=7816, episode_reward=25.0, epsilon=0.851496\n",
            "episode=363, global_step=7836, episode_reward=20.0, epsilon=0.851116\n",
            "episode=364, global_step=7848, episode_reward=12.0, epsilon=0.850888\n",
            "episode=365, global_step=7863, episode_reward=15.0, epsilon=0.850603\n",
            "episode=366, global_step=7881, episode_reward=18.0, epsilon=0.850261\n",
            "episode=367, global_step=7894, episode_reward=13.0, epsilon=0.850014\n",
            "episode=368, global_step=7908, episode_reward=14.0, epsilon=0.8497480000000001\n",
            "episode=369, global_step=7923, episode_reward=15.0, epsilon=0.8494630000000001\n",
            "episode=370, global_step=7957, episode_reward=34.0, epsilon=0.848817\n",
            "episode=371, global_step=7997, episode_reward=40.0, epsilon=0.8480570000000001\n",
            "episode=372, global_step=8023, episode_reward=26.0, epsilon=0.8475630000000001\n",
            "episode=373, global_step=8039, episode_reward=16.0, epsilon=0.847259\n",
            "episode=374, global_step=8050, episode_reward=11.0, epsilon=0.8470500000000001\n",
            "episode=375, global_step=8062, episode_reward=12.0, epsilon=0.846822\n",
            "episode=376, global_step=8086, episode_reward=24.0, epsilon=0.846366\n",
            "episode=377, global_step=8114, episode_reward=28.0, epsilon=0.845834\n",
            "episode=378, global_step=8128, episode_reward=14.0, epsilon=0.845568\n",
            "episode=379, global_step=8152, episode_reward=24.0, epsilon=0.8451120000000001\n",
            "episode=380, global_step=8214, episode_reward=62.0, epsilon=0.843934\n",
            "episode=381, global_step=8269, episode_reward=55.0, epsilon=0.842889\n",
            "episode=382, global_step=8282, episode_reward=13.0, epsilon=0.842642\n",
            "episode=383, global_step=8303, episode_reward=21.0, epsilon=0.8422430000000001\n",
            "episode=384, global_step=8340, episode_reward=37.0, epsilon=0.84154\n",
            "episode=385, global_step=8366, episode_reward=26.0, epsilon=0.841046\n",
            "episode=386, global_step=8406, episode_reward=40.0, epsilon=0.8402860000000001\n",
            "episode=387, global_step=8418, episode_reward=12.0, epsilon=0.840058\n",
            "episode=388, global_step=8469, episode_reward=51.0, epsilon=0.839089\n",
            "episode=389, global_step=8480, episode_reward=11.0, epsilon=0.8388800000000001\n",
            "episode=390, global_step=8494, episode_reward=14.0, epsilon=0.838614\n",
            "episode=391, global_step=8509, episode_reward=15.0, epsilon=0.838329\n",
            "episode=392, global_step=8519, episode_reward=10.0, epsilon=0.838139\n",
            "episode=393, global_step=8544, episode_reward=25.0, epsilon=0.837664\n",
            "episode=394, global_step=8557, episode_reward=13.0, epsilon=0.8374170000000001\n",
            "episode=395, global_step=8585, episode_reward=28.0, epsilon=0.836885\n",
            "episode=396, global_step=8600, episode_reward=15.0, epsilon=0.8366\n",
            "episode=397, global_step=8615, episode_reward=15.0, epsilon=0.836315\n",
            "episode=398, global_step=8643, episode_reward=28.0, epsilon=0.835783\n",
            "episode=399, global_step=8657, episode_reward=14.0, epsilon=0.8355170000000001\n",
            "episode=400, global_step=8672, episode_reward=15.0, epsilon=0.835232\n",
            "episode=401, global_step=8691, episode_reward=19.0, epsilon=0.834871\n",
            "episode=402, global_step=8700, episode_reward=9.0, epsilon=0.8347\n",
            "episode=403, global_step=8715, episode_reward=15.0, epsilon=0.834415\n",
            "episode=404, global_step=8735, episode_reward=20.0, epsilon=0.8340350000000001\n",
            "episode=405, global_step=8746, episode_reward=11.0, epsilon=0.833826\n",
            "episode=406, global_step=8762, episode_reward=16.0, epsilon=0.833522\n",
            "episode=407, global_step=8780, episode_reward=18.0, epsilon=0.83318\n",
            "episode=408, global_step=8794, episode_reward=14.0, epsilon=0.832914\n",
            "episode=409, global_step=8813, episode_reward=19.0, epsilon=0.832553\n",
            "episode=410, global_step=8829, episode_reward=16.0, epsilon=0.832249\n",
            "episode=411, global_step=8844, episode_reward=15.0, epsilon=0.831964\n",
            "episode=412, global_step=8854, episode_reward=10.0, epsilon=0.831774\n",
            "episode=413, global_step=8870, episode_reward=16.0, epsilon=0.83147\n",
            "episode=414, global_step=8882, episode_reward=12.0, epsilon=0.831242\n",
            "episode=415, global_step=8897, episode_reward=15.0, epsilon=0.8309570000000001\n",
            "episode=416, global_step=8920, episode_reward=23.0, epsilon=0.83052\n",
            "episode=417, global_step=8955, episode_reward=35.0, epsilon=0.829855\n",
            "episode=418, global_step=8981, episode_reward=26.0, epsilon=0.829361\n",
            "episode=419, global_step=8992, episode_reward=11.0, epsilon=0.829152\n",
            "episode=420, global_step=9002, episode_reward=10.0, epsilon=0.828962\n",
            "episode=421, global_step=9018, episode_reward=16.0, epsilon=0.828658\n",
            "episode=422, global_step=9057, episode_reward=39.0, epsilon=0.827917\n",
            "episode=423, global_step=9075, episode_reward=18.0, epsilon=0.8275750000000001\n",
            "episode=424, global_step=9090, episode_reward=15.0, epsilon=0.8272900000000001\n",
            "episode=425, global_step=9105, episode_reward=15.0, epsilon=0.827005\n",
            "episode=426, global_step=9119, episode_reward=14.0, epsilon=0.826739\n",
            "episode=427, global_step=9139, episode_reward=20.0, epsilon=0.8263590000000001\n",
            "episode=428, global_step=9156, episode_reward=17.0, epsilon=0.826036\n",
            "episode=429, global_step=9170, episode_reward=14.0, epsilon=0.82577\n",
            "episode=430, global_step=9185, episode_reward=15.0, epsilon=0.825485\n",
            "episode=431, global_step=9207, episode_reward=22.0, epsilon=0.825067\n",
            "episode=432, global_step=9222, episode_reward=15.0, epsilon=0.824782\n",
            "episode=433, global_step=9254, episode_reward=32.0, epsilon=0.824174\n",
            "episode=434, global_step=9298, episode_reward=44.0, epsilon=0.823338\n",
            "episode=435, global_step=9357, episode_reward=59.0, epsilon=0.822217\n",
            "episode=436, global_step=9387, episode_reward=30.0, epsilon=0.821647\n",
            "episode=437, global_step=9400, episode_reward=13.0, epsilon=0.8214\n",
            "episode=438, global_step=9410, episode_reward=10.0, epsilon=0.82121\n",
            "episode=439, global_step=9422, episode_reward=12.0, epsilon=0.820982\n",
            "episode=440, global_step=9447, episode_reward=25.0, epsilon=0.820507\n",
            "episode=441, global_step=9459, episode_reward=12.0, epsilon=0.820279\n",
            "episode=442, global_step=9483, episode_reward=24.0, epsilon=0.819823\n",
            "episode=443, global_step=9494, episode_reward=11.0, epsilon=0.8196140000000001\n",
            "episode=444, global_step=9506, episode_reward=12.0, epsilon=0.8193860000000001\n",
            "episode=445, global_step=9537, episode_reward=31.0, epsilon=0.818797\n",
            "episode=446, global_step=9546, episode_reward=9.0, epsilon=0.8186260000000001\n",
            "episode=447, global_step=9557, episode_reward=11.0, epsilon=0.8184170000000001\n",
            "episode=448, global_step=9573, episode_reward=16.0, epsilon=0.8181130000000001\n",
            "episode=449, global_step=9585, episode_reward=12.0, epsilon=0.817885\n",
            "episode=450, global_step=9600, episode_reward=15.0, epsilon=0.8176\n",
            "episode=451, global_step=9621, episode_reward=21.0, epsilon=0.8172010000000001\n",
            "episode=452, global_step=9650, episode_reward=29.0, epsilon=0.81665\n",
            "episode=453, global_step=9660, episode_reward=10.0, epsilon=0.81646\n",
            "episode=454, global_step=9693, episode_reward=33.0, epsilon=0.815833\n",
            "episode=455, global_step=9712, episode_reward=19.0, epsilon=0.815472\n",
            "episode=456, global_step=9750, episode_reward=38.0, epsilon=0.8147500000000001\n",
            "episode=457, global_step=9770, episode_reward=20.0, epsilon=0.81437\n",
            "episode=458, global_step=9780, episode_reward=10.0, epsilon=0.81418\n",
            "episode=459, global_step=9829, episode_reward=49.0, epsilon=0.813249\n",
            "episode=460, global_step=9848, episode_reward=19.0, epsilon=0.812888\n",
            "episode=461, global_step=9898, episode_reward=50.0, epsilon=0.811938\n",
            "episode=462, global_step=9914, episode_reward=16.0, epsilon=0.811634\n",
            "episode=463, global_step=9932, episode_reward=18.0, epsilon=0.811292\n",
            "episode=464, global_step=9949, episode_reward=17.0, epsilon=0.810969\n",
            "episode=465, global_step=9972, episode_reward=23.0, epsilon=0.810532\n",
            "episode=466, global_step=9990, episode_reward=18.0, epsilon=0.81019\n",
            "episode=467, global_step=10012, episode_reward=22.0, epsilon=0.809772\n",
            "episode=468, global_step=10033, episode_reward=21.0, epsilon=0.809373\n",
            "episode=469, global_step=10045, episode_reward=12.0, epsilon=0.809145\n",
            "episode=470, global_step=10066, episode_reward=21.0, epsilon=0.808746\n",
            "episode=471, global_step=10087, episode_reward=21.0, epsilon=0.808347\n",
            "episode=472, global_step=10103, episode_reward=16.0, epsilon=0.8080430000000001\n",
            "episode=473, global_step=10115, episode_reward=12.0, epsilon=0.8078150000000001\n",
            "episode=474, global_step=10133, episode_reward=18.0, epsilon=0.807473\n",
            "episode=475, global_step=10145, episode_reward=12.0, epsilon=0.807245\n",
            "episode=476, global_step=10173, episode_reward=28.0, epsilon=0.806713\n",
            "episode=477, global_step=10193, episode_reward=20.0, epsilon=0.806333\n",
            "episode=478, global_step=10210, episode_reward=17.0, epsilon=0.80601\n",
            "episode=479, global_step=10224, episode_reward=14.0, epsilon=0.805744\n",
            "episode=480, global_step=10240, episode_reward=16.0, epsilon=0.80544\n",
            "episode=481, global_step=10267, episode_reward=27.0, epsilon=0.8049270000000001\n",
            "episode=482, global_step=10292, episode_reward=25.0, epsilon=0.8044520000000001\n",
            "episode=483, global_step=10323, episode_reward=31.0, epsilon=0.803863\n",
            "episode=484, global_step=10333, episode_reward=10.0, epsilon=0.8036730000000001\n",
            "episode=485, global_step=10350, episode_reward=17.0, epsilon=0.80335\n",
            "episode=486, global_step=10375, episode_reward=25.0, epsilon=0.802875\n",
            "episode=487, global_step=10385, episode_reward=10.0, epsilon=0.8026850000000001\n",
            "episode=488, global_step=10404, episode_reward=19.0, epsilon=0.802324\n",
            "episode=489, global_step=10415, episode_reward=11.0, epsilon=0.802115\n",
            "episode=490, global_step=10424, episode_reward=9.0, epsilon=0.801944\n",
            "episode=491, global_step=10441, episode_reward=17.0, epsilon=0.801621\n",
            "episode=492, global_step=10471, episode_reward=30.0, epsilon=0.801051\n",
            "episode=493, global_step=10496, episode_reward=25.0, epsilon=0.8005760000000001\n",
            "episode=494, global_step=10509, episode_reward=13.0, epsilon=0.8003290000000001\n",
            "episode=495, global_step=10540, episode_reward=31.0, epsilon=0.79974\n",
            "episode=496, global_step=10558, episode_reward=18.0, epsilon=0.799398\n",
            "episode=497, global_step=10578, episode_reward=20.0, epsilon=0.799018\n",
            "episode=498, global_step=10629, episode_reward=51.0, epsilon=0.798049\n",
            "episode=499, global_step=10642, episode_reward=13.0, epsilon=0.797802\n",
            "episode=500, global_step=10684, episode_reward=42.0, epsilon=0.797004\n",
            "episode=501, global_step=10693, episode_reward=9.0, epsilon=0.796833\n",
            "episode=502, global_step=10702, episode_reward=9.0, epsilon=0.796662\n",
            "episode=503, global_step=10721, episode_reward=19.0, epsilon=0.796301\n",
            "episode=504, global_step=10771, episode_reward=50.0, epsilon=0.795351\n",
            "episode=505, global_step=10805, episode_reward=34.0, epsilon=0.794705\n",
            "episode=506, global_step=10818, episode_reward=13.0, epsilon=0.794458\n",
            "episode=507, global_step=10828, episode_reward=10.0, epsilon=0.794268\n",
            "episode=508, global_step=10846, episode_reward=18.0, epsilon=0.793926\n",
            "episode=509, global_step=10861, episode_reward=15.0, epsilon=0.793641\n",
            "episode=510, global_step=10881, episode_reward=20.0, epsilon=0.793261\n",
            "episode=511, global_step=10897, episode_reward=16.0, epsilon=0.792957\n",
            "episode=512, global_step=10916, episode_reward=19.0, epsilon=0.7925960000000001\n",
            "episode=513, global_step=10925, episode_reward=9.0, epsilon=0.792425\n",
            "episode=514, global_step=10936, episode_reward=11.0, epsilon=0.792216\n",
            "episode=515, global_step=10947, episode_reward=11.0, epsilon=0.792007\n",
            "episode=516, global_step=10963, episode_reward=16.0, epsilon=0.791703\n",
            "episode=517, global_step=10975, episode_reward=12.0, epsilon=0.791475\n",
            "episode=518, global_step=11000, episode_reward=25.0, epsilon=0.791\n",
            "episode=519, global_step=11011, episode_reward=11.0, epsilon=0.790791\n",
            "episode=520, global_step=11029, episode_reward=18.0, epsilon=0.790449\n",
            "episode=521, global_step=11042, episode_reward=13.0, epsilon=0.7902020000000001\n",
            "episode=522, global_step=11057, episode_reward=15.0, epsilon=0.789917\n",
            "episode=523, global_step=11073, episode_reward=16.0, epsilon=0.789613\n",
            "episode=524, global_step=11089, episode_reward=16.0, epsilon=0.789309\n",
            "episode=525, global_step=11107, episode_reward=18.0, epsilon=0.788967\n",
            "episode=526, global_step=11125, episode_reward=18.0, epsilon=0.788625\n",
            "episode=527, global_step=11144, episode_reward=19.0, epsilon=0.7882640000000001\n",
            "episode=528, global_step=11161, episode_reward=17.0, epsilon=0.787941\n",
            "episode=529, global_step=11178, episode_reward=17.0, epsilon=0.787618\n",
            "episode=530, global_step=11203, episode_reward=25.0, epsilon=0.787143\n",
            "episode=531, global_step=11243, episode_reward=40.0, epsilon=0.786383\n",
            "episode=532, global_step=11306, episode_reward=63.0, epsilon=0.785186\n",
            "episode=533, global_step=11327, episode_reward=21.0, epsilon=0.784787\n",
            "episode=534, global_step=11347, episode_reward=20.0, epsilon=0.7844070000000001\n",
            "episode=535, global_step=11363, episode_reward=16.0, epsilon=0.784103\n",
            "episode=536, global_step=11399, episode_reward=36.0, epsilon=0.7834190000000001\n",
            "episode=537, global_step=11424, episode_reward=25.0, epsilon=0.7829440000000001\n",
            "episode=538, global_step=11438, episode_reward=14.0, epsilon=0.782678\n",
            "episode=539, global_step=11465, episode_reward=27.0, epsilon=0.782165\n",
            "episode=540, global_step=11493, episode_reward=28.0, epsilon=0.781633\n",
            "episode=541, global_step=11512, episode_reward=19.0, epsilon=0.781272\n",
            "episode=542, global_step=11531, episode_reward=19.0, epsilon=0.780911\n",
            "episode=543, global_step=11554, episode_reward=23.0, epsilon=0.780474\n",
            "episode=544, global_step=11567, episode_reward=13.0, epsilon=0.780227\n",
            "episode=545, global_step=11579, episode_reward=12.0, epsilon=0.779999\n",
            "episode=546, global_step=11596, episode_reward=17.0, epsilon=0.779676\n",
            "episode=547, global_step=11616, episode_reward=20.0, epsilon=0.779296\n",
            "episode=548, global_step=11633, episode_reward=17.0, epsilon=0.778973\n",
            "episode=549, global_step=11654, episode_reward=21.0, epsilon=0.778574\n",
            "episode=550, global_step=11672, episode_reward=18.0, epsilon=0.778232\n",
            "episode=551, global_step=11688, episode_reward=16.0, epsilon=0.7779280000000001\n",
            "episode=552, global_step=11698, episode_reward=10.0, epsilon=0.777738\n",
            "episode=553, global_step=11714, episode_reward=16.0, epsilon=0.777434\n",
            "episode=554, global_step=11726, episode_reward=12.0, epsilon=0.7772060000000001\n",
            "episode=555, global_step=11762, episode_reward=36.0, epsilon=0.776522\n",
            "episode=556, global_step=11777, episode_reward=15.0, epsilon=0.7762370000000001\n",
            "episode=557, global_step=11790, episode_reward=13.0, epsilon=0.7759900000000001\n",
            "episode=558, global_step=11804, episode_reward=14.0, epsilon=0.7757240000000001\n",
            "episode=559, global_step=11816, episode_reward=12.0, epsilon=0.775496\n",
            "episode=560, global_step=11829, episode_reward=13.0, epsilon=0.7752490000000001\n",
            "episode=561, global_step=11843, episode_reward=14.0, epsilon=0.774983\n",
            "episode=562, global_step=11873, episode_reward=30.0, epsilon=0.774413\n",
            "episode=563, global_step=11888, episode_reward=15.0, epsilon=0.774128\n",
            "episode=564, global_step=11898, episode_reward=10.0, epsilon=0.773938\n",
            "episode=565, global_step=11928, episode_reward=30.0, epsilon=0.773368\n",
            "episode=566, global_step=11945, episode_reward=17.0, epsilon=0.773045\n",
            "episode=567, global_step=11959, episode_reward=14.0, epsilon=0.772779\n",
            "episode=568, global_step=11987, episode_reward=28.0, epsilon=0.772247\n",
            "episode=569, global_step=12026, episode_reward=39.0, epsilon=0.771506\n",
            "episode=570, global_step=12035, episode_reward=9.0, epsilon=0.771335\n",
            "episode=571, global_step=12052, episode_reward=17.0, epsilon=0.771012\n",
            "episode=572, global_step=12070, episode_reward=18.0, epsilon=0.77067\n",
            "episode=573, global_step=12087, episode_reward=17.0, epsilon=0.770347\n",
            "episode=574, global_step=12099, episode_reward=12.0, epsilon=0.770119\n",
            "episode=575, global_step=12109, episode_reward=10.0, epsilon=0.7699290000000001\n",
            "episode=576, global_step=12144, episode_reward=35.0, epsilon=0.7692640000000001\n",
            "episode=577, global_step=12157, episode_reward=13.0, epsilon=0.7690170000000001\n",
            "episode=578, global_step=12230, episode_reward=73.0, epsilon=0.76763\n",
            "episode=579, global_step=12241, episode_reward=11.0, epsilon=0.767421\n",
            "episode=580, global_step=12251, episode_reward=10.0, epsilon=0.767231\n",
            "episode=581, global_step=12266, episode_reward=15.0, epsilon=0.766946\n",
            "episode=582, global_step=12280, episode_reward=14.0, epsilon=0.76668\n",
            "episode=583, global_step=12291, episode_reward=11.0, epsilon=0.766471\n",
            "episode=584, global_step=12330, episode_reward=39.0, epsilon=0.76573\n",
            "episode=585, global_step=12347, episode_reward=17.0, epsilon=0.7654070000000001\n",
            "episode=586, global_step=12368, episode_reward=21.0, epsilon=0.765008\n",
            "episode=587, global_step=12385, episode_reward=17.0, epsilon=0.7646850000000001\n",
            "episode=588, global_step=12399, episode_reward=14.0, epsilon=0.7644190000000001\n",
            "episode=589, global_step=12412, episode_reward=13.0, epsilon=0.7641720000000001\n",
            "episode=590, global_step=12424, episode_reward=12.0, epsilon=0.7639440000000001\n",
            "episode=591, global_step=12434, episode_reward=10.0, epsilon=0.763754\n",
            "episode=592, global_step=12475, episode_reward=41.0, epsilon=0.7629750000000001\n",
            "episode=593, global_step=12488, episode_reward=13.0, epsilon=0.7627280000000001\n",
            "episode=594, global_step=12504, episode_reward=16.0, epsilon=0.762424\n",
            "episode=595, global_step=12530, episode_reward=26.0, epsilon=0.76193\n",
            "episode=596, global_step=12540, episode_reward=10.0, epsilon=0.7617400000000001\n",
            "episode=597, global_step=12554, episode_reward=14.0, epsilon=0.761474\n",
            "episode=598, global_step=12566, episode_reward=12.0, epsilon=0.7612460000000001\n",
            "episode=599, global_step=12593, episode_reward=27.0, epsilon=0.760733\n",
            "episode=600, global_step=12603, episode_reward=10.0, epsilon=0.760543\n",
            "episode=601, global_step=12621, episode_reward=18.0, epsilon=0.760201\n",
            "episode=602, global_step=12641, episode_reward=20.0, epsilon=0.7598210000000001\n",
            "episode=603, global_step=12655, episode_reward=14.0, epsilon=0.759555\n",
            "episode=604, global_step=12672, episode_reward=17.0, epsilon=0.759232\n",
            "episode=605, global_step=12700, episode_reward=28.0, epsilon=0.7587\n",
            "episode=606, global_step=12711, episode_reward=11.0, epsilon=0.758491\n",
            "episode=607, global_step=12723, episode_reward=12.0, epsilon=0.758263\n",
            "episode=608, global_step=12735, episode_reward=12.0, epsilon=0.758035\n",
            "episode=609, global_step=12749, episode_reward=14.0, epsilon=0.757769\n",
            "episode=610, global_step=12779, episode_reward=30.0, epsilon=0.7571990000000001\n",
            "episode=611, global_step=12790, episode_reward=11.0, epsilon=0.75699\n",
            "episode=612, global_step=12805, episode_reward=15.0, epsilon=0.756705\n",
            "episode=613, global_step=12816, episode_reward=11.0, epsilon=0.7564960000000001\n",
            "episode=614, global_step=12826, episode_reward=10.0, epsilon=0.756306\n",
            "episode=615, global_step=12836, episode_reward=10.0, epsilon=0.756116\n",
            "episode=616, global_step=12850, episode_reward=14.0, epsilon=0.75585\n",
            "episode=617, global_step=12870, episode_reward=20.0, epsilon=0.7554700000000001\n",
            "episode=618, global_step=12903, episode_reward=33.0, epsilon=0.754843\n",
            "episode=619, global_step=12922, episode_reward=19.0, epsilon=0.7544820000000001\n",
            "episode=620, global_step=12960, episode_reward=38.0, epsilon=0.75376\n",
            "episode=621, global_step=12970, episode_reward=10.0, epsilon=0.7535700000000001\n",
            "episode=622, global_step=12990, episode_reward=20.0, epsilon=0.75319\n",
            "episode=623, global_step=13006, episode_reward=16.0, epsilon=0.752886\n",
            "episode=624, global_step=13018, episode_reward=12.0, epsilon=0.752658\n",
            "episode=625, global_step=13058, episode_reward=40.0, epsilon=0.7518980000000001\n",
            "episode=626, global_step=13069, episode_reward=11.0, epsilon=0.751689\n",
            "episode=627, global_step=13088, episode_reward=19.0, epsilon=0.751328\n",
            "episode=628, global_step=13115, episode_reward=27.0, epsilon=0.750815\n",
            "episode=629, global_step=13130, episode_reward=15.0, epsilon=0.75053\n",
            "episode=630, global_step=13143, episode_reward=13.0, epsilon=0.750283\n",
            "episode=631, global_step=13156, episode_reward=13.0, epsilon=0.750036\n",
            "episode=632, global_step=13166, episode_reward=10.0, epsilon=0.749846\n",
            "episode=633, global_step=13185, episode_reward=19.0, epsilon=0.749485\n",
            "episode=634, global_step=13208, episode_reward=23.0, epsilon=0.749048\n",
            "episode=635, global_step=13259, episode_reward=51.0, epsilon=0.748079\n",
            "episode=636, global_step=13272, episode_reward=13.0, epsilon=0.747832\n",
            "episode=637, global_step=13288, episode_reward=16.0, epsilon=0.747528\n",
            "episode=638, global_step=13316, episode_reward=28.0, epsilon=0.746996\n",
            "episode=639, global_step=13337, episode_reward=21.0, epsilon=0.746597\n",
            "episode=640, global_step=13349, episode_reward=12.0, epsilon=0.7463690000000001\n",
            "episode=641, global_step=13368, episode_reward=19.0, epsilon=0.746008\n",
            "episode=642, global_step=13387, episode_reward=19.0, epsilon=0.745647\n",
            "episode=643, global_step=13406, episode_reward=19.0, epsilon=0.745286\n",
            "episode=644, global_step=13417, episode_reward=11.0, epsilon=0.745077\n",
            "episode=645, global_step=13463, episode_reward=46.0, epsilon=0.7442030000000001\n",
            "episode=646, global_step=13474, episode_reward=11.0, epsilon=0.743994\n",
            "episode=647, global_step=13490, episode_reward=16.0, epsilon=0.74369\n",
            "episode=648, global_step=13499, episode_reward=9.0, epsilon=0.743519\n",
            "episode=649, global_step=13511, episode_reward=12.0, epsilon=0.743291\n",
            "episode=650, global_step=13529, episode_reward=18.0, epsilon=0.7429490000000001\n",
            "episode=651, global_step=13566, episode_reward=37.0, epsilon=0.742246\n",
            "episode=652, global_step=13582, episode_reward=16.0, epsilon=0.7419420000000001\n",
            "episode=653, global_step=13593, episode_reward=11.0, epsilon=0.741733\n",
            "episode=654, global_step=13612, episode_reward=19.0, epsilon=0.741372\n",
            "episode=655, global_step=13620, episode_reward=8.0, epsilon=0.74122\n",
            "episode=656, global_step=13629, episode_reward=9.0, epsilon=0.7410490000000001\n",
            "episode=657, global_step=13642, episode_reward=13.0, epsilon=0.740802\n",
            "episode=658, global_step=13652, episode_reward=10.0, epsilon=0.740612\n",
            "episode=659, global_step=13663, episode_reward=11.0, epsilon=0.740403\n",
            "episode=660, global_step=13674, episode_reward=11.0, epsilon=0.740194\n",
            "episode=661, global_step=13684, episode_reward=10.0, epsilon=0.7400040000000001\n",
            "episode=662, global_step=13707, episode_reward=23.0, epsilon=0.7395670000000001\n",
            "episode=663, global_step=13735, episode_reward=28.0, epsilon=0.7390350000000001\n",
            "episode=664, global_step=13749, episode_reward=14.0, epsilon=0.738769\n",
            "episode=665, global_step=13771, episode_reward=22.0, epsilon=0.738351\n",
            "episode=666, global_step=13805, episode_reward=34.0, epsilon=0.737705\n",
            "episode=667, global_step=13818, episode_reward=13.0, epsilon=0.7374580000000001\n",
            "episode=668, global_step=13836, episode_reward=18.0, epsilon=0.7371160000000001\n",
            "episode=669, global_step=13848, episode_reward=12.0, epsilon=0.736888\n",
            "episode=670, global_step=13859, episode_reward=11.0, epsilon=0.7366790000000001\n",
            "episode=671, global_step=13869, episode_reward=10.0, epsilon=0.736489\n",
            "episode=672, global_step=13906, episode_reward=37.0, epsilon=0.735786\n",
            "episode=673, global_step=13956, episode_reward=50.0, epsilon=0.734836\n",
            "episode=674, global_step=13971, episode_reward=15.0, epsilon=0.734551\n",
            "episode=675, global_step=13984, episode_reward=13.0, epsilon=0.7343040000000001\n",
            "episode=676, global_step=14006, episode_reward=22.0, epsilon=0.733886\n",
            "episode=677, global_step=14023, episode_reward=17.0, epsilon=0.733563\n",
            "episode=678, global_step=14042, episode_reward=19.0, epsilon=0.733202\n",
            "episode=679, global_step=14061, episode_reward=19.0, epsilon=0.7328410000000001\n",
            "episode=680, global_step=14087, episode_reward=26.0, epsilon=0.7323470000000001\n",
            "episode=681, global_step=14105, episode_reward=18.0, epsilon=0.732005\n",
            "episode=682, global_step=14125, episode_reward=20.0, epsilon=0.731625\n",
            "episode=683, global_step=14137, episode_reward=12.0, epsilon=0.7313970000000001\n",
            "episode=684, global_step=14147, episode_reward=10.0, epsilon=0.731207\n",
            "episode=685, global_step=14160, episode_reward=13.0, epsilon=0.73096\n",
            "episode=686, global_step=14185, episode_reward=25.0, epsilon=0.730485\n",
            "episode=687, global_step=14200, episode_reward=15.0, epsilon=0.7302\n",
            "episode=688, global_step=14226, episode_reward=26.0, epsilon=0.729706\n",
            "episode=689, global_step=14267, episode_reward=41.0, epsilon=0.7289270000000001\n",
            "episode=690, global_step=14284, episode_reward=17.0, epsilon=0.728604\n",
            "episode=691, global_step=14296, episode_reward=12.0, epsilon=0.728376\n",
            "episode=692, global_step=14310, episode_reward=14.0, epsilon=0.72811\n",
            "episode=693, global_step=14322, episode_reward=12.0, epsilon=0.727882\n",
            "episode=694, global_step=14343, episode_reward=21.0, epsilon=0.7274830000000001\n",
            "episode=695, global_step=14374, episode_reward=31.0, epsilon=0.726894\n",
            "episode=696, global_step=14387, episode_reward=13.0, epsilon=0.726647\n",
            "episode=697, global_step=14398, episode_reward=11.0, epsilon=0.726438\n",
            "episode=698, global_step=14414, episode_reward=16.0, epsilon=0.7261340000000001\n",
            "episode=699, global_step=14426, episode_reward=12.0, epsilon=0.725906\n",
            "episode=700, global_step=14450, episode_reward=24.0, epsilon=0.72545\n",
            "episode=701, global_step=14473, episode_reward=23.0, epsilon=0.725013\n",
            "episode=702, global_step=14488, episode_reward=15.0, epsilon=0.724728\n",
            "episode=703, global_step=14516, episode_reward=28.0, epsilon=0.7241960000000001\n",
            "episode=704, global_step=14531, episode_reward=15.0, epsilon=0.723911\n",
            "episode=705, global_step=14546, episode_reward=15.0, epsilon=0.7236260000000001\n",
            "episode=706, global_step=14643, episode_reward=97.0, epsilon=0.7217830000000001\n",
            "episode=707, global_step=14666, episode_reward=23.0, epsilon=0.721346\n",
            "episode=708, global_step=14688, episode_reward=22.0, epsilon=0.720928\n",
            "episode=709, global_step=14700, episode_reward=12.0, epsilon=0.7207\n",
            "episode=710, global_step=14720, episode_reward=20.0, epsilon=0.7203200000000001\n",
            "episode=711, global_step=14742, episode_reward=22.0, epsilon=0.719902\n",
            "episode=712, global_step=14761, episode_reward=19.0, epsilon=0.719541\n",
            "episode=713, global_step=14783, episode_reward=22.0, epsilon=0.719123\n",
            "episode=714, global_step=14826, episode_reward=43.0, epsilon=0.7183060000000001\n",
            "episode=715, global_step=14841, episode_reward=15.0, epsilon=0.718021\n",
            "episode=716, global_step=14863, episode_reward=22.0, epsilon=0.717603\n",
            "episode=717, global_step=14889, episode_reward=26.0, epsilon=0.717109\n",
            "episode=718, global_step=14905, episode_reward=16.0, epsilon=0.716805\n",
            "episode=719, global_step=14927, episode_reward=22.0, epsilon=0.7163870000000001\n",
            "episode=720, global_step=14949, episode_reward=22.0, epsilon=0.7159690000000001\n",
            "episode=721, global_step=14966, episode_reward=17.0, epsilon=0.715646\n",
            "episode=722, global_step=14981, episode_reward=15.0, epsilon=0.715361\n",
            "episode=723, global_step=15037, episode_reward=56.0, epsilon=0.714297\n",
            "episode=724, global_step=15063, episode_reward=26.0, epsilon=0.713803\n",
            "episode=725, global_step=15103, episode_reward=40.0, epsilon=0.7130430000000001\n",
            "episode=726, global_step=15116, episode_reward=13.0, epsilon=0.712796\n",
            "episode=727, global_step=15203, episode_reward=87.0, epsilon=0.7111430000000001\n",
            "episode=728, global_step=15223, episode_reward=20.0, epsilon=0.710763\n",
            "episode=729, global_step=15252, episode_reward=29.0, epsilon=0.7102120000000001\n",
            "episode=730, global_step=15280, episode_reward=28.0, epsilon=0.7096800000000001\n",
            "episode=731, global_step=15306, episode_reward=26.0, epsilon=0.7091860000000001\n",
            "episode=732, global_step=15363, episode_reward=57.0, epsilon=0.708103\n",
            "episode=733, global_step=15465, episode_reward=102.0, epsilon=0.706165\n",
            "episode=734, global_step=15495, episode_reward=30.0, epsilon=0.705595\n",
            "episode=735, global_step=15549, episode_reward=54.0, epsilon=0.704569\n",
            "episode=736, global_step=15657, episode_reward=108.0, epsilon=0.7025170000000001\n",
            "episode=737, global_step=15691, episode_reward=34.0, epsilon=0.701871\n",
            "episode=738, global_step=15712, episode_reward=21.0, epsilon=0.7014720000000001\n",
            "episode=739, global_step=15739, episode_reward=27.0, epsilon=0.7009590000000001\n",
            "episode=740, global_step=15753, episode_reward=14.0, epsilon=0.700693\n",
            "episode=741, global_step=15775, episode_reward=22.0, epsilon=0.700275\n",
            "episode=742, global_step=15818, episode_reward=43.0, epsilon=0.699458\n",
            "episode=743, global_step=15833, episode_reward=15.0, epsilon=0.699173\n",
            "episode=744, global_step=15867, episode_reward=34.0, epsilon=0.6985270000000001\n",
            "episode=745, global_step=15903, episode_reward=36.0, epsilon=0.697843\n",
            "episode=746, global_step=15964, episode_reward=61.0, epsilon=0.6966840000000001\n",
            "episode=747, global_step=15987, episode_reward=23.0, epsilon=0.6962470000000001\n",
            "episode=748, global_step=16016, episode_reward=29.0, epsilon=0.6956960000000001\n",
            "episode=749, global_step=16060, episode_reward=44.0, epsilon=0.69486\n",
            "episode=750, global_step=16099, episode_reward=39.0, epsilon=0.694119\n",
            "episode=751, global_step=16120, episode_reward=21.0, epsilon=0.6937200000000001\n",
            "episode=752, global_step=16146, episode_reward=26.0, epsilon=0.6932260000000001\n",
            "episode=753, global_step=16172, episode_reward=26.0, epsilon=0.692732\n",
            "episode=754, global_step=16186, episode_reward=14.0, epsilon=0.692466\n",
            "episode=755, global_step=16300, episode_reward=114.0, epsilon=0.6903\n",
            "episode=756, global_step=16330, episode_reward=30.0, epsilon=0.6897300000000001\n",
            "episode=757, global_step=16410, episode_reward=80.0, epsilon=0.68821\n",
            "episode=758, global_step=16424, episode_reward=14.0, epsilon=0.6879440000000001\n",
            "episode=759, global_step=16460, episode_reward=36.0, epsilon=0.68726\n",
            "episode=760, global_step=16501, episode_reward=41.0, epsilon=0.6864810000000001\n",
            "episode=761, global_step=16529, episode_reward=28.0, epsilon=0.685949\n",
            "episode=762, global_step=16574, episode_reward=45.0, epsilon=0.6850940000000001\n",
            "episode=763, global_step=16718, episode_reward=144.0, epsilon=0.682358\n",
            "episode=764, global_step=16781, episode_reward=63.0, epsilon=0.681161\n",
            "episode=765, global_step=16854, episode_reward=73.0, epsilon=0.6797740000000001\n",
            "episode=766, global_step=16887, episode_reward=33.0, epsilon=0.6791470000000001\n",
            "episode=767, global_step=16941, episode_reward=54.0, epsilon=0.678121\n",
            "episode=768, global_step=17053, episode_reward=112.0, epsilon=0.6759930000000001\n",
            "episode=769, global_step=17174, episode_reward=121.0, epsilon=0.673694\n",
            "episode=770, global_step=17206, episode_reward=32.0, epsilon=0.6730860000000001\n",
            "episode=771, global_step=17307, episode_reward=101.0, epsilon=0.6711670000000001\n",
            "episode=772, global_step=17408, episode_reward=101.0, epsilon=0.6692480000000001\n",
            "episode=773, global_step=17443, episode_reward=35.0, epsilon=0.668583\n",
            "episode=774, global_step=17529, episode_reward=86.0, epsilon=0.666949\n",
            "episode=775, global_step=17550, episode_reward=21.0, epsilon=0.66655\n",
            "episode=776, global_step=17641, episode_reward=91.0, epsilon=0.6648210000000001\n",
            "episode=777, global_step=17670, episode_reward=29.0, epsilon=0.66427\n",
            "episode=778, global_step=17832, episode_reward=162.0, epsilon=0.661192\n",
            "episode=779, global_step=17845, episode_reward=13.0, epsilon=0.6609450000000001\n",
            "episode=780, global_step=17870, episode_reward=25.0, epsilon=0.6604700000000001\n",
            "episode=781, global_step=17976, episode_reward=106.0, epsilon=0.658456\n",
            "episode=782, global_step=18000, episode_reward=24.0, epsilon=0.658\n",
            "episode=783, global_step=18141, episode_reward=141.0, epsilon=0.655321\n",
            "episode=784, global_step=18262, episode_reward=121.0, epsilon=0.653022\n",
            "episode=785, global_step=18326, episode_reward=64.0, epsilon=0.6518060000000001\n",
            "episode=786, global_step=18392, episode_reward=66.0, epsilon=0.650552\n",
            "episode=787, global_step=18536, episode_reward=144.0, epsilon=0.6478160000000001\n",
            "episode=788, global_step=18670, episode_reward=134.0, epsilon=0.64527\n",
            "episode=789, global_step=18725, episode_reward=55.0, epsilon=0.644225\n",
            "episode=790, global_step=18791, episode_reward=66.0, epsilon=0.6429710000000001\n",
            "episode=791, global_step=18833, episode_reward=42.0, epsilon=0.6421730000000001\n",
            "episode=792, global_step=18950, episode_reward=117.0, epsilon=0.63995\n",
            "episode=793, global_step=18961, episode_reward=11.0, epsilon=0.6397410000000001\n",
            "episode=794, global_step=18973, episode_reward=12.0, epsilon=0.639513\n",
            "episode=795, global_step=19001, episode_reward=28.0, epsilon=0.638981\n",
            "episode=796, global_step=19041, episode_reward=40.0, epsilon=0.638221\n",
            "episode=797, global_step=19114, episode_reward=73.0, epsilon=0.6368340000000001\n",
            "episode=798, global_step=19330, episode_reward=216.0, epsilon=0.63273\n",
            "episode=799, global_step=19404, episode_reward=74.0, epsilon=0.631324\n",
            "episode=800, global_step=19518, episode_reward=114.0, epsilon=0.6291580000000001\n",
            "episode=801, global_step=19546, episode_reward=28.0, epsilon=0.628626\n",
            "episode=802, global_step=19561, episode_reward=15.0, epsilon=0.628341\n",
            "episode=803, global_step=19616, episode_reward=55.0, epsilon=0.6272960000000001\n",
            "episode=804, global_step=19659, episode_reward=43.0, epsilon=0.626479\n",
            "episode=805, global_step=19687, episode_reward=28.0, epsilon=0.625947\n",
            "episode=806, global_step=19713, episode_reward=26.0, epsilon=0.625453\n",
            "episode=807, global_step=19747, episode_reward=34.0, epsilon=0.6248070000000001\n",
            "episode=808, global_step=19775, episode_reward=28.0, epsilon=0.624275\n",
            "episode=809, global_step=19788, episode_reward=13.0, epsilon=0.624028\n",
            "episode=810, global_step=19843, episode_reward=55.0, epsilon=0.6229830000000001\n",
            "episode=811, global_step=19957, episode_reward=114.0, epsilon=0.6208170000000001\n",
            "episode=812, global_step=19991, episode_reward=34.0, epsilon=0.620171\n",
            "episode=813, global_step=20228, episode_reward=237.0, epsilon=0.6156680000000001\n",
            "episode=814, global_step=20286, episode_reward=58.0, epsilon=0.6145660000000001\n",
            "episode=815, global_step=20476, episode_reward=190.0, epsilon=0.610956\n",
            "episode=816, global_step=20511, episode_reward=35.0, epsilon=0.610291\n",
            "episode=817, global_step=20530, episode_reward=19.0, epsilon=0.6099300000000001\n",
            "episode=818, global_step=20581, episode_reward=51.0, epsilon=0.6089610000000001\n",
            "episode=819, global_step=20673, episode_reward=92.0, epsilon=0.607213\n",
            "episode=820, global_step=20733, episode_reward=60.0, epsilon=0.6060730000000001\n",
            "episode=821, global_step=20771, episode_reward=38.0, epsilon=0.605351\n",
            "episode=822, global_step=20798, episode_reward=27.0, epsilon=0.604838\n",
            "episode=823, global_step=20825, episode_reward=27.0, epsilon=0.604325\n",
            "episode=824, global_step=20872, episode_reward=47.0, epsilon=0.603432\n",
            "episode=825, global_step=20890, episode_reward=18.0, epsilon=0.6030900000000001\n",
            "episode=826, global_step=20929, episode_reward=39.0, epsilon=0.602349\n",
            "episode=827, global_step=20963, episode_reward=34.0, epsilon=0.6017030000000001\n",
            "episode=828, global_step=21039, episode_reward=76.0, epsilon=0.6002590000000001\n",
            "episode=829, global_step=21055, episode_reward=16.0, epsilon=0.599955\n",
            "episode=830, global_step=21139, episode_reward=84.0, epsilon=0.5983590000000001\n",
            "episode=831, global_step=21151, episode_reward=12.0, epsilon=0.598131\n",
            "episode=832, global_step=21191, episode_reward=40.0, epsilon=0.5973710000000001\n",
            "episode=833, global_step=21281, episode_reward=90.0, epsilon=0.595661\n",
            "episode=834, global_step=21345, episode_reward=64.0, epsilon=0.5944450000000001\n",
            "episode=835, global_step=21363, episode_reward=18.0, epsilon=0.594103\n",
            "episode=836, global_step=21375, episode_reward=12.0, epsilon=0.593875\n",
            "episode=837, global_step=21427, episode_reward=52.0, epsilon=0.592887\n",
            "episode=838, global_step=21538, episode_reward=111.0, epsilon=0.590778\n",
            "episode=839, global_step=21562, episode_reward=24.0, epsilon=0.590322\n",
            "episode=840, global_step=21626, episode_reward=64.0, epsilon=0.5891060000000001\n",
            "episode=841, global_step=21645, episode_reward=19.0, epsilon=0.5887450000000001\n",
            "episode=842, global_step=21670, episode_reward=25.0, epsilon=0.5882700000000001\n",
            "episode=843, global_step=21774, episode_reward=104.0, epsilon=0.5862940000000001\n",
            "episode=844, global_step=21794, episode_reward=20.0, epsilon=0.585914\n",
            "episode=845, global_step=21826, episode_reward=32.0, epsilon=0.5853060000000001\n",
            "episode=846, global_step=21945, episode_reward=119.0, epsilon=0.583045\n",
            "episode=847, global_step=22105, episode_reward=160.0, epsilon=0.5800050000000001\n",
            "episode=848, global_step=22210, episode_reward=105.0, epsilon=0.5780100000000001\n",
            "episode=849, global_step=22268, episode_reward=58.0, epsilon=0.576908\n",
            "episode=850, global_step=22327, episode_reward=59.0, epsilon=0.575787\n",
            "episode=851, global_step=22384, episode_reward=57.0, epsilon=0.5747040000000001\n",
            "episode=852, global_step=22402, episode_reward=18.0, epsilon=0.574362\n",
            "episode=853, global_step=22439, episode_reward=37.0, epsilon=0.573659\n",
            "episode=854, global_step=22560, episode_reward=121.0, epsilon=0.5713600000000001\n",
            "episode=855, global_step=22651, episode_reward=91.0, epsilon=0.569631\n",
            "episode=856, global_step=22692, episode_reward=41.0, epsilon=0.5688520000000001\n",
            "episode=857, global_step=22705, episode_reward=13.0, epsilon=0.568605\n",
            "episode=858, global_step=22791, episode_reward=86.0, epsilon=0.5669710000000001\n",
            "episode=859, global_step=22827, episode_reward=36.0, epsilon=0.566287\n",
            "episode=860, global_step=22923, episode_reward=96.0, epsilon=0.564463\n",
            "episode=861, global_step=23059, episode_reward=136.0, epsilon=0.561879\n",
            "episode=862, global_step=23084, episode_reward=25.0, epsilon=0.561404\n",
            "episode=863, global_step=23192, episode_reward=108.0, epsilon=0.5593520000000001\n",
            "episode=864, global_step=23273, episode_reward=81.0, epsilon=0.5578130000000001\n",
            "episode=865, global_step=23367, episode_reward=94.0, epsilon=0.556027\n",
            "episode=866, global_step=23498, episode_reward=131.0, epsilon=0.5535380000000001\n",
            "episode=867, global_step=23572, episode_reward=74.0, epsilon=0.5521320000000001\n",
            "episode=868, global_step=23624, episode_reward=52.0, epsilon=0.5511440000000001\n",
            "episode=869, global_step=23715, episode_reward=91.0, epsilon=0.549415\n",
            "episode=870, global_step=23887, episode_reward=172.0, epsilon=0.546147\n",
            "episode=871, global_step=23903, episode_reward=16.0, epsilon=0.5458430000000001\n",
            "episode=872, global_step=23962, episode_reward=59.0, epsilon=0.544722\n",
            "episode=873, global_step=24075, episode_reward=113.0, epsilon=0.542575\n",
            "episode=874, global_step=24211, episode_reward=136.0, epsilon=0.5399910000000001\n",
            "episode=875, global_step=24246, episode_reward=35.0, epsilon=0.5393260000000001\n",
            "episode=876, global_step=24268, episode_reward=22.0, epsilon=0.538908\n",
            "episode=877, global_step=24435, episode_reward=167.0, epsilon=0.5357350000000001\n",
            "episode=878, global_step=24530, episode_reward=95.0, epsilon=0.53393\n",
            "episode=879, global_step=24636, episode_reward=106.0, epsilon=0.531916\n",
            "episode=880, global_step=24928, episode_reward=292.0, epsilon=0.5263680000000001\n",
            "episode=881, global_step=24976, episode_reward=48.0, epsilon=0.525456\n",
            "episode=882, global_step=25023, episode_reward=47.0, epsilon=0.5245630000000001\n",
            "episode=883, global_step=25127, episode_reward=104.0, epsilon=0.5225870000000001\n",
            "episode=884, global_step=25154, episode_reward=27.0, epsilon=0.522074\n",
            "episode=885, global_step=25330, episode_reward=176.0, epsilon=0.5187300000000001\n",
            "episode=886, global_step=25467, episode_reward=137.0, epsilon=0.516127\n",
            "episode=887, global_step=25660, episode_reward=193.0, epsilon=0.5124600000000001\n",
            "episode=888, global_step=25797, episode_reward=137.0, epsilon=0.509857\n",
            "episode=889, global_step=25879, episode_reward=82.0, epsilon=0.5082990000000001\n",
            "episode=890, global_step=25960, episode_reward=81.0, epsilon=0.5067600000000001\n",
            "episode=891, global_step=26111, episode_reward=151.0, epsilon=0.5038910000000001\n",
            "episode=892, global_step=26208, episode_reward=97.0, epsilon=0.502048\n",
            "episode=893, global_step=26278, episode_reward=70.0, epsilon=0.500718\n",
            "episode=894, global_step=26427, episode_reward=149.0, epsilon=0.4978870000000001\n",
            "episode=895, global_step=26488, episode_reward=61.0, epsilon=0.49672800000000006\n",
            "episode=896, global_step=26558, episode_reward=70.0, epsilon=0.4953980000000001\n",
            "episode=897, global_step=26750, episode_reward=192.0, epsilon=0.49175\n",
            "episode=898, global_step=26794, episode_reward=44.0, epsilon=0.4909140000000001\n",
            "episode=899, global_step=26939, episode_reward=145.0, epsilon=0.488159\n",
            "episode=900, global_step=27034, episode_reward=95.0, epsilon=0.48635400000000006\n",
            "episode=901, global_step=27088, episode_reward=54.0, epsilon=0.4853280000000001\n",
            "episode=902, global_step=27127, episode_reward=39.0, epsilon=0.4845870000000001\n",
            "episode=903, global_step=27146, episode_reward=19.0, epsilon=0.48422600000000005\n",
            "episode=904, global_step=27194, episode_reward=48.0, epsilon=0.483314\n",
            "episode=905, global_step=27239, episode_reward=45.0, epsilon=0.4824590000000001\n",
            "episode=906, global_step=27255, episode_reward=16.0, epsilon=0.4821550000000001\n",
            "episode=907, global_step=27398, episode_reward=143.0, epsilon=0.47943800000000003\n",
            "episode=908, global_step=27564, episode_reward=166.0, epsilon=0.47628400000000004\n",
            "episode=909, global_step=27726, episode_reward=162.0, epsilon=0.473206\n",
            "episode=910, global_step=27740, episode_reward=14.0, epsilon=0.47294\n",
            "episode=911, global_step=27845, episode_reward=105.0, epsilon=0.47094500000000006\n",
            "episode=912, global_step=28128, episode_reward=283.0, epsilon=0.4655680000000001\n",
            "episode=913, global_step=28233, episode_reward=105.0, epsilon=0.463573\n",
            "episode=914, global_step=28284, episode_reward=51.0, epsilon=0.462604\n",
            "episode=915, global_step=28419, episode_reward=135.0, epsilon=0.4600390000000001\n",
            "episode=916, global_step=28600, episode_reward=181.0, epsilon=0.4566000000000001\n",
            "episode=917, global_step=28676, episode_reward=76.0, epsilon=0.4551560000000001\n",
            "episode=918, global_step=28745, episode_reward=69.0, epsilon=0.45384500000000005\n",
            "episode=919, global_step=28861, episode_reward=116.0, epsilon=0.45164100000000007\n",
            "episode=920, global_step=28895, episode_reward=34.0, epsilon=0.45099500000000003\n",
            "episode=921, global_step=28923, episode_reward=28.0, epsilon=0.45046300000000006\n",
            "episode=922, global_step=29055, episode_reward=132.0, epsilon=0.4479550000000001\n",
            "episode=923, global_step=29294, episode_reward=239.0, epsilon=0.4434140000000001\n",
            "episode=924, global_step=29311, episode_reward=17.0, epsilon=0.4430910000000001\n",
            "episode=925, global_step=29465, episode_reward=154.0, epsilon=0.44016500000000003\n",
            "episode=926, global_step=29508, episode_reward=43.0, epsilon=0.43934800000000007\n",
            "episode=927, global_step=29666, episode_reward=158.0, epsilon=0.4363460000000001\n",
            "episode=928, global_step=29754, episode_reward=88.0, epsilon=0.4346740000000001\n",
            "episode=929, global_step=29870, episode_reward=116.0, epsilon=0.43247\n",
            "episode=930, global_step=30036, episode_reward=166.0, epsilon=0.42931600000000003\n",
            "episode=931, global_step=30059, episode_reward=23.0, epsilon=0.4288790000000001\n",
            "episode=932, global_step=30211, episode_reward=152.0, epsilon=0.4259910000000001\n",
            "episode=933, global_step=30385, episode_reward=174.0, epsilon=0.4226850000000001\n",
            "episode=934, global_step=30505, episode_reward=120.0, epsilon=0.42040500000000003\n",
            "episode=935, global_step=30647, episode_reward=142.0, epsilon=0.41770700000000005\n",
            "episode=936, global_step=30778, episode_reward=131.0, epsilon=0.4152180000000001\n",
            "episode=937, global_step=30901, episode_reward=123.0, epsilon=0.41288100000000005\n",
            "episode=938, global_step=31035, episode_reward=134.0, epsilon=0.4103350000000001\n",
            "episode=939, global_step=31047, episode_reward=12.0, epsilon=0.4101070000000001\n",
            "episode=940, global_step=31070, episode_reward=23.0, epsilon=0.4096700000000001\n",
            "episode=941, global_step=31232, episode_reward=162.0, epsilon=0.40659200000000006\n",
            "episode=942, global_step=31258, episode_reward=26.0, epsilon=0.40609800000000007\n",
            "episode=943, global_step=31399, episode_reward=141.0, epsilon=0.4034190000000001\n",
            "episode=944, global_step=31547, episode_reward=148.0, epsilon=0.40060700000000005\n",
            "episode=945, global_step=31702, episode_reward=155.0, epsilon=0.39766200000000007\n",
            "episode=946, global_step=31867, episode_reward=165.0, epsilon=0.39452700000000007\n",
            "episode=947, global_step=32046, episode_reward=179.0, epsilon=0.3911260000000001\n",
            "episode=948, global_step=32392, episode_reward=346.0, epsilon=0.3845520000000001\n",
            "episode=949, global_step=32524, episode_reward=132.0, epsilon=0.38204400000000005\n",
            "episode=950, global_step=32656, episode_reward=132.0, epsilon=0.3795360000000001\n",
            "episode=951, global_step=32726, episode_reward=70.0, epsilon=0.37820600000000004\n",
            "episode=952, global_step=32805, episode_reward=79.0, epsilon=0.37670500000000007\n",
            "episode=953, global_step=32918, episode_reward=113.0, epsilon=0.37455800000000006\n",
            "episode=954, global_step=33059, episode_reward=141.0, epsilon=0.37187900000000007\n",
            "episode=955, global_step=33213, episode_reward=154.0, epsilon=0.3689530000000001\n",
            "episode=956, global_step=33364, episode_reward=151.0, epsilon=0.3660840000000001\n",
            "episode=957, global_step=33533, episode_reward=169.0, epsilon=0.3628730000000001\n",
            "episode=958, global_step=33673, episode_reward=140.0, epsilon=0.3602130000000001\n",
            "episode=959, global_step=33806, episode_reward=133.0, epsilon=0.35768600000000006\n",
            "episode=960, global_step=33945, episode_reward=139.0, epsilon=0.35504500000000005\n",
            "episode=961, global_step=34091, episode_reward=146.0, epsilon=0.3522710000000001\n",
            "episode=962, global_step=34221, episode_reward=130.0, epsilon=0.34980100000000003\n",
            "episode=963, global_step=34369, episode_reward=148.0, epsilon=0.3469890000000001\n",
            "episode=964, global_step=34402, episode_reward=33.0, epsilon=0.34636200000000006\n",
            "episode=965, global_step=34531, episode_reward=129.0, epsilon=0.3439110000000001\n",
            "episode=966, global_step=34661, episode_reward=130.0, epsilon=0.3414410000000001\n",
            "episode=967, global_step=34793, episode_reward=132.0, epsilon=0.33893300000000004\n",
            "episode=968, global_step=34889, episode_reward=96.0, epsilon=0.3371090000000001\n",
            "episode=969, global_step=35015, episode_reward=126.0, epsilon=0.3347150000000001\n",
            "episode=970, global_step=35161, episode_reward=146.0, epsilon=0.33194100000000004\n",
            "episode=971, global_step=35516, episode_reward=355.0, epsilon=0.32519600000000004\n",
            "episode=972, global_step=35645, episode_reward=129.0, epsilon=0.32274500000000006\n",
            "episode=973, global_step=35805, episode_reward=160.0, epsilon=0.31970500000000013\n",
            "episode=974, global_step=35937, episode_reward=132.0, epsilon=0.31719700000000006\n",
            "episode=975, global_step=35951, episode_reward=14.0, epsilon=0.3169310000000001\n",
            "episode=976, global_step=36109, episode_reward=158.0, epsilon=0.3139290000000001\n",
            "episode=977, global_step=36125, episode_reward=16.0, epsilon=0.31362500000000004\n",
            "episode=978, global_step=36262, episode_reward=137.0, epsilon=0.31102200000000013\n",
            "episode=979, global_step=36291, episode_reward=29.0, epsilon=0.31047100000000005\n",
            "episode=980, global_step=36423, episode_reward=132.0, epsilon=0.3079630000000001\n",
            "episode=981, global_step=36436, episode_reward=13.0, epsilon=0.3077160000000001\n",
            "episode=982, global_step=36561, episode_reward=125.0, epsilon=0.3053410000000001\n",
            "episode=983, global_step=36700, episode_reward=139.0, epsilon=0.3027000000000001\n",
            "episode=984, global_step=36831, episode_reward=131.0, epsilon=0.3002110000000001\n",
            "episode=985, global_step=36958, episode_reward=127.0, epsilon=0.2977980000000001\n",
            "episode=986, global_step=36978, episode_reward=20.0, epsilon=0.29741800000000007\n",
            "episode=987, global_step=37101, episode_reward=123.0, epsilon=0.29508100000000004\n",
            "episode=988, global_step=37237, episode_reward=136.0, epsilon=0.2924970000000001\n",
            "episode=989, global_step=37379, episode_reward=142.0, epsilon=0.28979900000000014\n",
            "episode=990, global_step=37393, episode_reward=14.0, epsilon=0.28953300000000004\n",
            "episode=991, global_step=37525, episode_reward=132.0, epsilon=0.2870250000000001\n",
            "episode=992, global_step=37666, episode_reward=141.0, epsilon=0.2843460000000001\n",
            "episode=993, global_step=37678, episode_reward=12.0, epsilon=0.2841180000000001\n",
            "episode=994, global_step=37819, episode_reward=141.0, epsilon=0.2814390000000001\n",
            "episode=995, global_step=37953, episode_reward=134.0, epsilon=0.27889300000000006\n",
            "episode=996, global_step=38094, episode_reward=141.0, epsilon=0.27621400000000007\n",
            "episode=997, global_step=38222, episode_reward=128.0, epsilon=0.2737820000000001\n",
            "episode=998, global_step=38450, episode_reward=228.0, epsilon=0.2694500000000001\n",
            "episode=999, global_step=38586, episode_reward=136.0, epsilon=0.26686600000000005\n",
            "episode=1000, global_step=39035, episode_reward=449.0, epsilon=0.2583350000000001\n",
            "episode=1001, global_step=39287, episode_reward=252.0, epsilon=0.2535470000000001\n",
            "episode=1002, global_step=39512, episode_reward=225.0, epsilon=0.24927200000000005\n",
            "episode=1003, global_step=39859, episode_reward=347.0, epsilon=0.2426790000000001\n",
            "episode=1004, global_step=40036, episode_reward=177.0, epsilon=0.23931600000000008\n",
            "episode=1005, global_step=40354, episode_reward=318.0, epsilon=0.2332740000000001\n",
            "episode=1006, global_step=40483, episode_reward=129.0, epsilon=0.2308230000000001\n",
            "episode=1007, global_step=40532, episode_reward=49.0, epsilon=0.2298920000000001\n",
            "episode=1008, global_step=40577, episode_reward=45.0, epsilon=0.22903700000000005\n",
            "episode=1009, global_step=40825, episode_reward=248.0, epsilon=0.2243250000000001\n",
            "episode=1010, global_step=41051, episode_reward=226.0, epsilon=0.2200310000000001\n",
            "episode=1011, global_step=41257, episode_reward=206.0, epsilon=0.21611700000000011\n",
            "episode=1012, global_step=41418, episode_reward=161.0, epsilon=0.21305800000000008\n",
            "episode=1013, global_step=41431, episode_reward=13.0, epsilon=0.21281100000000008\n",
            "episode=1014, global_step=41575, episode_reward=144.0, epsilon=0.21007500000000012\n",
            "episode=1015, global_step=41816, episode_reward=241.0, epsilon=0.20549600000000012\n",
            "episode=1016, global_step=41950, episode_reward=134.0, epsilon=0.20295000000000007\n",
            "episode=1017, global_step=42067, episode_reward=117.0, epsilon=0.2007270000000001\n",
            "episode=1018, global_step=42292, episode_reward=225.0, epsilon=0.19645200000000007\n",
            "episode=1019, global_step=42792, episode_reward=500.0, epsilon=0.18695200000000012\n",
            "episode=1020, global_step=42935, episode_reward=143.0, epsilon=0.18423500000000015\n",
            "episode=1021, global_step=43092, episode_reward=157.0, epsilon=0.18125200000000008\n",
            "episode=1022, global_step=43246, episode_reward=154.0, epsilon=0.1783260000000001\n",
            "episode=1023, global_step=43395, episode_reward=149.0, epsilon=0.17549500000000007\n",
            "episode=1024, global_step=43534, episode_reward=139.0, epsilon=0.17285400000000006\n",
            "episode=1025, global_step=43697, episode_reward=163.0, epsilon=0.16975700000000016\n",
            "episode=1026, global_step=43869, episode_reward=172.0, epsilon=0.1664890000000001\n",
            "episode=1027, global_step=44026, episode_reward=157.0, epsilon=0.16350600000000015\n",
            "episode=1028, global_step=44350, episode_reward=324.0, epsilon=0.1573500000000001\n",
            "episode=1029, global_step=44555, episode_reward=205.0, epsilon=0.15345500000000012\n",
            "episode=1030, global_step=44741, episode_reward=186.0, epsilon=0.14992100000000008\n",
            "episode=1031, global_step=44899, episode_reward=158.0, epsilon=0.14691900000000013\n",
            "episode=1032, global_step=45089, episode_reward=190.0, epsilon=0.14330900000000013\n",
            "episode=1033, global_step=45417, episode_reward=328.0, epsilon=0.13707700000000012\n",
            "episode=1034, global_step=45592, episode_reward=175.0, epsilon=0.1337520000000001\n",
            "episode=1035, global_step=45752, episode_reward=160.0, epsilon=0.13071200000000016\n",
            "episode=1036, global_step=45897, episode_reward=145.0, epsilon=0.1279570000000001\n",
            "episode=1037, global_step=46085, episode_reward=188.0, epsilon=0.12438500000000008\n",
            "episode=1038, global_step=46280, episode_reward=195.0, epsilon=0.12068000000000012\n",
            "episode=1039, global_step=46449, episode_reward=169.0, epsilon=0.11746900000000016\n",
            "episode=1040, global_step=46780, episode_reward=331.0, epsilon=0.11118000000000006\n",
            "episode=1041, global_step=46941, episode_reward=161.0, epsilon=0.10812100000000013\n",
            "episode=1042, global_step=47191, episode_reward=250.0, epsilon=0.1033710000000001\n",
            "episode=1043, global_step=47359, episode_reward=168.0, epsilon=0.10017900000000013\n",
            "episode=1044, global_step=47616, episode_reward=257.0, epsilon=0.09529600000000016\n",
            "episode=1045, global_step=47770, episode_reward=154.0, epsilon=0.09237000000000006\n",
            "episode=1046, global_step=47970, episode_reward=200.0, epsilon=0.08857000000000015\n",
            "episode=1047, global_step=48175, episode_reward=205.0, epsilon=0.08467500000000017\n",
            "episode=1048, global_step=48428, episode_reward=253.0, epsilon=0.07986800000000016\n",
            "episode=1049, global_step=48706, episode_reward=278.0, epsilon=0.07458600000000015\n",
            "episode=1050, global_step=48900, episode_reward=194.0, epsilon=0.07090000000000007\n",
            "episode=1051, global_step=49093, episode_reward=193.0, epsilon=0.0672330000000001\n",
            "episode=1052, global_step=49295, episode_reward=202.0, epsilon=0.06339500000000009\n",
            "episode=1053, global_step=49520, episode_reward=225.0, epsilon=0.05912000000000006\n",
            "episode=1054, global_step=49805, episode_reward=285.0, epsilon=0.053705000000000114\n",
            "episode=1055, global_step=50076, episode_reward=271.0, epsilon=0.05\n",
            "episode=1056, global_step=50345, episode_reward=269.0, epsilon=0.05\n",
            "episode=1057, global_step=50655, episode_reward=310.0, epsilon=0.05\n",
            "episode=1058, global_step=51028, episode_reward=373.0, epsilon=0.05\n",
            "episode=1059, global_step=51439, episode_reward=411.0, epsilon=0.05\n",
            "episode=1060, global_step=51786, episode_reward=347.0, epsilon=0.05\n",
            "episode=1061, global_step=52031, episode_reward=245.0, epsilon=0.05\n",
            "episode=1062, global_step=52209, episode_reward=178.0, epsilon=0.05\n",
            "episode=1063, global_step=52384, episode_reward=175.0, epsilon=0.05\n",
            "episode=1064, global_step=52568, episode_reward=184.0, epsilon=0.05\n",
            "episode=1065, global_step=52753, episode_reward=185.0, epsilon=0.05\n",
            "episode=1066, global_step=52944, episode_reward=191.0, epsilon=0.05\n",
            "episode=1067, global_step=53131, episode_reward=187.0, epsilon=0.05\n",
            "episode=1068, global_step=53292, episode_reward=161.0, epsilon=0.05\n",
            "episode=1069, global_step=53484, episode_reward=192.0, epsilon=0.05\n",
            "episode=1070, global_step=53688, episode_reward=204.0, epsilon=0.05\n",
            "episode=1071, global_step=53896, episode_reward=208.0, epsilon=0.05\n",
            "episode=1072, global_step=54101, episode_reward=205.0, epsilon=0.05\n",
            "episode=1073, global_step=54342, episode_reward=241.0, epsilon=0.05\n",
            "episode=1074, global_step=54603, episode_reward=261.0, epsilon=0.05\n",
            "episode=1075, global_step=54829, episode_reward=226.0, epsilon=0.05\n",
            "episode=1076, global_step=55054, episode_reward=225.0, epsilon=0.05\n",
            "episode=1077, global_step=55303, episode_reward=249.0, epsilon=0.05\n",
            "episode=1078, global_step=55518, episode_reward=215.0, epsilon=0.05\n",
            "episode=1079, global_step=55782, episode_reward=264.0, epsilon=0.05\n",
            "episode=1080, global_step=56156, episode_reward=374.0, epsilon=0.05\n",
            "episode=1081, global_step=56549, episode_reward=393.0, epsilon=0.05\n",
            "episode=1082, global_step=56868, episode_reward=319.0, epsilon=0.05\n",
            "episode=1083, global_step=57323, episode_reward=455.0, epsilon=0.05\n",
            "episode=1084, global_step=57584, episode_reward=261.0, epsilon=0.05\n",
            "episode=1085, global_step=57878, episode_reward=294.0, epsilon=0.05\n",
            "episode=1086, global_step=58065, episode_reward=187.0, epsilon=0.05\n",
            "episode=1087, global_step=58476, episode_reward=411.0, epsilon=0.05\n",
            "episode=1088, global_step=58670, episode_reward=194.0, epsilon=0.05\n",
            "episode=1089, global_step=58862, episode_reward=192.0, epsilon=0.05\n",
            "episode=1090, global_step=59161, episode_reward=299.0, epsilon=0.05\n",
            "episode=1091, global_step=59472, episode_reward=311.0, epsilon=0.05\n",
            "episode=1092, global_step=59856, episode_reward=384.0, epsilon=0.05\n",
            "episode=1093, global_step=60213, episode_reward=357.0, epsilon=0.05\n",
            "episode=1094, global_step=60680, episode_reward=467.0, epsilon=0.05\n",
            "episode=1095, global_step=60965, episode_reward=285.0, epsilon=0.05\n",
            "episode=1096, global_step=61159, episode_reward=194.0, epsilon=0.05\n",
            "episode=1097, global_step=61366, episode_reward=207.0, epsilon=0.05\n",
            "episode=1098, global_step=61667, episode_reward=301.0, epsilon=0.05\n",
            "episode=1099, global_step=61966, episode_reward=299.0, epsilon=0.05\n",
            "episode=1100, global_step=62217, episode_reward=251.0, epsilon=0.05\n",
            "episode=1101, global_step=62696, episode_reward=479.0, epsilon=0.05\n",
            "episode=1102, global_step=63042, episode_reward=346.0, epsilon=0.05\n",
            "episode=1103, global_step=63448, episode_reward=406.0, epsilon=0.05\n",
            "episode=1104, global_step=63761, episode_reward=313.0, epsilon=0.05\n",
            "episode=1105, global_step=64070, episode_reward=309.0, epsilon=0.05\n",
            "episode=1106, global_step=64386, episode_reward=316.0, epsilon=0.05\n",
            "episode=1107, global_step=64727, episode_reward=341.0, epsilon=0.05\n",
            "episode=1108, global_step=65227, episode_reward=500.0, epsilon=0.05\n",
            "episode=1109, global_step=65534, episode_reward=307.0, epsilon=0.05\n",
            "episode=1110, global_step=66034, episode_reward=500.0, epsilon=0.05\n",
            "episode=1111, global_step=66534, episode_reward=500.0, epsilon=0.05\n",
            "episode=1112, global_step=66820, episode_reward=286.0, epsilon=0.05\n",
            "episode=1113, global_step=67162, episode_reward=342.0, epsilon=0.05\n",
            "episode=1114, global_step=67466, episode_reward=304.0, epsilon=0.05\n",
            "episode=1115, global_step=67751, episode_reward=285.0, epsilon=0.05\n",
            "episode=1116, global_step=68119, episode_reward=368.0, epsilon=0.05\n",
            "episode=1117, global_step=68619, episode_reward=500.0, epsilon=0.05\n",
            "episode=1118, global_step=69119, episode_reward=500.0, epsilon=0.05\n",
            "episode=1119, global_step=69619, episode_reward=500.0, epsilon=0.05\n",
            "episode=1120, global_step=70119, episode_reward=500.0, epsilon=0.05\n",
            "episode=1121, global_step=70619, episode_reward=500.0, epsilon=0.05\n",
            "episode=1122, global_step=71119, episode_reward=500.0, epsilon=0.05\n",
            "episode=1123, global_step=71619, episode_reward=500.0, epsilon=0.05\n",
            "episode=1124, global_step=72119, episode_reward=500.0, epsilon=0.05\n",
            "episode=1125, global_step=72619, episode_reward=500.0, epsilon=0.05\n",
            "episode=1126, global_step=73119, episode_reward=500.0, epsilon=0.05\n",
            "episode=1127, global_step=73619, episode_reward=500.0, epsilon=0.05\n",
            "episode=1128, global_step=74119, episode_reward=500.0, epsilon=0.05\n",
            "episode=1129, global_step=74619, episode_reward=500.0, epsilon=0.05\n",
            "episode=1130, global_step=75119, episode_reward=500.0, epsilon=0.05\n",
            "episode=1131, global_step=75619, episode_reward=500.0, epsilon=0.05\n",
            "episode=1132, global_step=76119, episode_reward=500.0, epsilon=0.05\n",
            "episode=1133, global_step=76619, episode_reward=500.0, epsilon=0.05\n",
            "episode=1134, global_step=77119, episode_reward=500.0, epsilon=0.05\n",
            "episode=1135, global_step=77619, episode_reward=500.0, epsilon=0.05\n",
            "episode=1136, global_step=78119, episode_reward=500.0, epsilon=0.05\n",
            "episode=1137, global_step=78333, episode_reward=214.0, epsilon=0.05\n",
            "episode=1138, global_step=78509, episode_reward=176.0, epsilon=0.05\n",
            "episode=1139, global_step=79009, episode_reward=500.0, epsilon=0.05\n",
            "episode=1140, global_step=79224, episode_reward=215.0, epsilon=0.05\n",
            "episode=1141, global_step=79724, episode_reward=500.0, epsilon=0.05\n",
            "episode=1142, global_step=79892, episode_reward=168.0, epsilon=0.05\n",
            "episode=1143, global_step=80392, episode_reward=500.0, epsilon=0.05\n",
            "episode=1144, global_step=80892, episode_reward=500.0, epsilon=0.05\n",
            "episode=1145, global_step=81392, episode_reward=500.0, epsilon=0.05\n",
            "episode=1146, global_step=81892, episode_reward=500.0, epsilon=0.05\n",
            "episode=1147, global_step=82165, episode_reward=273.0, epsilon=0.05\n",
            "episode=1148, global_step=82380, episode_reward=215.0, epsilon=0.05\n",
            "episode=1149, global_step=82880, episode_reward=500.0, epsilon=0.05\n",
            "episode=1150, global_step=83066, episode_reward=186.0, epsilon=0.05\n",
            "episode=1151, global_step=83248, episode_reward=182.0, epsilon=0.05\n",
            "episode=1152, global_step=83553, episode_reward=305.0, epsilon=0.05\n",
            "episode=1153, global_step=84053, episode_reward=500.0, epsilon=0.05\n",
            "episode=1154, global_step=84553, episode_reward=500.0, epsilon=0.05\n",
            "episode=1155, global_step=85053, episode_reward=500.0, epsilon=0.05\n",
            "episode=1156, global_step=85553, episode_reward=500.0, epsilon=0.05\n",
            "episode=1157, global_step=86053, episode_reward=500.0, epsilon=0.05\n",
            "episode=1158, global_step=86299, episode_reward=246.0, epsilon=0.05\n",
            "episode=1159, global_step=86799, episode_reward=500.0, epsilon=0.05\n",
            "episode=1160, global_step=87299, episode_reward=500.0, epsilon=0.05\n",
            "episode=1161, global_step=87608, episode_reward=309.0, epsilon=0.05\n",
            "episode=1162, global_step=87894, episode_reward=286.0, epsilon=0.05\n",
            "episode=1163, global_step=88394, episode_reward=500.0, epsilon=0.05\n",
            "episode=1164, global_step=88689, episode_reward=295.0, epsilon=0.05\n",
            "episode=1165, global_step=89189, episode_reward=500.0, epsilon=0.05\n",
            "episode=1166, global_step=89471, episode_reward=282.0, epsilon=0.05\n",
            "episode=1167, global_step=89971, episode_reward=500.0, epsilon=0.05\n",
            "episode=1168, global_step=90387, episode_reward=416.0, epsilon=0.05\n",
            "episode=1169, global_step=90830, episode_reward=443.0, epsilon=0.05\n",
            "episode=1170, global_step=91328, episode_reward=498.0, epsilon=0.05\n",
            "episode=1171, global_step=91828, episode_reward=500.0, epsilon=0.05\n",
            "episode=1172, global_step=92316, episode_reward=488.0, epsilon=0.05\n",
            "episode=1173, global_step=92815, episode_reward=499.0, epsilon=0.05\n",
            "episode=1174, global_step=93315, episode_reward=500.0, epsilon=0.05\n",
            "episode=1175, global_step=93737, episode_reward=422.0, epsilon=0.05\n",
            "episode=1176, global_step=93990, episode_reward=253.0, epsilon=0.05\n",
            "episode=1177, global_step=94490, episode_reward=500.0, epsilon=0.05\n",
            "episode=1178, global_step=94762, episode_reward=272.0, epsilon=0.05\n",
            "episode=1179, global_step=95018, episode_reward=256.0, epsilon=0.05\n",
            "episode=1180, global_step=95518, episode_reward=500.0, epsilon=0.05\n",
            "episode=1181, global_step=96018, episode_reward=500.0, epsilon=0.05\n",
            "episode=1182, global_step=96209, episode_reward=191.0, epsilon=0.05\n",
            "episode=1183, global_step=96709, episode_reward=500.0, epsilon=0.05\n",
            "episode=1184, global_step=97209, episode_reward=500.0, epsilon=0.05\n",
            "episode=1185, global_step=97661, episode_reward=452.0, epsilon=0.05\n",
            "episode=1186, global_step=98107, episode_reward=446.0, epsilon=0.05\n",
            "episode=1187, global_step=98607, episode_reward=500.0, epsilon=0.05\n",
            "episode=1188, global_step=99107, episode_reward=500.0, epsilon=0.05\n",
            "episode=1189, global_step=99607, episode_reward=500.0, epsilon=0.05\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-wNmFb41cdKp"
      },
      "source": [
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VEMSnKf9lsuB",
        "outputId": "9850fc7a-343f-4d75-dcfa-d73387c22be7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        }
      },
      "source": [
        "plt.plot(episode_rewards)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f01a64e5668>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deZgU1bn/v2/PMOz7JgIyIIiixg0RxQXFDUnUa4wxvyTiFhOjUWPuVdRc741xwWjUmBiX6wbRuEQxIquAKIJswzLsMwwwDMMyC8vAMMxMT/f5/VGnuquqa+/qme6e9/M8PFN16tSpU93Nt956z3veQ0IIMAzDMNlFqKU7wDAMwwQPizvDMEwWwuLOMAyThbC4MwzDZCEs7gzDMFlIbkt3AAB69eol8vPzW7obDMMwGcWqVauqhRC9zY6lhbjn5+ejoKCgpbvBMAyTURDRTqtj7JZhGIbJQljcGYZhshAWd4ZhmCyExZ1hGCYLYXFnGIbJQlyJOxGVEtF6IlpLRAWyrAcRzSOirfJvd1lORPQyEZUQ0ToiOjuVN8AwDMMk4sVyv1QIcaYQYqTcnwRggRBiGIAFch8AxgMYJv/dBeDVoDrLMAzDuCOZOPfrAIyV21MAfA3gYVk+VSi5hJcRUTci6ieE2JtMRxmGYZwoqTyC/bWNOG9Iz8Db/mpLBeZs2IebR52Ab4qqYEyXfqCuERt2H8aYoT3RNjcHOSHCmrJDGNC9PdaVH8L1Z/VH2f46NDRFcbCuEW1yQhjYvT3GndIXZwzsFnh/3Yq7APAlEQkArwsh3gDQVyPY+wD0ldv9AezSnFsuy3TiTkR3QbHsccIJJ/jrPcMwjIbLX1gEACidPCHwtm9/V5lo+XFBOQCAKH5Mq/Nrdx0yPX91mXl5ny7tWlTcLxRC7CaiPgDmEdEW7UEhhJDC7xr5gHgDAEaOHMkrhjAMkzF0yMvBpieuju3f/d4qzN6wz1dbPxs9KKhu6XDlcxdC7JZ/KwF8BmAUgAoi6gcA8m+lrL4bwEDN6QNkGcMwTFZAzlVaHEdxJ6KORNRZ3QZwJYANAKYDmCirTQTwudyeDuAWGTUzGkAN+9sZhskmiMiw30IdscGNW6YvgM/kzeQC+KcQYg4RrQTwMRHdAWAngJtk/VkArgFQAqAOwG2B95phGKYFSUMtT8BR3IUQ2wGcYVK+H8A4k3IB4J5AescwDJOGGC11SkO55xmqDMMwHjG6ZdIRFneGYRiPJGh7Gmp9WizWwTAMk67c8e5KLCyq1JVlgLazuDMMw9ixYEtlQlmI3TIMwzDZR8KAahqKPYs7wzCMZ9JPzI2wW4ZhGMaEpkgUVbUNpscSQyHTDxZ3hmEYE56YsQlTl+40PRZKRzU3wG4ZhmEYExZsThxIVTFOWkpDlzuLO8MwjFcywS3D4s4wDOORdBRzIyzuDMMwEiEE5m2qSFhlyUhiVsj0k3sWd4ZhGMmHK3fhF1ML8NHKXbYCn4ZangCLO8MwjGTvoWMAgH2H62Fnu7PPnWEYJkW8vGAr8ifNTEnbBIKdZyYdU/waYXFnGCYjeWFecYtdOyHOPQ21nsWdYRhGYj+MGidhQNWnur9/53m+znMDizvDMIwBIkDYSH1QhvqYob0CaikRFneGYRiJQwRknISskIF3JWlY3BmGYQwQ7IWe87kzDMOkGKcJR6kgA8ZTWdwZhslsUqXtnuLc01DdWdwZhmFM4Dh3hmGYFiRIw12NkFEscffpB9JR7FncGYbJaIL0uatNOSUCS8dEYUZY3BmGyWhSNZxq75Yx7Keh1rO4MwyT0bRAsExairkRFneGYTIau5mkybVrjTHOPR3FnsWdYRhG4j63TEJJwD1JHhZ3hmEyGj9umdLqo/j9v9cjEtWfrLb13NwiHKprtDw//aQ8ERZ3hmFaHfd+sBrvLSvDpj2HLetEPcxiymi3DBHlENEaIpoh9wcT0XIiKiGij4goT5a3lfsl8nh+arrOMAzjz3KPhzz6u2ZCPvcUn+frWh7q3g9gs2b/WQAvCiGGAjgI4A5ZfgeAg7L8RVmPYRgmJfgZUE02wsZvbpnmjI93Je5ENADABABvyn0CcBmAT2SVKQCul9vXyX3I4+MoEyL+GYbJSIIMhXT7oMgESXNrub8E4CEAUbnfE8AhIUST3C8H0F9u9wewCwDk8RpZXwcR3UVEBURUUFVV5bP7DMMw3lEl3K9G+53ElFZuGSL6PoBKIcSqIC8shHhDCDFSCDGyd+/eQTbNMEwrwo/hrqYs8JsTxm9umebMQZPros4YANcS0TUA2gHoAuAvALoRUa60zgcA2C3r7wYwEEA5EeUC6Apgf+A9ZxiGQbC5ZdaWHXJVz7dbJp0sdyHEI0KIAUKIfAA3A/hKCPFTAAsB3CirTQTwudyeLvchj38lWiKbPsMwrYJkxMWo0ct3HHB3nkM7bs9LJcnEuT8M4EEiKoHiU39Llr8FoKcsfxDApOS6yDAMY00yoZB+zU7fhnszqrsbt0wMIcTXAL6W29sBjDKpUw/gRwH0jWEYxhk/4i5PivpUd6Pv3HUoZDPa7jxDlWGYVoeq6X7FPWRQTrc++Oa03FncGYbJaJLJCmnMLeOWdFx5yQiLO8MwGY0vn7v861Pbk0hbwG4ZhmEYVyQT5+7XLeOXTImWYRiGaXH8RFrHLHeN6e7FRdOcFrhfWNwZhslokrG9I5oHQ1M0alNTTyaEQrK4MwzT+jCJc/fyAuBXo9MuKyTDMEy6ksyAqtYV40ncfYo0W+4MwzAuSSoUUqPoXtrx63PnAVWGYRgHYvrqK/2A0P0FvIVF+k3dy24ZhmEYlyQ1oKoZQ/USdcOWO8MwTBJUHqlHfThieixmuCc1icmn5e5TOdnnzjAMA2DUUwtwy1srAm83lltGq+jNMKDanLY7izvDMGnNilL7HOu+FsiOZYX0147RLeM6nztb7gzDMO5IJoNApLkHVP2d5gsWd4ZhWpyZ6/ZihctVkFRU18i1f1vs+XpmbhkvA6r+JzH5PNEHnhbrYBiGSQX3/HM1AKB08gTP51bXNno+J7YSE/xa7pxbhmEYJiUkI6+xrJDaUEgPPvfmjFf3C4s7wzCtDrNQSC++e/8+d46WYRiGSRlmC2R7E3e23BmGYdIOswWyvSzc4XcSU3OSAV1kGIZJxK3xXB+O4K8LtqKxKe5gVwdPfc5hgl+PP0fLMAzDBMDK0gOYv7kCr3+zHZ3a5eK2MYMBaEIhtZa7pwVVm3d5Pj+wuDMMk5Eog5PWIru/tgE/em1pbL8+rF1pKTErpBc8LNrUYrBbhmGYrKNsfx1++uZyy+PCxC3jxece8flQ4BmqDMMwSfDkzE3Ysu+I5fGoST53L3ptfBA0Z4ijW1jcGYbJeLy6V8wGVL1Y7snks2kuWNwZhsl4Ply5S7fvFJUSm6GqW2bPPV4eBFp4JSaGYRgnNDpZdqDOcMheRFVp1k9i8uBz9xRZ0zKwuDMM0/owCYX0YoyzW4ZhGMYnqUzBG425ZTTX83F+OuMo7kTUjohWEFEhEW0koj/I8sFEtJyISojoIyLKk+Vt5X6JPJ6f2ltgGCYbSUY/HX3u8q/b9AMd8nJ0+1kh7gAaAFwmhDgDwJkAriai0QCeBfCiEGIogIMA7pD17wBwUJa/KOsxDMMESjJDk15DIccO740Fv7skth/xOYkprZbZEwq1creN/CcAXAbgE1k+BcD1cvs6uQ95fBxlQvJjhmHSCi+2sRuF0dbxEwp5Yu9Ose1IBkxRdeVzJ6IcIloLoBLAPADbABwSQjTJKuUA+svt/gB2AYA8XgOgZ5CdZhgm+3HyudsJuq9QSJvLfV1Updtv9Gu6NyOuxF0IERFCnAlgAIBRAE5O9sJEdBcRFRBRQVVVlfMJDMO0KlLp1VZDGXUDqjYXrGuM6Pa1GSa9kFZuGS1CiEMAFgI4H0A3IlITjw0AsFtu7wYwEADk8a4A9pu09YYQYqQQYmTv3r19dp9hGCYRuzj3aFTERF3nc/fwOPEr7s05DusmWqY3EXWT2+0BXAFgMxSRv1FWmwjgc7k9Xe5DHv9K+E29xjBMWlNSWYv8STNRWn008La9qIaX3C4RiwgZL/OSGnyKe3PixnLvB2AhEa0DsBLAPCHEDAAPA3iQiEqg+NTfkvXfAtBTlj8IYFLw3WYYJh2YtrocADBj3Z7A23aypP0m69LOLtW7ZTxY7j597mm1WIcQYh2As0zKt0PxvxvL6wH8KJDeMQzTaknV+75WmIUuWsZ9G2GDuKdjPCDPUGUYxjduRK244gjqwxHniklc21UopPx7zV++jZXprXVndR/SqyMA4NR+XV30sGXhlZgYhvGNk3V98GgjrnxxEa4/83i8dHOCA6BFKD94LLbt1ef+1X+Oxdpdh3BS307OlSXdOrTBobowurTLbda872y5MwyTNFbzFI82KlNhVpYe9NxmKtwyxoFft6GQWs4c2A0d8vR2sdO5L9x0Bmb85iJ3FwgIttwZhklLvIQmuj137PNf6/bd5pZJlhvOHpCytq1gy51hGN+kMsbZSWvJYtuyPYdrpDJRmdd6QcDizjBMq8BsgQ19+oHsmo7D4s4wTFriJLVe8xGazSr1u8xeJsDizjCMb5yM3WSM4aAtabOJR34GVJOhOcPhWdwZhklLPGmtCys+rLHcxwztiT6d2+oeIJmwAIcXWNwZhvGNk6YmM4AYtNZqLfcQEUJEUNOy7zl0DItLqoO9YAvDoZAMw/imJY1dr8+NcIK4x631q15chCMNTVanBkZzrlvEljvDMEmTEs3ylBXSmQ9W7Ipt54QIRBTzuTeHsDc3LO4Mw6QljpOYknighIgQCmVf+KMWFneGYXyTzCxSx7ZTqLshUgS+uQdRx53cp9muxeLOMEzSNGdCLJWcUPyaZQfqdMecNDsnpAyoupX2m0YGkz7gkWtOwfJHxwXSlhM8oMowTFriJLy5obht+tma3bjgxJ6u2w6FCETuc7j/6cYzXLdtR06I0LdLu0DacoItd4Zh0hInf3hejv5tYdVO95knc9RQyGZwy9x9yYkpv4YZbLkzDJOWOFruOXrbdGtlreu2FZ976gdUSydPSGn7drDlzjBMytBqZ304gufnFiWsyuRXYNskYbmHQvpJTMmShqvssbgzTGuloPQAfvz60oT1QD3hQZff/a4Uf1tYgv9btF3fhEUbTprfJse/fOWQGufOoZAMw2QZD32yDst3HEiINPGD1SQmrXY2hJWHiPFhYiWvTmGWduLuLlrG26LYmQaLO8MwzYKVWFu6ZRyENzfHvzOESImW4UlMDMMwJjhJo1bQVR015lexttztqTkWdqhhTU6oZSYxNScs7gzDJI2VDa1bxk6tS9Z13FIfjmB71VHnihbEfe6+m0h7WNwZhkkZOu20UHFrd411u2YLb2hxTkWszwqZjbC4M0wrJ5X6ZubTNqYqsHa5+++Y0z3l5YaU9AOyXqe22Tflh8WdYVorzRCcLUy23aYHTuVDJy8npLPcm4IKeE8jWNwZppUghMCi4qpmjRDR+dzVAVWbOs1FXm5IF+ceyULnO4s7w7QS3l9ehlveXoHphXsCa9P5QaGJlrHyrVuW213X4bIOKG4ZJc5dCIFwhMWdYZgMZddBZbLSnkP1gbftZhKTVV3rGarWgpvs24filiFAxCcy/WTUCXjntnN9tRfSpB82pkVoKVjcGabVk8IFN7TbXuPcbboVjOWuuGXUGbMDe7THpcP9LaYR0tzTCT06JNe5gGBxZxjGN04iaxbnHgROIYxOkTaKz11pZ9rq3QCAon1HAPhbDzZXY7k35yLYdjiKOxENJKKFRLSJiDYS0f2yvAcRzSOirfJvd1lORPQyEZUQ0ToiOjvVN8EwTDIkL0ZWKzGZzVBNqOPDDE/2QdE2ZrkD326tAgAca1SyVfr5NLSrQqULbiz3JgC/E0KMADAawD1ENALAJAALhBDDACyQ+wAwHsAw+e8uAK8G3muGYTICveWu7CT43F2ca8TJcjfmejeihkIKIdCtQx4A4JkbTpf98y7UOsvd89mpwVHchRB7hRCr5fYRAJsB9AdwHYApstoUANfL7esATBUKywB0I6J+gfecYZjMIhYKmfwkJidjv22uvbR1bJsbs9xrG5owuFdH9OzU1r5RG0IZarnHIKJ8AGcBWA6grxBirzy0D0Bfud0fwC7NaeWyzNjWXURUQEQFVVVVHrvNMEwm4Ca3jJWGJzOg2iZkL23dO+TF4txr68O6Gap+ZFrvc/fRQApwLe5E1AnApwAeEEIc1h4TitPMkxtMCPGGEGKkEGJk7969vZzKMEygJDHNX/6NCoG6xiaT42bpB5zrOJHsgGq3Dm1ice61DU16cfchzpnqcwcRtYEi7O8LIabJ4grV3SL/Vsry3QAGak4fIMsYhmlJDHoXpBw9M3sLRjw+F/XhCP66YCu+LlLkQD9D1cLn7iOdu6O4OzwvunfMQ4gIVUcaULirBh11lnu8g98b0BUPXT3cvjEYfe7pIfRuomUIwFsANgshXtAcmg5gotyeCOBzTfktMmpmNIAajfuGYZg048V5WwNrqyEcxZ/nFePWd1YmHBNWPneLtuwnMdn3w+7w5/eMQae2uQiFgOraBjRGomjXJlEK/3DtqZh+74X49dih9heDP8t99JAens/xgptUaGMA/BzAeiJaK8seBTAZwMdEdAeAnQBuksdmAbgGQAmAOgC3BdpjhmH8YaE/M9fvxSs+mwwizt1KxJNJP2Bn2Z8xsBsA/UOmtkHjUpLFPz5X64CwJ0fj43fj1il+cnzKXTmO4i6EWAzrN7hxJvUFgHuS7BfDMBmI0ddt6nNvhlBIr2587apO0+6+AF8U7nGMuNGSq0s/4Hxenoe2/ZJ9SYwZhmkxjMkVVQ3efegY6uQkISN+Ugk4neJmEY7yg+YLg5/WvytO69/VU3+0VnjX9m08nZsqWNwZhgkMY+pc7d4HK8oAmOWW8b6KqnO0jDNaaz3ZXDVGF8vvJ5yC/t3aJ9dokrC4MwwTGEbRdZNaoMki3W4yce5u0rNr/ezJpjMwivudFw1JssXk4cRhDNNaaIaU5XaWuwoB2LinBsu27wcAa3eNzXWcHhpuHiqn9OuiPcGxvn1bnZM6PxWwuDMM45kPVpRh3qaKBJdKgrhb5HOf8PJi3PzGMgDxhF1ecLLMZ6xzjr5++eaz8IMzjlf66bkHeob26YxXf5peORJZ3BmmtRBg5N0j09bjF1MLEsoTfeHOM1SPama2lh+sw7xNFcqZdm6ZAF5DunfMw50XDna8lls6tUsvLzeLO8O0UoLIO/7OklLdvtFy3yJzpGtpjOgXo9amLfj+XxfHHhp2Ah7UetbqRxDEw6JHRyW75LC+nZJuKwjS61HDMExGo7Xc99c24LHPNiTUeXrWFt2+1ud+qE6JYBFCJBfn7pEgmjv1+K74+Jfn40w5SaqlYcudYbKQ3YeO4amZmxB1EzYSIFqj/FjY2Ze+pKQaryzcBkC/9qjxDSBVqMvjBfWsGDW4R7NMUHJDevSCYZhAuf+DNfi/b3egsPxQvNCHgD3xxSb87uNC1/W1otzY5Ow7+emby7F5r5JkVl00AwDCEb3l3tCkf1AEZbn36aLkcL/q1OMCaS+dYHFnmCwkbGP5enG1v71kBz5dXe66vlZ0jb51J7RT+BsjUZ0ffPjv5xiu46lp/OXmM03L+3RuhzX/fQV+c5lzcrBMg8WdYVoZAburdXi13LVo+9UUiTpMYnK+ic7tctG7c9vYthXdO+al5UpKycLizjCthWbQrx3VR2PbXsVdO2M0bDFrVcWN5f7yzWfhLDm4qfblv65yzs2eLXC0DMNkI6k0z2144KO1se3kxN3+XDeWe15uCG3b5AAAGpqiKJ08wVN/Mh0Wd4bJYoKIZfdLg0efu5aL/rQQFw7tZXnczaMrLzeEPJl+1+uDJhtgtwzDZAHRqMDna3ebhhDO3bgPh+vDJmellsPHkrvm4pJqy2NuQjzzckLo1kFJvxtKl1WrmxG23BkmC/jXql14+NP12F/biNvllHoAWLZ9PybP3oKxw3tjeN/mTW5VuKsm8DYPHG3Ei/OKdWueWhEiwoNXnITO7XJx7ZnHB96XdIfFnWGygOraRgBA5ZEGAHG3xeTZymzQzXsPx8Q9NuU+xX75iIccASFyHiRdtfMAHvtsg2lKAzOOhSPo2DYXD1x+kut+ZBPslmGYLMDJ63CkXpO7XOj/eqFsf53rWa8RDxfIdbE03Q9fXepK2NU1Uof07uj6+tkIW+4M0wqoa4wkhEJ61fYd1Udx6fNf44HLh7mq7yWFQE6APvGrTz0On98zJrD2MhW23Bkmi1BndbqRSq9umT2HjgEA5mzY56q+F3H3q+2Tbzg9oSzopGKZCos7w2QBBOO6pM54lUBVNN36vL1EQrbRuGXye3ZwfZ7ZzNLWGPZoBos7w2QTHhTbq4XrNZ+Ll/a1Me1Tbz/P9XmdTaJmGljcAbC4M0zrwaC1Xr0XXt04TV587hoL3C4PjJbzh/TE1aclZnM0ZpBsrbC4M0wWoPqsaxuaUFNnP3koHgrpvv39tQ2od5GfXYuXXPLajJC5Oe4c8NecfpzpDNz6MFvuAEfLMExW8f7yMry/vAzfG9DVso6f8cZznpzv+RxP0TIacW/jIiwSgOUoLFvuCmy5M0wWYJQ5MwFftuOAoU6KJzF5inPXWO4u0+9aVWOfuwJb7gyTwUSiAl8XVboaRy3cpazKtL2qFgCwp6Y+hT1zdssQxR9CWss9x6W4GyOEVO69NPsW3vADW+4Mk8G8vXgH7phSgNkuY88B4F+rnFdW2l/b4Jh21wmnAVWtNOeG4lLkNpOlWbWrTu2LU/p1cXV+tsOWO8NkMOUH6wAAVYeDs8Ibm6I458n5uPGcAUm14xQKGSKK1XFrrWtpfXkevcGWO8NkCBWH65E/aSYWFlXGylKRr11d+3Tmur1JtdPksJqStutu/exazNL4WrlqWiOO4k5EbxNRJRFt0JT1IKJ5RLRV/u0uy4mIXiaiEiJaR0Rnp7LzDNOaWCt95u8vK2uW6x3zGPpoxMly1wqxH8udsceN5f4ugKsNZZMALBBCDAOwQO4DwHgAw+S/uwC8Gkw3GYaZXrgHgLs8LMn4y4OKonEMhUzScmfscRR3IcQiAAcMxdcBmCK3pwC4XlM+VSgsA9CNiPoF1VmGac14cZM45X+xi2TxkIbdlrCDuGv13E3KX8Ybfj/RvkII9Ze2D0Bfud0fwC5NvXJZlgAR3UVEBURUUFVV5bMbDJNdzN9UgTVlB23rBGHj2sWge4lPt72Gw1NC6zP35ZYxOaUVrqZnSdKPS6G8w3n+NQgh3hBCjBRCjOzdu3ey3WCYrODOqQX4j79/Z1snCAGzc5kElTLXcUBVs+1L3Dmzry1+xb1CdbfIv+rw/W4AAzX1BsgyhmECwiwixGvUjFHAN+05HD/mNf2jBU5+f63lbuVzn/GbCwPpS2vEr7hPBzBRbk8E8Lmm/BYZNTMaQI3GfcMwTAAEYbkbJxhd8/K3AIDKI/X45Xurkr+AyTWMaO/DynLv26Wdp2uyWyaO4yQmIvoAwFgAvYioHMD/AJgM4GMiugPATgA3yeqzAFwDoARAHYDbUtBnhmlV7Kupx6Ji83Epv2JmZZ2/OK8Ya8oO+WvUgJNbRouVuNveHwu5LY7iLoT4icWhcSZ1BYB7ku0UwzBxJr69AkUV8eiXVPrcN+w+bFruhyanAVUX+WTMJirFMLkFnsQUh+OPGCbNqapt0O1rBcyvmJlFxFQdacD63TW+2jMj7GC5a4XbSsRZqv3T6sU9GhVYWWoM42eYNMZE8XbLxavdYmZU1zU2+eyQOU0OA6q6aBkLcbe13BlbWr24T1laih+9thQLt1Q61k0HPlxRhhGPz/G0EALDGDFzmXhZFs+M28bke2pPG+FjttC1UimpLrVqWr24l1Qqua3V7HrpzuOfb0RdYyTpdKxM5mDUN+2+/wHVxLI7pxT4a0zSr6s+ssVL4jCrCapsuPun1Yt7pv14gppgwiTyzKzNuO5vi1u6G44EkQnSzOe+o/poUm0amww7DKhq70J1v3TIy9HV8eyWybD/z6mE87lLMkUyVXFnt0zwvL5oe0t3wRVkse2W/t3ap+T3Y2zRyQ7RCrc6+NqlXRtdHdv7YyG3pdWLuxptENSsvFSjdjOo/B9M5pHq9APNhfY++nZpi5+PHoSfnz9IV8d7KCSj0urdMuo4zhdJLkzQ3Ah2ubcagnYdRoVIjeUugMtO7uO6vtEt88frT8NJfTvr67Ba+6ZVifvBo42Ytlq/fqTqv1y10z4TX7qRLpZ7bUMTPlpZFlgO8CBZsLkCZ/zhSxxrTG7RiXSDoLxphiNRX+IXFSIlYzcCAoePhV3XT8UqUqloM1NpVeL+24/X4sGPC2Orv2cyQVtef12wFasdUs2a8cyszXj40/VYUrLf03lCCKwvD27CjBlPzdqMmmNh7D6UGZFQbiEi/GJqAYY9NtvX+UKkxi0jBDC0TyfX9TVrYlu6U2zdMiaH0tHIaCmyWtw/XFGGS5//OrZfLWf6HW3IfEvO649YCGEbPvnnecW4wSLVrN15qlW8p8bbJJpPV+/GD/62GHM37vN0nhfUcZRMnggjhEiIFycAC5KYlxEVqXnzE0Lgf6891XV9N7Nrbb861nFbslrcJ01brwvvypPBtI2RuLhn6v/7ORv34cMV8bU0a46FUbbf2kJ9b9lODHtsNiqP1Hu6zvxNFRj22GxdSlgt3TvmAVBcXl7YvFdpz67PyaIKWCavzzl59hYcqrN2dfhxQ1TXNmBtQMnBtAgBtGuT41xR4qbrXh/M7JaJk1Xi/srCEvy//1uWUK5acHm5yu02NNmPRj4zazPuNqQ9jURFWr3yPf75Rkyath75k2biWGMEP3z1O1z83ELL+p+tUdLqm4mp3Sv6V0WKhWjlsmkv/zM3NEU9veqrdZMRXqfrqWHWzfG1RVP0+5iytDSx0ONHdvFJiYvhPDFjk6/+2OH17t0IN0u1f7JK3J+bW4TvtiX6flULLi9XEaJGjbibvRq+vmg7Zm+IuwuO1Idx4rQknCIAABbBSURBVKOz8No36RkH/cSMTbGZtlaCp65RaTYl3M7toub8cBLSBZsrcOKjs1BcYb92p4o6/T03J/Hzn7q0FG8t3mF7/r6aepz46Cw8+PFay7rqNZKdVu9EY1MUQx6dhee/LLKtV1MXxn//ewPqw+7dgma/zzYaZ7Ub8bt0ePOsdOb12eam715T/vLDIE5WibsVkahAY1M07pbRiruLX8P+WsXl8NHKMtPjy7fvx+inF6C2IZjES1srjmDFDiWZmRDOFuE+jb/77cU7TOurK92YTQm3Ez/VsnYS90I5OLp2l7vXfSvLfUf1UTz++Ub80cGyVAfFp63ebVlXvUaqZ/Wq4w5Tl+60/b6e+3IL/rFsJz5f635xMrMXG7MHoh1WqxwlQxuTPggb233q7aMSynT/96xSy3iMc2fitApxLztQh5N+PxvzN1cAcHbLGFF/Q1Y/tD/NLcK+w/XYslfvl66ubUDVkQbTc+y44sVFuOn1pQCAwY/Mwo2vLbWtHxVxEXhq1mbM3ViRUEcVUdWa/XRVOV5ZWKKUaSz3pkgUT3yxCZWHFd+8+uoctECqDxljNsDqWneflxtjXBV3L4tG+EH9bAjAi/O3YvAjs9DQlGid18mB/JyQ+/92Zr85rVjvdzHW4eV6bjF72Nv9RM4Y0C2hLDWhkIE3mbG0CnE3JgU7pnktDvK3MGPdXhyujw9+jXxyPs59ar5l/UhU4MoXv3GMGHGKwY8KgTaazEtH6vUDcIu3VuPbrdUA4kL3u38V4rm5RVhSUq3Lu71k2368vWQHHvv3BgDxhE5WlrudtWaH2p4xasPv92FmLceukWK3jJpDJRQivLtEcRHVNyYaEKpR0TbX/L+d9h4e/mQdnpm12fTzmLJ0Z2z7k1XlJjX0eLX03aD9SEcN7qE7lt+zQ0L9HJM+JP1CIc//5Ffnx8bTmDgZ/YnM2bAXd7y7Eo1NUdvc0Y0GS13r87R70k+evQWVR+rxpMvBp3e/K8Vjn21wVRdQRLi4ohb/9a9C1+eYERXCdmDyf7/YGNs2pnot3X9UV6Zaoepnplp9RtfNwqJKfLxyl6d+HqkPY4/MO662Z/xu3FpexoeK2duY2mWnFYHKD9bhqMaltq2qFrUNTcifNBN/XbDV8ryC0gOY8l1p7OEYIor1yuyhp1rzqrgXVxxB/qSZsXTT2o/4o4JdeH3R9kAs0d6d2ybfiA3qT0/t/pwHLkb3DvocMWb52oNaNWlkfg88+8PTA2krm8hocS8/eAwLtlSivimCP88rtqxXH9b/565zOWPxtW+24Ya/fxeLKXbzU1TdGVqK9h3R+aLv+2ANpnxXGvvPnKzHIxrV/+exe901ro4TItK5LdQz1RLVcjfm3rntnZV46NN1nvp+7d+W4ILJXwGIW9NGcf/Ve6tdtWU0xhvCiQKuDhQ7We4XPrtQF2U17s/f4JFp6wEoD2wzfvWPVbjxtaX4n+kbEZb3oP3UF22tTnhjVB9AqiW9XI6rfLmpQt5Tat4w2uXm4EfnDEhJ24Am6kX2v12bHHRtrxd3M89Q0g8u9rnbktHi3laG4X3vf7/Eq19vs6xnjE74cEUZ8ifNxL6aeke/n5up69rX6fZ5iXG+V720CNe/sgTPzd2CpkgU0wv3KKKgio9Q3DMXPLPA8VpmRITQLXZAiOepN/YvHInq9kOkj5ZRPw8RixG3jrJRr+2G95fv1M05UK9pFHftGIWX0EIzH3ejvMaNry3VtfuHLzYif9JM3TUKDbNll5pEXWmZo3GlqfdCFLdF7/tgDa54YVGsTmn10dhvSf241YdCXo71uEZjAHn7BURKfdHqW6P2J2J8k3QKe+S1T4Mno8W9nQvfJZAo7qUy1ntbVa3jT0onXrLyxj012CDXmhRCYLVmQkh7m0kcryzcpptZqApbVAgUV9RiT029pXvJzj0QjerdMi8tKMblL3yDeZsSB1brw1FM1fhsiUhn2cYsd1mkCo+VyERcDlYa3VVqe9p2CwzLHdpZ3InfcWL/tFW0KSfeWVIaa+OYRViiKthmPTDG/Kv3YPSMqW3vq6nH2Oe/RoEcO1G/Y9VdpI6XmD3LzO7LD6mcpRszCDSfVq7BVDdzyyTdJw6FtCWjU/5azYaLCkA7fvN54R7Tem5+CGYunAkvKws6PHD5MLw0Xy+6VoNlKlrrdbvc1v4Htsop/ud5xfjNuGGmx6JC6IRl1wHFr11ccQRXjOirE6iGpojORRQi0rlqYm/Y0FvuB2rNozKsxNEOIUTsc21siuK6V5Zgr8kaoE1RATk1AUIIDH5kFn55yRBcOLQXbn1npa5uvYnlrsXs4TRz/V48P9c8Nl198GofImX76zBtTTm+Ka7S91N+flZaZYyYCsuHlvq5v7l4Bz5cuQsrHhtnew9uGNanE7ZWGnInidTO3FQfVtqHk3HZPLNl9OyCeKbePsqTmyqN5hemDVkp7k3RKHJC8WNrLKZaR00SKG01TMIxug20GIUdAP69dg+GH9cFFw3rZXrOvpq4T/73/16fcHyKhY/Xju3VR22nqGtpaIrq/xOSfsBRPab+jchj+4+au0veX66P/Z+zYR8e+mQdfnXJiZg0/mQAiQslhyMi5qJoaIqi0CI2PhyJol2bHDw7Zwt27lcehK9/sx2vm0wmc5oY9PO3VmDTE1ehQ178J3/vP9dY1lcfBlEBTC/cg5LKWrws3556ddIPUG6SIbAVh83DOH/78VrdvvqZan9btQ1NeG/ZTiTLvAcvwaz1e/Hr9+NjFwKpDRFUH/Da/0luYuvtXDFms2oTMBF0Tj8QJ6PF3Sr8qXhfLU4f0NXx/J+9tTyh7IoXF5nUVFB/Np3a5tpOWHp2zhY8O8f8mDa3i2ph64/HBUL1CzthJezPzS3C6CH6MLX6cERnEYWINJExFJ/VGRF4fm4R/iZj4edvrsQLXxahS/s2GJmvb1PLV9Lt9No322LibkzU1tAUQV2j8vlZDViqfQBgO54Svy+l328s2oZLh/fBMENecAAora7DiOO7OLYFxB/6NcfCuO8D/UOg0fCWoA6+miGE0I1/AMDzc4vx248K0a6N/vf79KwtrvpmxRPXWSftsnL3hch5zkDb3JDt3JDzBvfEmrJDuOrU42JlZtFbb986Ere/G1+nVavDxgFYJnky2udulTv6B39bjAkvf5uy63br4P+HOGu9/yyIRneAG56cuRnbq+KuoIamqM7giQoRe1BFonF3yYrSAzFhV3n5qxI8OXMzrn9liac+1DbqH4R1jRFsq3Jer/OsP84zjT4yoz4cQWNTFE/P2oIfvvqd6WDsvf90F4njxBEPM5EPH0usu1u6oLz6052sYa24ajm+W3vLCDF1jGhIr46W7Rpdjc/coA87HNSzA0onT8CZA+MTlczE/bKT+5peG4Drh64OzSXYLZNIRov792ys840WWQzt2LzX/pxtVUdx6uNzUH7QW3rboJj49grP5xhdUg1hfbRMdW0D9hyKC6idq8Irf/hiI0oqa3XpEQDgvKfdRwWNcln3qZmb8dUWZQC5tqEJe2sSHwrbq4+6fhuyw4uQHKjzli3TjmvPON72uCrCPWSmTpXBvTrG3FbGBajViDO7xazbakT4ujOPx09GnaA7fm5+94RznBLC3XfZUDws3+x6dcqzresFdsrEyWhxH9TT2trww/i/OFv7RzN8VZ+3l+zQvT08PWsLHv3M2q2QDO8sKcXlL3yDH75qnz4hCIoqjsRi5KMCsXj6lubvhrefZLj/8mF48cdnxPZf//k5uuOqm3L0kJ54+GpFOFWLXPWLd++gF9IrTlGs6X5d2ltet7dmjMEYEls6eQKG9kl0gV3i4DN/8Mrh6NxW8Qp3apu8d5gN90QyWtyTwejvTGeO79qupbsAwDkSiEnEjyvNikE9O2Jg9/jUfuOYU54mBcUFJ/YEAHSUwqkOYHfvqHcpPjz+ZHw36TK8ZnhQAMC3D12K0skT8MYt5+D0/spbshr66uSavPuSEy2P/Xqsckx1S3X0IO6Tb3CYicqme4xW+791/Gn9km5j1OAe+Oed58X2/eQm/+a/xjrWefwHI3T7zT34pIq6l4UYkuWBy83DPt1yzqDuKH5yvKvPN5VU+kgcp3JCjw4Yf5rej65NJTAqvwfO0+R1ydWIuxrto7poVJ/7Ly4aghH9usQeBEIIHN+tPXp0zEPxk+N11xrYQ3mQDOjeIfZ9qA+UOfdfjH/96nzLvodChCWTLksoL508AQ/Jtwp1rMeLuPfsZJ5KQc1SyQZInFbzSdwvY8Q3P3E1ZvzmQtxzaaJlMf3eMfjHHaNw/ZmKb9PudfH+ccPw8S/PxwVDe6FHxzycfFxnLHroUhQ+fmVCqNwlJ/XW+UEfvOKk2HZ3g3901OAeWPqI/j9F785t8fPRg2L7RxuaUPLUePx67IkofPxKPHn9abr6Rp/oyEGKT9Ru0EzlzgsHJ5TdJz+7m88dGCv77+/HHzhP/cdp6Kd5u7js5D6W7d80cgCm3zsGhY9fic7trD/fi0/qjVsvyI/t33L+oIQ6A7pbuxLevvVc5OWGMKhnR6x87HLMuu8iAEDPjsH5d72g/ezfmjgytm20vjf+4SoAwB+vOxWLHroUr/7sHLx727l47WdnA1Cs928fuhTbn74GHdvmYsrto/DtQ5di+aP6GPlzTuiO31w2FH++SXHjDJbXP3NgN8y6/yLMe/BiPHT1cJ1Y5uWG0MciD83Y4X1w76VDY8voHde1Hc61iZoCgP7d2mPzE1dbxu8P6a30aeL5+bbtrHhsHL75r7G4b9wwXH5Kn5j1fs6guK9/wun98OuxJ2LS+FNs22pNUDqsLjRy5EhRUFDgXNEEswGyH549AJ3a5mD2hn0QUES7X9dEIdDGA5dOnqA7JoTA37/ehufmFuFPN34P5wzqjo55udheVYsLhprHsKvsOlCH3/2rEBNO74cvN+3De3ecByGANxdvx9OztmDGby7Es3O24MoRffGz0YNw6zsr8U1xFf7+07NxzenKG0XlkXrc/d5qjB7SA/955XAASpjhn78sxoNXnITLR+gjDx6Ztg4zCvfivnHD8IuLh2BhUSU+W70b0wv3YOVjl2P5jv24YkRf/OCvi1FcUYsvf3sxVu08GAvjm33/RYhEBU7r3xVCCBARCncdQpf2bXBCjw6YvWEvJpzeTxdH/F1JNY7v1h75UjjeXrwD/1i2E5/efQHOe3o+whGB+Q9ejEenbcB/XjUcL80vxqTxJ+N7Mv3rwqJKvL+sDJv3HsbuQ8dw25h8rNhxADkhwkd3nZ+QyuHvX5fgT3OUSUcv/fhMEAH3f7gWbXIIY4b2wtdFVfjBGcfj12NPxCn9EqMvolGB8oPH8MBHa/DmxHMRjkRx55QCrJezjUf064LcHMKFQ3vh+rP64/dyVu2K0gOYfMPp6NyuDb7bVo2C0oM4b0gP7K9txITv9UPhrkM4rms73DZmMIr2HcFVL+nDaYf06oiZ912EJ2Zswkl9O+G2MYOxqLgKU5eW4vWfj0SIlCiqy07uY5q+Igj2HDqG4oojGDvc+sHLZB5EtEoIMdL0WCrEnYiuBvAXADkA3hRCTLarn4y41xwLoykSRX1TFF9u3IcR/brgvCE9XZ9fH46g6khD7BVUSyQqUH6wLrCB26ZIFOUHj8XEMNU0NEVQUdOAEzQpWI82NKG2oQl9uyiWdn04gv1HG9G/m7UV7IeaujAiQiREbpjR2BRFxeF60+/AjO1VtRjcqyOiAli/uwanHt8FuSHC9uqjOLF3J+99PRZGOBJNeOPyS01dGOFoFF8U7sFJfTtjjIMxwDB+aVZxJ6IcAMUArgBQDmAlgJ8IISzz5iYj7gzDMK0VO3FPhc99FIASIcR2IUQjgA8BXJeC6zAMwzAWpELc+wPQruJQLst0ENFdRFRARAVVVcGFizEMwzAtGC0jhHhDCDFSCDGyd+/mWZ2dYRimtZAKcd8NYKBmf4AsYxiGYZqJVIj7SgDDiGgwEeUBuBnA9BRch2EYhrEg8JS/QogmIroXwFwooZBvCyE2OpzGMAzDBEhK8rkLIWYBmJWKthmGYRhnWk36AYZhmNZEWqQfIKIqAH7XGOsFoDrA7rQ02XQ/fC/pCd9L+uL1fgYJIUzDDdNC3JOBiAqsZmhlItl0P3wv6QnfS/oS5P2wW4ZhGCYLYXFnGIbJQrJB3N9o6Q4ETDbdD99LesL3kr4Edj8Z73NnGIZhEskGy51hGIYxwOLOMAyThWS0uBPR1URUREQlRDSppfvjBBENJKKFRLSJiDYS0f2yvAcRzSOirfJvd1lORPSyvL91RHR2y95BIkSUQ0RriGiG3B9MRMtlnz+S+YVARG3lfok8nt+S/TZCRN2I6BMi2kJEm4no/Az/Xn4rf2MbiOgDImqXKd8NEb1NRJVEtEFT5vm7IKKJsv5WIpqYRvfynPydrSOiz4iom+bYI/JeiojoKk25d60TQmTkPyh5a7YBGAIgD0AhgBEt3S+HPvcDcLbc7gxlxaoRAP4EYJIsnwTgWbl9DYDZAAjAaADLW/oeTO7pQQD/BDBD7n8M4Ga5/RqAu+X2rwG8JrdvBvBRS/fdcB9TANwpt/MAdMvU7wXK+gk7ALTXfCe3Zsp3A+BiAGcD2KAp8/RdAOgBYLv8211ud0+Te7kSQK7cflZzLyOkjrUFMFjqW45frWvxH2ISH9r5AOZq9h8B8EhL98vjPXwOZTnCIgD9ZFk/AEVy+3UoSxSq9WP10uEflHTOCwBcBmCG/A9Wrfnhxr4jKInkzpfbubIetfQ9yP50lWJIhvJM/V7UBXN6yM96BoCrMum7AZBvEERP3wWAnwB4XVOuq9eS92I49h8A3pfbOg1Tvxe/WpfJbhlXKz6lK/LV9ywAywH0FULslYf2Aegrt9P9Hl8C8BCAqNzvCeCQEKJJ7mv7G7sXebxG1k8HBgOoAvCOdDG9SUQdkaHfixBiN4DnAZQB2Avls16FzPxuVLx+F2n9HWm4HcqbBxDwvWSyuGcsRNQJwKcAHhBCHNYeE8qjOe3jU4no+wAqhRCrWrovAZAL5dX5VSHEWQCOQnn1j5Ep3wsASH/0dVAeWscD6Ajg6hbtVIBk0ndhBxE9BqAJwPupaD+TxT0jV3wiojZQhP19IcQ0WVxBRP3k8X4AKmV5Ot/jGADXElEplEXQLwPwFwDdiEhNJa3tb+xe5PGuAPY3Z4dtKAdQLoRYLvc/gSL2mfi9AMDlAHYIIaqEEGEA06B8X5n43ah4/S7S+jsiolsBfB/AT+XDCgj4XjJZ3DNuxSciIgBvAdgshHhBc2g6AHU0fyIUX7xafouMCBgNoEbzatqiCCEeEUIMEELkQ/nsvxJC/BTAQgA3ymrGe1Hv8UZZPy2sLyHEPgC7iGi4LBoHYBMy8HuRlAEYTUQd5G9OvZ+M+240eP0u5gK4koi6yzeZK2VZi0NEV0NxZ14rhKjTHJoO4GYZvTQYwDAAK+BX61py0CSAgYproEScbAPwWEv3x0V/L4TyOrkOwFr57xoo/s0FALYCmA+gh6xPAF6R97cewMiWvgeL+xqLeLTMEPmDLAHwLwBtZXk7uV8ijw9p6X4b7uFMAAXyu/k3lAiLjP1eAPwBwBYAGwD8A0oERkZ8NwA+gDJWEIbyVnWHn+8Cij+7RP67LY3upQSKD13VgNc09R+T91IEYLym3LPWcfoBhmGYLCST3TIMwzCMBSzuDMMwWQiLO8MwTBbC4s4wDJOFsLgzDMNkISzuDMMwWQiLO8MwTBby/wE7yFAAfy7c2wAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W6vLJACbE6Fu"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}