Deep Reinforcement Learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 0. Install Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow==2.3.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (2.3.0)\n",
      "Requirement already satisfied: protobuf>=3.9.2 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (3.12.2)\n",
      "Requirement already satisfied: gast==0.3.3 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (0.3.3)\n",
      "Requirement already satisfied: scipy==1.4.1 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (1.4.1)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (3.1.0)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (0.9.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (1.1.0)\n",
      "Requirement already satisfied: tensorflow-estimator<2.4.0,>=2.3.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (2.3.0)\n",
      "Requirement already satisfied: numpy<1.19.0,>=1.16.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (1.17.2)\n",
      "Requirement already satisfied: wrapt>=1.11.1 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (1.11.2)\n",
      "Requirement already satisfied: keras-preprocessing<1.2,>=1.1.1 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (1.1.2)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (1.16.1)\n",
      "Requirement already satisfied: tensorboard<3,>=2.3.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (2.3.0)\n",
      "Requirement already satisfied: wheel>=0.26 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (0.33.6)\n",
      "Requirement already satisfied: astunparse==1.6.3 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (1.6.3)\n",
      "Requirement already satisfied: six>=1.12.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (1.12.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.8 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (0.2.0)\n",
      "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow==2.3.0) (2.10.0)\n",
      "Requirement already satisfied: setuptools in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from protobuf>=3.9.2->tensorflow==2.3.0) (41.4.0)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3.0) (1.18.0)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3.0) (2.22.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3.0) (0.4.1)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3.0) (1.6.0.post3)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3.0) (0.16.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3.0) (3.1.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow==2.3.0) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3\" in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow==2.3.0) (4.6)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow==2.3.0) (4.1.0)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow==2.3.0) (1.24.2)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow==2.3.0) (2.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow==2.3.0) (2019.9.11)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow==2.3.0) (3.0.4)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow==2.3.0) (1.3.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow==2.3.0) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow==2.3.0) (3.1.0)\n",
      "Requirement already satisfied: gym in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (0.17.2)\n",
      "Requirement already satisfied: numpy>=1.10.4 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from gym) (1.17.2)\n",
      "Requirement already satisfied: scipy in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from gym) (1.4.1)\n",
      "Requirement already satisfied: cloudpickle<1.4.0,>=1.2.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from gym) (1.2.2)\n",
      "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from gym) (1.5.0)\n",
      "Requirement already satisfied: future in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from pyglet<=1.5.0,>=1.4.0->gym) (0.17.1)\n",
      "Requirement already satisfied: keras in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (2.4.3)\n",
      "Requirement already satisfied: numpy>=1.9.1 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from keras) (1.17.2)\n",
      "Requirement already satisfied: pyyaml in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from keras) (5.1.2)\n",
      "Requirement already satisfied: h5py in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from keras) (2.10.0)\n",
      "Requirement already satisfied: scipy>=0.14 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from keras) (1.4.1)\n",
      "Requirement already satisfied: six in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from h5py->keras) (1.12.0)\n",
      "Requirement already satisfied: keras-rl2 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (1.0.4)\n",
      "Requirement already satisfied: tensorflow>=2.1.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from keras-rl2) (2.3.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (3.1.0)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (1.16.1)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (0.9.0)\n",
      "Requirement already satisfied: tensorflow-estimator<2.4.0,>=2.3.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (2.3.0)\n",
      "Requirement already satisfied: wheel>=0.26 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (0.33.6)\n",
      "Requirement already satisfied: six>=1.12.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (1.12.0)\n",
      "Requirement already satisfied: numpy<1.19.0,>=1.16.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (1.17.2)\n",
      "Requirement already satisfied: google-pasta>=0.1.8 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (0.2.0)\n",
      "Requirement already satisfied: gast==0.3.3 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (0.3.3)\n",
      "Requirement already satisfied: keras-preprocessing<1.2,>=1.1.1 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (1.1.2)\n",
      "Requirement already satisfied: scipy==1.4.1 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (1.4.1)\n",
      "Requirement already satisfied: tensorboard<3,>=2.3.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (2.3.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: wrapt>=1.11.1 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (1.11.2)\n",
      "Requirement already satisfied: astunparse==1.6.3 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (1.6.3)\n",
      "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (2.10.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (1.1.0)\n",
      "Requirement already satisfied: protobuf>=3.9.2 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorflow>=2.1.0->keras-rl2) (3.12.2)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (1.6.0.post3)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (2.22.0)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (1.18.0)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (41.4.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (0.4.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (3.1.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (0.16.0)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (2.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (2019.9.11)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (1.24.2)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (3.0.4)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3\" in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (4.6)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (4.1.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (0.2.8)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (1.3.0)\n",
      "Requirement already satisfied: pyasn1>=0.1.3 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from rsa<5,>=3.1.4; python_version >= \"3\"->google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow>=2.1.0->keras-rl2) (3.1.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow==2.3.0\n",
    "!pip install gym\n",
    "!pip install keras\n",
    "!pip install keras-rl2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Test Random Environment with OpenAI Gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym \n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "states = env.observation_space.shape[0]\n",
    "actions = env.action_space.n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode:1 Score:28.0\n",
      "Episode:2 Score:18.0\n",
      "Episode:3 Score:11.0\n",
      "Episode:4 Score:13.0\n",
      "Episode:5 Score:11.0\n",
      "Episode:6 Score:16.0\n",
      "Episode:7 Score:11.0\n",
      "Episode:8 Score:15.0\n",
      "Episode:9 Score:14.0\n",
      "Episode:10 Score:30.0\n"
     ]
    }
   ],
   "source": [
    "episodes = 10\n",
    "for episode in range(1, episodes+1):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    score = 0 \n",
    "    \n",
    "    while not done:\n",
    "        env.render()\n",
    "        action = random.choice([0,1])\n",
    "        n_state, reward, done, info = env.step(action)\n",
    "        score+=reward\n",
    "    print('Episode:{} Score:{}'.format(episode, score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Create a Deep Learning Model with Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Flatten\n",
    "from tensorflow.keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(states, actions):\n",
    "    model = Sequential()\n",
    "    model.add(Flatten(input_shape=(1,states)))\n",
    "    model.add(Dense(24, activation='relu'))\n",
    "    model.add(Dense(24, activation='relu'))\n",
    "    model.add(Dense(actions, activation='linear'))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = build_model(states, actions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_1 (Flatten)          (None, 4)                 0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 24)                120       \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 24)                600       \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 2)                 50        \n",
      "=================================================================\n",
      "Total params: 770\n",
      "Trainable params: 770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Build Agent with Keras-RL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rl.agents import DQNAgent\n",
    "from rl.policy import BoltzmannQPolicy\n",
    "from rl.memory import SequentialMemory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_agent(model, actions):\n",
    "    policy = BoltzmannQPolicy()\n",
    "    memory = SequentialMemory(limit=50000, window_length=1)\n",
    "    dqn = DQNAgent(model=model, memory=memory, policy=policy, \n",
    "                  nb_actions=actions, nb_steps_warmup=10, target_model_update=1e-2)\n",
    "    return dqn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training for 50000 steps ...\n",
      "Interval 1 (0 steps performed)\n",
      "10000/10000 [==============================] - 49s 5ms/step - reward: 1.0000\n",
      "51 episodes - episode_reward: 193.118 [58.000, 200.000] - loss: 5.376 - mae: 39.195 - mean_q: 78.705\n",
      "\n",
      "Interval 2 (10000 steps performed)\n",
      "10000/10000 [==============================] - 49s 5ms/step - reward: 1.0000\n",
      "54 episodes - episode_reward: 185.056 [64.000, 200.000] - loss: 7.553 - mae: 40.527 - mean_q: 81.044\n",
      "\n",
      "Interval 3 (20000 steps performed)\n",
      "10000/10000 [==============================] - 51s 5ms/step - reward: 1.0000\n",
      "52 episodes - episode_reward: 193.462 [33.000, 200.000] - loss: 8.935 - mae: 40.588 - mean_q: 81.322\n",
      "\n",
      "Interval 4 (30000 steps performed)\n",
      "10000/10000 [==============================] - 53s 5ms/step - reward: 1.0000\n",
      "50 episodes - episode_reward: 200.000 [200.000, 200.000] - loss: 11.357 - mae: 41.679 - mean_q: 83.338\n",
      "\n",
      "Interval 5 (40000 steps performed)\n",
      "10000/10000 [==============================] - 54s 5ms/step - reward: 1.0000\n",
      "done, took 256.117 seconds\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7ff0e46ad650>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dqn = build_agent(model, actions)\n",
    "dqn.compile(Adam(lr=1e-3), metrics=['mae'])\n",
    "dqn.fit(env, nb_steps=50000, visualize=False, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing for 100 episodes ...\n",
      "Episode 1: reward: 200.000, steps: 200\n",
      "Episode 2: reward: 200.000, steps: 200\n",
      "Episode 3: reward: 200.000, steps: 200\n",
      "Episode 4: reward: 200.000, steps: 200\n",
      "Episode 5: reward: 200.000, steps: 200\n",
      "Episode 6: reward: 200.000, steps: 200\n",
      "Episode 7: reward: 200.000, steps: 200\n",
      "Episode 8: reward: 200.000, steps: 200\n",
      "Episode 9: reward: 200.000, steps: 200\n",
      "Episode 10: reward: 200.000, steps: 200\n",
      "Episode 11: reward: 200.000, steps: 200\n",
      "Episode 12: reward: 200.000, steps: 200\n",
      "Episode 13: reward: 200.000, steps: 200\n",
      "Episode 14: reward: 200.000, steps: 200\n",
      "Episode 15: reward: 200.000, steps: 200\n",
      "Episode 16: reward: 200.000, steps: 200\n",
      "Episode 17: reward: 200.000, steps: 200\n",
      "Episode 18: reward: 200.000, steps: 200\n",
      "Episode 19: reward: 200.000, steps: 200\n",
      "Episode 20: reward: 200.000, steps: 200\n",
      "Episode 21: reward: 200.000, steps: 200\n",
      "Episode 22: reward: 200.000, steps: 200\n",
      "Episode 23: reward: 200.000, steps: 200\n",
      "Episode 24: reward: 200.000, steps: 200\n",
      "Episode 25: reward: 200.000, steps: 200\n",
      "Episode 26: reward: 200.000, steps: 200\n",
      "Episode 27: reward: 200.000, steps: 200\n",
      "Episode 28: reward: 200.000, steps: 200\n",
      "Episode 29: reward: 200.000, steps: 200\n",
      "Episode 30: reward: 200.000, steps: 200\n",
      "Episode 31: reward: 200.000, steps: 200\n",
      "Episode 32: reward: 200.000, steps: 200\n",
      "Episode 33: reward: 200.000, steps: 200\n",
      "Episode 34: reward: 200.000, steps: 200\n",
      "Episode 35: reward: 200.000, steps: 200\n",
      "Episode 36: reward: 200.000, steps: 200\n",
      "Episode 37: reward: 200.000, steps: 200\n",
      "Episode 38: reward: 200.000, steps: 200\n",
      "Episode 39: reward: 200.000, steps: 200\n",
      "Episode 40: reward: 200.000, steps: 200\n",
      "Episode 41: reward: 200.000, steps: 200\n",
      "Episode 42: reward: 200.000, steps: 200\n",
      "Episode 43: reward: 200.000, steps: 200\n",
      "Episode 44: reward: 200.000, steps: 200\n",
      "Episode 45: reward: 200.000, steps: 200\n",
      "Episode 46: reward: 200.000, steps: 200\n",
      "Episode 47: reward: 200.000, steps: 200\n",
      "Episode 48: reward: 200.000, steps: 200\n",
      "Episode 49: reward: 200.000, steps: 200\n",
      "Episode 50: reward: 200.000, steps: 200\n",
      "Episode 51: reward: 200.000, steps: 200\n",
      "Episode 52: reward: 200.000, steps: 200\n",
      "Episode 53: reward: 200.000, steps: 200\n",
      "Episode 54: reward: 200.000, steps: 200\n",
      "Episode 55: reward: 200.000, steps: 200\n",
      "Episode 56: reward: 200.000, steps: 200\n",
      "Episode 57: reward: 200.000, steps: 200\n",
      "Episode 58: reward: 200.000, steps: 200\n",
      "Episode 59: reward: 200.000, steps: 200\n",
      "Episode 60: reward: 200.000, steps: 200\n",
      "Episode 61: reward: 200.000, steps: 200\n",
      "Episode 62: reward: 200.000, steps: 200\n",
      "Episode 63: reward: 200.000, steps: 200\n",
      "Episode 64: reward: 200.000, steps: 200\n",
      "Episode 65: reward: 200.000, steps: 200\n",
      "Episode 66: reward: 200.000, steps: 200\n",
      "Episode 67: reward: 200.000, steps: 200\n",
      "Episode 68: reward: 200.000, steps: 200\n",
      "Episode 69: reward: 200.000, steps: 200\n",
      "Episode 70: reward: 200.000, steps: 200\n",
      "Episode 71: reward: 200.000, steps: 200\n",
      "Episode 72: reward: 200.000, steps: 200\n",
      "Episode 73: reward: 200.000, steps: 200\n",
      "Episode 74: reward: 200.000, steps: 200\n",
      "Episode 75: reward: 200.000, steps: 200\n",
      "Episode 76: reward: 200.000, steps: 200\n",
      "Episode 77: reward: 200.000, steps: 200\n",
      "Episode 78: reward: 200.000, steps: 200\n",
      "Episode 79: reward: 200.000, steps: 200\n",
      "Episode 80: reward: 200.000, steps: 200\n",
      "Episode 81: reward: 200.000, steps: 200\n",
      "Episode 82: reward: 200.000, steps: 200\n",
      "Episode 83: reward: 200.000, steps: 200\n",
      "Episode 84: reward: 200.000, steps: 200\n",
      "Episode 85: reward: 200.000, steps: 200\n",
      "Episode 86: reward: 200.000, steps: 200\n",
      "Episode 87: reward: 200.000, steps: 200\n",
      "Episode 88: reward: 200.000, steps: 200\n",
      "Episode 89: reward: 200.000, steps: 200\n",
      "Episode 90: reward: 200.000, steps: 200\n",
      "Episode 91: reward: 200.000, steps: 200\n",
      "Episode 92: reward: 200.000, steps: 200\n",
      "Episode 93: reward: 200.000, steps: 200\n",
      "Episode 94: reward: 200.000, steps: 200\n",
      "Episode 95: reward: 200.000, steps: 200\n",
      "Episode 96: reward: 200.000, steps: 200\n",
      "Episode 97: reward: 200.000, steps: 200\n",
      "Episode 98: reward: 200.000, steps: 200\n",
      "Episode 99: reward: 200.000, steps: 200\n",
      "Episode 100: reward: 200.000, steps: 200\n",
      "200.0\n"
     ]
    }
   ],
   "source": [
    "scores = dqn.test(env, nb_episodes=100, visualize=False)\n",
    "print(np.mean(scores.history['episode_reward']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing for 15 episodes ...\n",
      "Episode 1: reward: 200.000, steps: 200\n",
      "Episode 2: reward: 200.000, steps: 200\n",
      "Episode 3: reward: 200.000, steps: 200\n",
      "Episode 4: reward: 200.000, steps: 200\n",
      "Episode 5: reward: 200.000, steps: 200\n",
      "Episode 6: reward: 200.000, steps: 200\n",
      "Episode 7: reward: 200.000, steps: 200\n",
      "Episode 8: reward: 200.000, steps: 200\n",
      "Episode 9: reward: 200.000, steps: 200\n",
      "Episode 10: reward: 200.000, steps: 200\n",
      "Episode 11: reward: 200.000, steps: 200\n",
      "Episode 12: reward: 200.000, steps: 200\n",
      "Episode 13: reward: 200.000, steps: 200\n",
      "Episode 14: reward: 200.000, steps: 200\n",
      "Episode 15: reward: 200.000, steps: 200\n"
     ]
    }
   ],
   "source": [
    "_ = dqn.test(env, nb_episodes=15, visualize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Reloading Agent from Memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "dqn.save_weights('dqn_weights.h5f', overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "del model\n",
    "del dqn\n",
    "del env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "actions = env.action_space.n\n",
    "states = env.observation_space.shape[0]\n",
    "model = build_model(states, actions)\n",
    "dqn = build_agent(model, actions)\n",
    "dqn.compile(Adam(lr=1e-3), metrics=['mae'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "dqn.load_weights('dqn_weights.h5f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing for 5 episodes ...\n",
      "WARNING:tensorflow:From /Users/nicholasrenotte/opt/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_v1.py:2070: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n",
      "Episode 1: reward: 200.000, steps: 200\n",
      "Episode 2: reward: 200.000, steps: 200\n",
      "Episode 3: reward: 200.000, steps: 200\n",
      "Episode 4: reward: 200.000, steps: 200\n",
      "Episode 5: reward: 200.000, steps: 200\n"
     ]
    }
   ],
   "source": [
    "_ = dqn.test(env, nb_episodes=5, visualize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}