cartpole_using_DQN_py.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "cartpole_using_DQN.py",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "[View in Colaboratory](https://colab.research.google.com/github/ZER-0-NE/Reinforcement-Learning_problems/blob/master/cartpole_using_DQN_py.ipynb)"
      ]
    },
    {
      "metadata": {
        "id": "AXKpcvaY9NWL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# This script makes use of OpenAI gym to train on the cartpole game.\n",
        "# Description of Game:\n",
        "\n",
        "# A pole is attached by an un-actuated joint to a cart, which moves along a frictionless track. \n",
        "# The system is controlled by applying a force of +1 or -1 to the cart. The pendulum starts upright, \n",
        "# and the goal is to prevent it from falling over. A reward of +1 is provided for every timestep that \n",
        "# the pole remains upright. The episode ends when the pole is more than 15 degrees from vertical, or \n",
        "# the cart moves more than 2.4 units from the center.\n",
        "# source : https://gym.openai.com/envs/CartPole-v1/\n",
        "\n",
        "\n",
        "# Parameters:\n",
        "\n",
        "# episodes - a number of games we want the agent to play.\n",
        "# gamma - aka decay or discount rate, to calculate the future discounted reward.\n",
        "# epsilon - aka exploration rate, this is the rate in which an agent randomly decides its action rather than prediction.\n",
        "# epsilon_decay - we want to decrease the number of explorations as it gets good at playing games.\n",
        "# epsilon_min - we want the agent to explore at least this amount.\n",
        "# learning_rate - Determines how much neural net learns in each iteration.\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "MFiDs2Sx9zEQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 415
        },
        "outputId": "ef23f5e3-adeb-428d-9136-abd2214c605a"
      },
      "cell_type": "code",
      "source": [
        "!pip install gym\n",
        "import random\n",
        "import gym\n",
        "import numpy as np\n",
        "from collections import deque\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "from keras.optimizers import Adam"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting gym\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9b/50/ed4a03d2be47ffd043be2ee514f329ce45d98a30fe2d1b9c61dea5a9d861/gym-0.10.5.tar.gz (1.5MB)\n",
            "\u001b[K    100% |████████████████████████████████| 1.5MB 8.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.10.4 in /usr/local/lib/python3.6/dist-packages (from gym) (1.14.5)\n",
            "Requirement already satisfied: requests>=2.0 in /usr/local/lib/python3.6/dist-packages (from gym) (2.18.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from gym) (1.11.0)\n",
            "Collecting pyglet>=1.2.0 (from gym)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1c/fc/dad5eaaab68f0c21e2f906a94ddb98175662cc5a654eee404d59554ce0fa/pyglet-1.3.2-py2.py3-none-any.whl (1.0MB)\n",
            "\u001b[K    100% |████████████████████████████████| 1.0MB 10.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym) (1.22)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym) (2018.8.24)\n",
            "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym) (2.6)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyglet>=1.2.0->gym) (0.16.0)\n",
            "Building wheels for collected packages: gym\n",
            "  Running setup.py bdist_wheel for gym ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/cb/14/71/f4ab006b1e6ff75c2b54985c2f98d0644fffe9c1dddc670925\n",
            "Successfully built gym\n",
            "Installing collected packages: pyglet, gym\n",
            "Successfully installed gym-0.10.5 pyglet-1.3.2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "f-k4t1_2-MXB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "### Inspired from the post of keon.io/deep-q-learning/\n",
        "\n",
        "Episodes = 1000\n",
        "\n",
        "'''\n",
        "By defining memory, we make sure that the state,action.reward and next_state\n",
        "are remembered, as the neural network in DQN tends to forget them after each \n",
        "iteration.\n",
        "'''\n",
        "\n",
        "class DQNAgent:\n",
        "  def __init__(self, state_size, action_size):\n",
        "    self.state_size = state_size\n",
        "    self.action_size = action_size\n",
        "    self.memory = deque(maxlen=2000)\n",
        "    self.gamma = 0.95 # discount rate\n",
        "    self.epsilon = 1.0 #exploration rate\n",
        "    self.epsilon_min = 0.01\n",
        "    self.epsilon_decay = 0.995\n",
        "    self.learning_rate = 0.001 \n",
        "    self.model = self._build_model()\n",
        "    \n",
        "    \n",
        "  def _build_model(self):\n",
        "    '''\n",
        "    Neural Network for DQN\n",
        "    '''\n",
        "    model = Sequential()\n",
        "    model.add(Dense(24, input_dim = self.state_size, activation = 'relu'))\n",
        "    model.add(Dense(24, activation = 'relu'))\n",
        "    model.add(Dense(self.action_size, activation='linear'))\n",
        "    \n",
        "    model.compile(loss = 'mse', optimizer = Adam(lr = self.learning_rate))\n",
        "    \n",
        "    return model\n",
        "  \n",
        "  def remember(self, state, action, reward, next_state, done):\n",
        "    '''\n",
        "    Keep appending the memory\n",
        "    '''\n",
        "    self.memory.append((state, action, reward, next_state, done))\n",
        "    \n",
        "  def act(self, state):\n",
        "    '''\n",
        "    The agent will select at first it's action at random because \n",
        "    it is better for the agent to try all kinds of things before \n",
        "    it starts to see the patterns. \n",
        "    '''\n",
        "    if np.random.rand() <= self.epsilon:\n",
        "      return random.randrange(self.action_size)\n",
        "    act_values = self.model.predict(state)\n",
        "    return np.argmax(act_values[0]) # argmax picks the highest value among\n",
        "    # the two values in act_values eg [0.67, 0.04]\n",
        "  \n",
        "  def replay(self, batch_size):\n",
        "    '''\n",
        "    Trains the neural net with experience in the memory.\n",
        "    We need to maximise the rewards in the long run, so we define gamma/discount\n",
        "    rate through which the agent will learn to maximise the discounted future \n",
        "    award in the long run.\n",
        "    '''\n",
        "    minibatch = random.sample(self.memory, batch_size)\n",
        "    for state, action, reward, next_state, done in minibatch:\n",
        "      target = reward # if done\n",
        "      \n",
        "      if not done:\n",
        "        target = (reward + self.gamma * np.amax(\n",
        "            self.model.predict(next_state)[0]))\n",
        "        target_f = self.model.predict(state)\n",
        "        target_f[0][action] = target\n",
        "        self.model.fit(state, target_f, epochs = 1, verbose = 0)\n",
        "      if self.epsilon > self.epsilon_min:\n",
        "        self.epsilon *= self.epsilon_decay\n",
        "        \n",
        "  def load(self, name):\n",
        "    self.model.load_weghts(name)\n",
        "    \n",
        "  def save(self, name):\n",
        "    self.model.save_weights(name)\n",
        "    \n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "v39smxwnWUyI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 6489
        },
        "outputId": "0b92aa3b-2a1f-4050-f77e-072353642ef9"
      },
      "cell_type": "code",
      "source": [
        "if __name__ == \"__main__\":\n",
        "  env = gym.make('CartPole-v1')\n",
        "  state_size = env.observation_space.shape[0]\n",
        "  action_size = env.action_space.n\n",
        "  agent = DQNAgent(state_size, action_size)\n",
        "  \n",
        "  done = False\n",
        "  batch_size = 32\n",
        "  \n",
        "  for a in range(Episodes):\n",
        "    state = env.reset()\n",
        "    state = np.reshape(state, [1, state_size])\n",
        "    for time in range(500):\n",
        "      \n",
        "      action = agent.act(state)\n",
        "      next_state, reward, done, _ = env.step(action)\n",
        "      reward = reward if not done else -10\n",
        "      next_state = np.reshape(next_state, [1,state_size])\n",
        "      agent.remember(state, action, reward, next_state, done)\n",
        "      state = next_state\n",
        "      if done:\n",
        "        print(\"Episode: {}/{}, score: {}, a : {:.2}\"\n",
        "             .format(a, Episodes, time, agent.epsilon))\n",
        "        break\n",
        "      if len(agent.memory) > batch_size:\n",
        "        agent.replay(batch_size)\n",
        "      "
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
            "Episode: 0/1000, score: 15, a : 1.0\n",
            "Episode: 1/1000, score: 17, a : 0.85\n",
            "Episode: 2/1000, score: 11, a : 0.15\n",
            "Episode: 3/1000, score: 8, a : 0.04\n",
            "Episode: 4/1000, score: 9, a : 0.01\n",
            "Episode: 5/1000, score: 8, a : 0.01\n",
            "Episode: 6/1000, score: 9, a : 0.01\n",
            "Episode: 7/1000, score: 9, a : 0.01\n",
            "Episode: 8/1000, score: 8, a : 0.01\n",
            "Episode: 9/1000, score: 8, a : 0.01\n",
            "Episode: 10/1000, score: 9, a : 0.01\n",
            "Episode: 11/1000, score: 8, a : 0.01\n",
            "Episode: 12/1000, score: 7, a : 0.01\n",
            "Episode: 13/1000, score: 8, a : 0.01\n",
            "Episode: 14/1000, score: 8, a : 0.01\n",
            "Episode: 15/1000, score: 7, a : 0.01\n",
            "Episode: 16/1000, score: 9, a : 0.01\n",
            "Episode: 17/1000, score: 8, a : 0.01\n",
            "Episode: 18/1000, score: 8, a : 0.01\n",
            "Episode: 19/1000, score: 8, a : 0.01\n",
            "Episode: 20/1000, score: 8, a : 0.01\n",
            "Episode: 21/1000, score: 9, a : 0.01\n",
            "Episode: 22/1000, score: 8, a : 0.01\n",
            "Episode: 23/1000, score: 9, a : 0.01\n",
            "Episode: 24/1000, score: 9, a : 0.01\n",
            "Episode: 25/1000, score: 7, a : 0.01\n",
            "Episode: 26/1000, score: 8, a : 0.01\n",
            "Episode: 27/1000, score: 7, a : 0.01\n",
            "Episode: 28/1000, score: 9, a : 0.01\n",
            "Episode: 29/1000, score: 9, a : 0.01\n",
            "Episode: 30/1000, score: 8, a : 0.01\n",
            "Episode: 31/1000, score: 7, a : 0.01\n",
            "Episode: 32/1000, score: 9, a : 0.01\n",
            "Episode: 33/1000, score: 8, a : 0.01\n",
            "Episode: 34/1000, score: 9, a : 0.01\n",
            "Episode: 35/1000, score: 9, a : 0.01\n",
            "Episode: 36/1000, score: 8, a : 0.01\n",
            "Episode: 37/1000, score: 7, a : 0.01\n",
            "Episode: 38/1000, score: 7, a : 0.01\n",
            "Episode: 39/1000, score: 8, a : 0.01\n",
            "Episode: 40/1000, score: 9, a : 0.01\n",
            "Episode: 41/1000, score: 8, a : 0.01\n",
            "Episode: 42/1000, score: 7, a : 0.01\n",
            "Episode: 43/1000, score: 8, a : 0.01\n",
            "Episode: 44/1000, score: 9, a : 0.01\n",
            "Episode: 45/1000, score: 9, a : 0.01\n",
            "Episode: 46/1000, score: 8, a : 0.01\n",
            "Episode: 47/1000, score: 9, a : 0.01\n",
            "Episode: 48/1000, score: 8, a : 0.01\n",
            "Episode: 49/1000, score: 7, a : 0.01\n",
            "Episode: 50/1000, score: 9, a : 0.01\n",
            "Episode: 51/1000, score: 9, a : 0.01\n",
            "Episode: 52/1000, score: 8, a : 0.01\n",
            "Episode: 53/1000, score: 8, a : 0.01\n",
            "Episode: 54/1000, score: 8, a : 0.01\n",
            "Episode: 55/1000, score: 8, a : 0.01\n",
            "Episode: 56/1000, score: 9, a : 0.01\n",
            "Episode: 57/1000, score: 8, a : 0.01\n",
            "Episode: 58/1000, score: 7, a : 0.01\n",
            "Episode: 59/1000, score: 9, a : 0.01\n",
            "Episode: 60/1000, score: 8, a : 0.01\n",
            "Episode: 61/1000, score: 7, a : 0.01\n",
            "Episode: 62/1000, score: 10, a : 0.01\n",
            "Episode: 63/1000, score: 7, a : 0.01\n",
            "Episode: 64/1000, score: 8, a : 0.01\n",
            "Episode: 65/1000, score: 11, a : 0.01\n",
            "Episode: 66/1000, score: 9, a : 0.01\n",
            "Episode: 67/1000, score: 9, a : 0.01\n",
            "Episode: 68/1000, score: 9, a : 0.01\n",
            "Episode: 69/1000, score: 9, a : 0.01\n",
            "Episode: 70/1000, score: 8, a : 0.01\n",
            "Episode: 71/1000, score: 8, a : 0.01\n",
            "Episode: 72/1000, score: 8, a : 0.01\n",
            "Episode: 73/1000, score: 8, a : 0.01\n",
            "Episode: 74/1000, score: 9, a : 0.01\n",
            "Episode: 75/1000, score: 9, a : 0.01\n",
            "Episode: 76/1000, score: 8, a : 0.01\n",
            "Episode: 77/1000, score: 7, a : 0.01\n",
            "Episode: 78/1000, score: 8, a : 0.01\n",
            "Episode: 79/1000, score: 7, a : 0.01\n",
            "Episode: 80/1000, score: 9, a : 0.01\n",
            "Episode: 81/1000, score: 9, a : 0.01\n",
            "Episode: 82/1000, score: 9, a : 0.01\n",
            "Episode: 83/1000, score: 9, a : 0.01\n",
            "Episode: 84/1000, score: 8, a : 0.01\n",
            "Episode: 85/1000, score: 8, a : 0.01\n",
            "Episode: 86/1000, score: 9, a : 0.01\n",
            "Episode: 87/1000, score: 9, a : 0.01\n",
            "Episode: 88/1000, score: 7, a : 0.01\n",
            "Episode: 89/1000, score: 9, a : 0.01\n",
            "Episode: 90/1000, score: 8, a : 0.01\n",
            "Episode: 91/1000, score: 8, a : 0.01\n",
            "Episode: 92/1000, score: 8, a : 0.01\n",
            "Episode: 93/1000, score: 9, a : 0.01\n",
            "Episode: 94/1000, score: 9, a : 0.01\n",
            "Episode: 95/1000, score: 8, a : 0.01\n",
            "Episode: 96/1000, score: 9, a : 0.01\n",
            "Episode: 97/1000, score: 7, a : 0.01\n",
            "Episode: 98/1000, score: 8, a : 0.01\n",
            "Episode: 99/1000, score: 9, a : 0.01\n",
            "Episode: 100/1000, score: 7, a : 0.01\n",
            "Episode: 101/1000, score: 10, a : 0.01\n",
            "Episode: 102/1000, score: 8, a : 0.01\n",
            "Episode: 103/1000, score: 8, a : 0.01\n",
            "Episode: 104/1000, score: 7, a : 0.01\n",
            "Episode: 105/1000, score: 9, a : 0.01\n",
            "Episode: 106/1000, score: 8, a : 0.01\n",
            "Episode: 107/1000, score: 9, a : 0.01\n",
            "Episode: 108/1000, score: 8, a : 0.01\n",
            "Episode: 109/1000, score: 9, a : 0.01\n",
            "Episode: 110/1000, score: 8, a : 0.01\n",
            "Episode: 111/1000, score: 9, a : 0.01\n",
            "Episode: 112/1000, score: 9, a : 0.01\n",
            "Episode: 113/1000, score: 9, a : 0.01\n",
            "Episode: 114/1000, score: 7, a : 0.01\n",
            "Episode: 115/1000, score: 7, a : 0.01\n",
            "Episode: 116/1000, score: 9, a : 0.01\n",
            "Episode: 117/1000, score: 8, a : 0.01\n",
            "Episode: 118/1000, score: 9, a : 0.01\n",
            "Episode: 119/1000, score: 9, a : 0.01\n",
            "Episode: 120/1000, score: 8, a : 0.01\n",
            "Episode: 121/1000, score: 8, a : 0.01\n",
            "Episode: 122/1000, score: 9, a : 0.01\n",
            "Episode: 123/1000, score: 9, a : 0.01\n",
            "Episode: 124/1000, score: 8, a : 0.01\n",
            "Episode: 125/1000, score: 9, a : 0.01\n",
            "Episode: 126/1000, score: 9, a : 0.01\n",
            "Episode: 127/1000, score: 8, a : 0.01\n",
            "Episode: 128/1000, score: 8, a : 0.01\n",
            "Episode: 129/1000, score: 9, a : 0.01\n",
            "Episode: 130/1000, score: 9, a : 0.01\n",
            "Episode: 131/1000, score: 9, a : 0.01\n",
            "Episode: 132/1000, score: 7, a : 0.01\n",
            "Episode: 133/1000, score: 8, a : 0.01\n",
            "Episode: 134/1000, score: 9, a : 0.01\n",
            "Episode: 135/1000, score: 8, a : 0.01\n",
            "Episode: 136/1000, score: 9, a : 0.01\n",
            "Episode: 137/1000, score: 9, a : 0.01\n",
            "Episode: 138/1000, score: 9, a : 0.01\n",
            "Episode: 139/1000, score: 8, a : 0.01\n",
            "Episode: 140/1000, score: 7, a : 0.01\n",
            "Episode: 141/1000, score: 11, a : 0.01\n",
            "Episode: 142/1000, score: 9, a : 0.01\n",
            "Episode: 143/1000, score: 8, a : 0.01\n",
            "Episode: 144/1000, score: 9, a : 0.01\n",
            "Episode: 145/1000, score: 9, a : 0.01\n",
            "Episode: 146/1000, score: 9, a : 0.01\n",
            "Episode: 147/1000, score: 9, a : 0.01\n",
            "Episode: 148/1000, score: 9, a : 0.01\n",
            "Episode: 149/1000, score: 8, a : 0.01\n",
            "Episode: 150/1000, score: 9, a : 0.01\n",
            "Episode: 151/1000, score: 8, a : 0.01\n",
            "Episode: 152/1000, score: 8, a : 0.01\n",
            "Episode: 153/1000, score: 9, a : 0.01\n",
            "Episode: 154/1000, score: 8, a : 0.01\n",
            "Episode: 155/1000, score: 10, a : 0.01\n",
            "Episode: 156/1000, score: 8, a : 0.01\n",
            "Episode: 157/1000, score: 8, a : 0.01\n",
            "Episode: 158/1000, score: 10, a : 0.01\n",
            "Episode: 159/1000, score: 9, a : 0.01\n",
            "Episode: 160/1000, score: 8, a : 0.01\n",
            "Episode: 161/1000, score: 9, a : 0.01\n",
            "Episode: 162/1000, score: 9, a : 0.01\n",
            "Episode: 163/1000, score: 9, a : 0.01\n",
            "Episode: 164/1000, score: 9, a : 0.01\n",
            "Episode: 165/1000, score: 8, a : 0.01\n",
            "Episode: 166/1000, score: 9, a : 0.01\n",
            "Episode: 167/1000, score: 9, a : 0.01\n",
            "Episode: 168/1000, score: 9, a : 0.01\n",
            "Episode: 169/1000, score: 8, a : 0.01\n",
            "Episode: 170/1000, score: 9, a : 0.01\n",
            "Episode: 171/1000, score: 8, a : 0.01\n",
            "Episode: 172/1000, score: 9, a : 0.01\n",
            "Episode: 173/1000, score: 8, a : 0.01\n",
            "Episode: 174/1000, score: 11, a : 0.01\n",
            "Episode: 175/1000, score: 8, a : 0.01\n",
            "Episode: 176/1000, score: 9, a : 0.01\n",
            "Episode: 177/1000, score: 7, a : 0.01\n",
            "Episode: 178/1000, score: 9, a : 0.01\n",
            "Episode: 179/1000, score: 9, a : 0.01\n",
            "Episode: 180/1000, score: 7, a : 0.01\n",
            "Episode: 181/1000, score: 7, a : 0.01\n",
            "Episode: 182/1000, score: 8, a : 0.01\n",
            "Episode: 183/1000, score: 8, a : 0.01\n",
            "Episode: 184/1000, score: 8, a : 0.01\n",
            "Episode: 185/1000, score: 9, a : 0.01\n",
            "Episode: 186/1000, score: 9, a : 0.01\n",
            "Episode: 187/1000, score: 8, a : 0.01\n",
            "Episode: 188/1000, score: 9, a : 0.01\n",
            "Episode: 189/1000, score: 8, a : 0.01\n",
            "Episode: 190/1000, score: 8, a : 0.01\n",
            "Episode: 191/1000, score: 9, a : 0.01\n",
            "Episode: 192/1000, score: 7, a : 0.01\n",
            "Episode: 193/1000, score: 9, a : 0.01\n",
            "Episode: 194/1000, score: 8, a : 0.01\n",
            "Episode: 195/1000, score: 9, a : 0.01\n",
            "Episode: 196/1000, score: 8, a : 0.01\n",
            "Episode: 197/1000, score: 9, a : 0.01\n",
            "Episode: 198/1000, score: 7, a : 0.01\n",
            "Episode: 199/1000, score: 8, a : 0.01\n",
            "Episode: 200/1000, score: 8, a : 0.01\n",
            "Episode: 201/1000, score: 8, a : 0.01\n",
            "Episode: 202/1000, score: 9, a : 0.01\n",
            "Episode: 203/1000, score: 8, a : 0.01\n",
            "Episode: 204/1000, score: 9, a : 0.01\n",
            "Episode: 205/1000, score: 9, a : 0.01\n",
            "Episode: 206/1000, score: 9, a : 0.01\n",
            "Episode: 207/1000, score: 9, a : 0.01\n",
            "Episode: 208/1000, score: 8, a : 0.01\n",
            "Episode: 209/1000, score: 8, a : 0.01\n",
            "Episode: 210/1000, score: 9, a : 0.01\n",
            "Episode: 211/1000, score: 7, a : 0.01\n",
            "Episode: 212/1000, score: 8, a : 0.01\n",
            "Episode: 213/1000, score: 8, a : 0.01\n",
            "Episode: 214/1000, score: 7, a : 0.01\n",
            "Episode: 215/1000, score: 8, a : 0.01\n",
            "Episode: 216/1000, score: 7, a : 0.01\n",
            "Episode: 217/1000, score: 8, a : 0.01\n",
            "Episode: 218/1000, score: 8, a : 0.01\n",
            "Episode: 219/1000, score: 9, a : 0.01\n",
            "Episode: 220/1000, score: 9, a : 0.01\n",
            "Episode: 221/1000, score: 7, a : 0.01\n",
            "Episode: 222/1000, score: 7, a : 0.01\n",
            "Episode: 223/1000, score: 9, a : 0.01\n",
            "Episode: 224/1000, score: 8, a : 0.01\n",
            "Episode: 225/1000, score: 8, a : 0.01\n",
            "Episode: 226/1000, score: 9, a : 0.01\n",
            "Episode: 227/1000, score: 7, a : 0.01\n",
            "Episode: 228/1000, score: 8, a : 0.01\n",
            "Episode: 229/1000, score: 9, a : 0.01\n",
            "Episode: 230/1000, score: 9, a : 0.01\n",
            "Episode: 231/1000, score: 8, a : 0.01\n",
            "Episode: 232/1000, score: 8, a : 0.01\n",
            "Episode: 233/1000, score: 10, a : 0.01\n",
            "Episode: 234/1000, score: 8, a : 0.01\n",
            "Episode: 235/1000, score: 8, a : 0.01\n",
            "Episode: 236/1000, score: 7, a : 0.01\n",
            "Episode: 237/1000, score: 8, a : 0.01\n",
            "Episode: 238/1000, score: 9, a : 0.01\n",
            "Episode: 239/1000, score: 8, a : 0.01\n",
            "Episode: 240/1000, score: 8, a : 0.01\n",
            "Episode: 241/1000, score: 8, a : 0.01\n",
            "Episode: 242/1000, score: 8, a : 0.01\n",
            "Episode: 243/1000, score: 10, a : 0.01\n",
            "Episode: 244/1000, score: 8, a : 0.01\n",
            "Episode: 245/1000, score: 9, a : 0.01\n",
            "Episode: 246/1000, score: 10, a : 0.01\n",
            "Episode: 247/1000, score: 7, a : 0.01\n",
            "Episode: 248/1000, score: 8, a : 0.01\n",
            "Episode: 249/1000, score: 8, a : 0.01\n",
            "Episode: 250/1000, score: 7, a : 0.01\n",
            "Episode: 251/1000, score: 9, a : 0.01\n",
            "Episode: 252/1000, score: 8, a : 0.01\n",
            "Episode: 253/1000, score: 9, a : 0.01\n",
            "Episode: 254/1000, score: 9, a : 0.01\n",
            "Episode: 255/1000, score: 8, a : 0.01\n",
            "Episode: 256/1000, score: 9, a : 0.01\n",
            "Episode: 257/1000, score: 7, a : 0.01\n",
            "Episode: 258/1000, score: 11, a : 0.01\n",
            "Episode: 259/1000, score: 9, a : 0.01\n",
            "Episode: 260/1000, score: 8, a : 0.01\n",
            "Episode: 261/1000, score: 8, a : 0.01\n",
            "Episode: 262/1000, score: 8, a : 0.01\n",
            "Episode: 263/1000, score: 8, a : 0.01\n",
            "Episode: 264/1000, score: 8, a : 0.01\n",
            "Episode: 265/1000, score: 10, a : 0.01\n",
            "Episode: 266/1000, score: 9, a : 0.01\n",
            "Episode: 267/1000, score: 8, a : 0.01\n",
            "Episode: 268/1000, score: 8, a : 0.01\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-10-b08f4229e0a5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmemory\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m         \u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplay\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-9-1bf94afc6437>\u001b[0m in \u001b[0;36mreplay\u001b[0;34m(self, batch_size)\u001b[0m\n\u001b[1;32m     68\u001b[0m         \u001b[0mtarget_f\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m         \u001b[0mtarget_f\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     71\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon_min\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon\u001b[0m \u001b[0;34m*=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon_decay\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/models.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1000\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1001\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1002\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1003\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1004\u001b[0m     def evaluate(self, x=None, y=None,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1703\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1704\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1705\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1706\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1707\u001b[0m     def evaluate(self, x=None, y=None,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m   1234\u001b[0m                         \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1236\u001b[0;31m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1237\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1238\u001b[0m                         \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2480\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2481\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2482\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2483\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2484\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    875\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    876\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 877\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    878\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    879\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1098\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1099\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1100\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1101\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1102\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1270\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1271\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1272\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1273\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1274\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1276\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1277\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1278\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1279\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1280\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1261\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1262\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1263\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1264\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1265\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1349\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1350\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1351\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "WoJtvOIGeAu_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 5067
        },
        "outputId": "20293d83-520c-4c7b-b286-b589aaea284e"
      },
      "cell_type": "code",
      "source": [
        "# -*- coding: utf-8 -*-\n",
        "import random\n",
        "import gym\n",
        "import numpy as np\n",
        "from collections import deque\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "from keras.optimizers import Adam\n",
        "\n",
        "EPISODES = 1000\n",
        "\n",
        "class DQNAgent:\n",
        "    def __init__(self, state_size, action_size):\n",
        "        self.state_size = state_size\n",
        "        self.action_size = action_size\n",
        "        self.memory = deque(maxlen=2000)\n",
        "        self.gamma = 0.95    # discount rate\n",
        "        self.epsilon = 1.0  # exploration rate\n",
        "        self.epsilon_min = 0.01\n",
        "        self.epsilon_decay = 0.995\n",
        "        self.learning_rate = 0.001\n",
        "        self.model = self._build_model()\n",
        "\n",
        "    def _build_model(self):\n",
        "        # Neural Net for Deep-Q learning Model\n",
        "        model = Sequential()\n",
        "        model.add(Dense(24, input_dim=self.state_size, activation='relu'))\n",
        "        model.add(Dense(24, activation='relu'))\n",
        "        model.add(Dense(self.action_size, activation='linear'))\n",
        "        model.compile(loss='mse',\n",
        "                      optimizer=Adam(lr=self.learning_rate))\n",
        "        return model\n",
        "\n",
        "    def remember(self, state, action, reward, next_state, done):\n",
        "        self.memory.append((state, action, reward, next_state, done))\n",
        "\n",
        "    def act(self, state):\n",
        "        if np.random.rand() <= self.epsilon:\n",
        "            return random.randrange(self.action_size)\n",
        "        act_values = self.model.predict(state)\n",
        "        return np.argmax(act_values[0])  # returns action\n",
        "\n",
        "    def replay(self, batch_size):\n",
        "        minibatch = random.sample(self.memory, batch_size)\n",
        "        for state, action, reward, next_state, done in minibatch:\n",
        "            target = reward\n",
        "            if not done:\n",
        "                target = (reward + self.gamma *\n",
        "                          np.amax(self.model.predict(next_state)[0]))\n",
        "            target_f = self.model.predict(state)\n",
        "            target_f[0][action] = target\n",
        "            self.model.fit(state, target_f, epochs=1, verbose=0)\n",
        "        if self.epsilon > self.epsilon_min:\n",
        "            self.epsilon *= self.epsilon_decay\n",
        "\n",
        "    def load(self, name):\n",
        "        self.model.load_weights(name)\n",
        "\n",
        "    def save(self, name):\n",
        "        self.model.save_weights(name)\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    env = gym.make('CartPole-v1')\n",
        "    state_size = env.observation_space.shape[0]\n",
        "    action_size = env.action_space.n\n",
        "    agent = DQNAgent(state_size, action_size)\n",
        "    # agent.load(\"./save/cartpole-dqn.h5\")\n",
        "    done = False\n",
        "    batch_size = 32\n",
        "\n",
        "    for e in range(EPISODES):\n",
        "        state = env.reset()\n",
        "        state = np.reshape(state, [1, state_size])\n",
        "        for time in range(500):\n",
        "            # env.render()\n",
        "            action = agent.act(state)\n",
        "            next_state, reward, done, _ = env.step(action)\n",
        "            reward = reward if not done else -10\n",
        "            next_state = np.reshape(next_state, [1, state_size])\n",
        "            agent.remember(state, action, reward, next_state, done)\n",
        "            state = next_state\n",
        "            if done:\n",
        "                print(\"episode: {}/{}, score: {}, e: {:.2}\"\n",
        "                      .format(e, EPISODES, time, agent.epsilon))\n",
        "                break\n",
        "            if len(agent.memory) > batch_size:\n",
        "                agent.replay(batch_size)\n",
        "        # if e % 10 == 0:\n",
        "        #     agent.save(\"./save/cartpole-dqn.h5\")"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
            "episode: 0/1000, score: 17, e: 1.0\n",
            "episode: 1/1000, score: 20, e: 0.97\n",
            "episode: 2/1000, score: 35, e: 0.81\n",
            "episode: 3/1000, score: 23, e: 0.73\n",
            "episode: 4/1000, score: 16, e: 0.67\n",
            "episode: 5/1000, score: 13, e: 0.63\n",
            "episode: 6/1000, score: 11, e: 0.59\n",
            "episode: 7/1000, score: 14, e: 0.55\n",
            "episode: 8/1000, score: 13, e: 0.52\n",
            "episode: 9/1000, score: 9, e: 0.5\n",
            "episode: 10/1000, score: 15, e: 0.46\n",
            "episode: 11/1000, score: 22, e: 0.41\n",
            "episode: 12/1000, score: 18, e: 0.38\n",
            "episode: 13/1000, score: 13, e: 0.35\n",
            "episode: 14/1000, score: 7, e: 0.34\n",
            "episode: 15/1000, score: 11, e: 0.32\n",
            "episode: 16/1000, score: 9, e: 0.31\n",
            "episode: 17/1000, score: 11, e: 0.29\n",
            "episode: 18/1000, score: 8, e: 0.28\n",
            "episode: 19/1000, score: 10, e: 0.27\n",
            "episode: 20/1000, score: 17, e: 0.24\n",
            "episode: 21/1000, score: 9, e: 0.23\n",
            "episode: 22/1000, score: 9, e: 0.22\n",
            "episode: 23/1000, score: 7, e: 0.22\n",
            "episode: 24/1000, score: 9, e: 0.21\n",
            "episode: 25/1000, score: 8, e: 0.2\n",
            "episode: 26/1000, score: 9, e: 0.19\n",
            "episode: 27/1000, score: 8, e: 0.18\n",
            "episode: 28/1000, score: 9, e: 0.17\n",
            "episode: 29/1000, score: 7, e: 0.17\n",
            "episode: 30/1000, score: 11, e: 0.16\n",
            "episode: 31/1000, score: 8, e: 0.15\n",
            "episode: 32/1000, score: 12, e: 0.14\n",
            "episode: 33/1000, score: 9, e: 0.14\n",
            "episode: 34/1000, score: 8, e: 0.13\n",
            "episode: 35/1000, score: 7, e: 0.13\n",
            "episode: 36/1000, score: 8, e: 0.12\n",
            "episode: 37/1000, score: 8, e: 0.12\n",
            "episode: 38/1000, score: 8, e: 0.11\n",
            "episode: 39/1000, score: 8, e: 0.11\n",
            "episode: 40/1000, score: 8, e: 0.1\n",
            "episode: 41/1000, score: 8, e: 0.1\n",
            "episode: 42/1000, score: 11, e: 0.095\n",
            "episode: 43/1000, score: 8, e: 0.091\n",
            "episode: 44/1000, score: 31, e: 0.078\n",
            "episode: 45/1000, score: 10, e: 0.074\n",
            "episode: 46/1000, score: 7, e: 0.072\n",
            "episode: 47/1000, score: 9, e: 0.068\n",
            "episode: 48/1000, score: 9, e: 0.065\n",
            "episode: 49/1000, score: 10, e: 0.062\n",
            "episode: 50/1000, score: 8, e: 0.06\n",
            "episode: 51/1000, score: 9, e: 0.057\n",
            "episode: 52/1000, score: 7, e: 0.055\n",
            "episode: 53/1000, score: 9, e: 0.053\n",
            "episode: 54/1000, score: 9, e: 0.05\n",
            "episode: 55/1000, score: 7, e: 0.049\n",
            "episode: 56/1000, score: 8, e: 0.047\n",
            "episode: 57/1000, score: 8, e: 0.045\n",
            "episode: 58/1000, score: 8, e: 0.043\n",
            "episode: 59/1000, score: 10, e: 0.041\n",
            "episode: 60/1000, score: 10, e: 0.039\n",
            "episode: 61/1000, score: 8, e: 0.038\n",
            "episode: 62/1000, score: 7, e: 0.036\n",
            "episode: 63/1000, score: 70, e: 0.025\n",
            "episode: 64/1000, score: 92, e: 0.016\n",
            "episode: 65/1000, score: 79, e: 0.011\n",
            "episode: 66/1000, score: 37, e: 0.01\n",
            "episode: 67/1000, score: 9, e: 0.01\n",
            "episode: 68/1000, score: 9, e: 0.01\n",
            "episode: 69/1000, score: 39, e: 0.01\n",
            "episode: 70/1000, score: 36, e: 0.01\n",
            "episode: 71/1000, score: 11, e: 0.01\n",
            "episode: 72/1000, score: 28, e: 0.01\n",
            "episode: 73/1000, score: 31, e: 0.01\n",
            "episode: 74/1000, score: 10, e: 0.01\n",
            "episode: 75/1000, score: 8, e: 0.01\n",
            "episode: 76/1000, score: 27, e: 0.01\n",
            "episode: 77/1000, score: 19, e: 0.01\n",
            "episode: 78/1000, score: 25, e: 0.01\n",
            "episode: 79/1000, score: 20, e: 0.01\n",
            "episode: 80/1000, score: 20, e: 0.01\n",
            "episode: 81/1000, score: 18, e: 0.01\n",
            "episode: 82/1000, score: 14, e: 0.01\n",
            "episode: 83/1000, score: 20, e: 0.01\n",
            "episode: 84/1000, score: 22, e: 0.01\n",
            "episode: 85/1000, score: 23, e: 0.01\n",
            "episode: 86/1000, score: 21, e: 0.01\n",
            "episode: 87/1000, score: 16, e: 0.01\n",
            "episode: 88/1000, score: 19, e: 0.01\n",
            "episode: 89/1000, score: 20, e: 0.01\n",
            "episode: 90/1000, score: 30, e: 0.01\n",
            "episode: 91/1000, score: 31, e: 0.01\n",
            "episode: 92/1000, score: 20, e: 0.01\n",
            "episode: 93/1000, score: 21, e: 0.01\n",
            "episode: 94/1000, score: 16, e: 0.01\n",
            "episode: 95/1000, score: 18, e: 0.01\n",
            "episode: 96/1000, score: 24, e: 0.01\n",
            "episode: 97/1000, score: 21, e: 0.01\n",
            "episode: 98/1000, score: 14, e: 0.01\n",
            "episode: 99/1000, score: 23, e: 0.01\n",
            "episode: 100/1000, score: 24, e: 0.01\n",
            "episode: 101/1000, score: 32, e: 0.01\n",
            "episode: 102/1000, score: 29, e: 0.01\n",
            "episode: 103/1000, score: 25, e: 0.01\n",
            "episode: 104/1000, score: 33, e: 0.01\n",
            "episode: 105/1000, score: 20, e: 0.01\n",
            "episode: 106/1000, score: 18, e: 0.01\n",
            "episode: 107/1000, score: 28, e: 0.01\n",
            "episode: 108/1000, score: 30, e: 0.01\n",
            "episode: 109/1000, score: 38, e: 0.01\n",
            "episode: 110/1000, score: 22, e: 0.01\n",
            "episode: 111/1000, score: 40, e: 0.01\n",
            "episode: 112/1000, score: 19, e: 0.01\n",
            "episode: 113/1000, score: 34, e: 0.01\n",
            "episode: 114/1000, score: 28, e: 0.01\n",
            "episode: 115/1000, score: 28, e: 0.01\n",
            "episode: 116/1000, score: 25, e: 0.01\n",
            "episode: 117/1000, score: 19, e: 0.01\n",
            "episode: 118/1000, score: 31, e: 0.01\n",
            "episode: 119/1000, score: 35, e: 0.01\n",
            "episode: 120/1000, score: 88, e: 0.01\n",
            "episode: 121/1000, score: 52, e: 0.01\n",
            "episode: 122/1000, score: 47, e: 0.01\n",
            "episode: 123/1000, score: 21, e: 0.01\n",
            "episode: 124/1000, score: 53, e: 0.01\n",
            "episode: 125/1000, score: 30, e: 0.01\n",
            "episode: 126/1000, score: 16, e: 0.01\n",
            "episode: 127/1000, score: 30, e: 0.01\n",
            "episode: 128/1000, score: 42, e: 0.01\n",
            "episode: 129/1000, score: 24, e: 0.01\n",
            "episode: 130/1000, score: 64, e: 0.01\n",
            "episode: 131/1000, score: 25, e: 0.01\n",
            "episode: 132/1000, score: 92, e: 0.01\n",
            "episode: 133/1000, score: 59, e: 0.01\n",
            "episode: 134/1000, score: 69, e: 0.01\n",
            "episode: 135/1000, score: 57, e: 0.01\n",
            "episode: 136/1000, score: 64, e: 0.01\n",
            "episode: 137/1000, score: 65, e: 0.01\n",
            "episode: 138/1000, score: 9, e: 0.01\n",
            "episode: 139/1000, score: 9, e: 0.01\n",
            "episode: 140/1000, score: 8, e: 0.01\n",
            "episode: 141/1000, score: 9, e: 0.01\n",
            "episode: 142/1000, score: 9, e: 0.01\n",
            "episode: 143/1000, score: 7, e: 0.01\n",
            "episode: 144/1000, score: 9, e: 0.01\n",
            "episode: 145/1000, score: 8, e: 0.01\n",
            "episode: 146/1000, score: 8, e: 0.01\n",
            "episode: 147/1000, score: 8, e: 0.01\n",
            "episode: 148/1000, score: 9, e: 0.01\n",
            "episode: 149/1000, score: 9, e: 0.01\n",
            "episode: 150/1000, score: 8, e: 0.01\n",
            "episode: 151/1000, score: 9, e: 0.01\n",
            "episode: 152/1000, score: 8, e: 0.01\n",
            "episode: 153/1000, score: 8, e: 0.01\n",
            "episode: 154/1000, score: 9, e: 0.01\n",
            "episode: 155/1000, score: 8, e: 0.01\n",
            "episode: 156/1000, score: 10, e: 0.01\n",
            "episode: 157/1000, score: 11, e: 0.01\n",
            "episode: 158/1000, score: 26, e: 0.01\n",
            "episode: 159/1000, score: 14, e: 0.01\n",
            "episode: 160/1000, score: 11, e: 0.01\n",
            "episode: 161/1000, score: 11, e: 0.01\n",
            "episode: 162/1000, score: 12, e: 0.01\n",
            "episode: 163/1000, score: 15, e: 0.01\n",
            "episode: 164/1000, score: 16, e: 0.01\n",
            "episode: 165/1000, score: 19, e: 0.01\n",
            "episode: 166/1000, score: 58, e: 0.01\n",
            "episode: 167/1000, score: 19, e: 0.01\n",
            "episode: 168/1000, score: 13, e: 0.01\n",
            "episode: 169/1000, score: 20, e: 0.01\n",
            "episode: 170/1000, score: 28, e: 0.01\n",
            "episode: 171/1000, score: 68, e: 0.01\n",
            "episode: 172/1000, score: 24, e: 0.01\n",
            "episode: 173/1000, score: 25, e: 0.01\n",
            "episode: 174/1000, score: 16, e: 0.01\n",
            "episode: 175/1000, score: 35, e: 0.01\n",
            "episode: 176/1000, score: 22, e: 0.01\n",
            "episode: 177/1000, score: 27, e: 0.01\n",
            "episode: 178/1000, score: 64, e: 0.01\n",
            "episode: 179/1000, score: 72, e: 0.01\n",
            "episode: 180/1000, score: 120, e: 0.01\n",
            "episode: 181/1000, score: 117, e: 0.01\n",
            "episode: 182/1000, score: 127, e: 0.01\n",
            "episode: 183/1000, score: 83, e: 0.01\n",
            "episode: 184/1000, score: 117, e: 0.01\n",
            "episode: 185/1000, score: 174, e: 0.01\n",
            "episode: 186/1000, score: 209, e: 0.01\n",
            "episode: 187/1000, score: 202, e: 0.01\n",
            "episode: 188/1000, score: 132, e: 0.01\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-11-e17ff3db0f2e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     85\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmemory\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m                 \u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplay\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0;31m# if e % 10 == 0:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m         \u001b[0;31m#     agent.save(\"./save/cartpole-dqn.h5\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-11-e17ff3db0f2e>\u001b[0m in \u001b[0;36mreplay\u001b[0;34m(self, batch_size)\u001b[0m\n\u001b[1;32m     49\u001b[0m             \u001b[0mtarget_f\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0mtarget_f\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon_min\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon\u001b[0m \u001b[0;34m*=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon_decay\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/models.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1000\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1001\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1002\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1003\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1004\u001b[0m     def evaluate(self, x=None, y=None,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1703\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1704\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1705\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1706\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1707\u001b[0m     def evaluate(self, x=None, y=None,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m   1234\u001b[0m                         \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1236\u001b[0;31m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1237\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1238\u001b[0m                         \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2480\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2481\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2482\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2483\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2484\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    875\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    876\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 877\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    878\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    879\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1098\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1099\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1100\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1101\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1102\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1270\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1271\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1272\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1273\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1274\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1276\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1277\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1278\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1279\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1280\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1261\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1262\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1263\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1264\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1265\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1349\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1350\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1351\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "5FkpdnqvYfb5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}