7.2 A2C Balance.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4590,
     "status": "ok",
     "timestamp": 1526810261274,
     "user": {
      "displayName": "Jannes Klaas",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "115843019312458301179"
     },
     "user_tz": -120
    },
    "id": "WvnZWyvt-ZPa",
    "outputId": "395cc121-bac6-4222-b727-12c46abcca59"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import gym\n",
    "import numpy as np\n",
    "from scipy.stats import norm\n",
    "from keras.layers import Dense, Input, Lambda\n",
    "from keras.models import Model\n",
    "from keras.optimizers import Adam\n",
    "from keras import backend as K\n",
    "from collections import deque\n",
    "import random\n",
    "\n",
    "EPISODES = 3000\n",
    "\n",
    "\n",
    "# A2C(Advantage Actor-Critic) agent for the Cartpole\n",
    "class A2CAgent:\n",
    "    def __init__(self, state_size, action_size):\n",
    "        # if you want to see Cartpole learning, then change to True\n",
    "        self.render = False\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.value_size = 1\n",
    "        \n",
    "        self.exp_replay = deque(maxlen=2000)\n",
    "\n",
    "        # get gym environment name\n",
    "        # these are hyper parameters for the A3C\n",
    "        self.actor_lr = 0.0001\n",
    "        self.critic_lr = 0.001\n",
    "        self.discount_factor = .9\n",
    "\n",
    "        # create model for actor and critic network\n",
    "        self.actor, self.critic = self.build_model()\n",
    "\n",
    "        # method for training actor and critic network\n",
    "        #self.optimizer = [self.actor_optimizer(), self.critic_optimizer()]\n",
    "        \n",
    "        self.optimize_actor = self.actor_optimizer() #5\n",
    "        self.optimize_critic = self.critic_optimizer() \n",
    "\n",
    "\n",
    "    def build_model(self):\n",
    "        state = Input(batch_shape=(None, self.state_size))\n",
    "        actor_input = Dense(30, activation='relu', kernel_initializer='he_uniform')(state)\n",
    "        # actor_hidden = Dense(self.hidden2, activation='relu')(actor_input)\n",
    "        mu_0 = Dense(self.action_size, activation='tanh', kernel_initializer='he_uniform')(actor_input)\n",
    "        sigma_0 = Dense(self.action_size, activation='softplus', kernel_initializer='he_uniform')(actor_input)\n",
    "\n",
    "        mu = Lambda(lambda x: x * 2)(mu_0)\n",
    "        sigma = Lambda(lambda x: x + 0.0001)(sigma_0)\n",
    "\n",
    "        critic_input = Dense(30, activation='relu', kernel_initializer='he_uniform')(state)\n",
    "        # value_hidden = Dense(self.hidden2, activation='relu')(critic_input)\n",
    "        state_value = Dense(1, activation='linear', kernel_initializer='he_uniform')(critic_input)\n",
    "\n",
    "        actor = Model(inputs=state, outputs=(mu, sigma))\n",
    "        critic = Model(inputs=state, outputs=state_value)\n",
    "\n",
    "        actor._make_predict_function()\n",
    "        critic._make_predict_function()\n",
    "\n",
    "        actor.summary()\n",
    "        critic.summary()\n",
    "\n",
    "        return actor, critic\n",
    "\n",
    "    def actor_optimizer(self):\n",
    "        action = K.placeholder(shape=(None, 1))\n",
    "        advantages = K.placeholder(shape=(None, 1))\n",
    "\n",
    "        # mu = K.placeholder(shape=(None, self.action_size))\n",
    "        # sigma_sq = K.placeholder(shape=(None, self.action_size))\n",
    "\n",
    "        mu, sigma_sq = self.actor.output\n",
    "\n",
    "        pdf = 1. / K.sqrt(2. * np.pi * sigma_sq) * K.exp(-K.square(action - mu) / (2. * sigma_sq))\n",
    "        log_pdf = K.log(pdf + K.epsilon())\n",
    "        entropy = K.sum(0.5 * (K.log(2. * np.pi * sigma_sq) + 1.))\n",
    "\n",
    "        exp_v = log_pdf * advantages\n",
    "\n",
    "        exp_v = K.sum(exp_v + 0.01 * entropy)\n",
    "        actor_loss = -exp_v\n",
    "\n",
    "        optimizer = Adam(lr=self.actor_lr)\n",
    "        updates = optimizer.get_updates(self.actor.trainable_weights, [], actor_loss)\n",
    "\n",
    "        train = K.function([self.actor.input, action, advantages], [], updates=updates)\n",
    "        return train\n",
    "\n",
    "    # make loss function for Value approximation\n",
    "    def critic_optimizer(self):\n",
    "        discounted_reward = K.placeholder(shape=(None, 1))\n",
    "\n",
    "        value = self.critic.output\n",
    "\n",
    "        loss = K.mean(K.square(discounted_reward - value))\n",
    "\n",
    "        optimizer = Adam(lr=self.critic_lr)\n",
    "        updates = optimizer.get_updates(self.critic.trainable_weights, [], loss)\n",
    "        train = K.function([self.critic.input, discounted_reward], [], updates=updates)\n",
    "        return train\n",
    "\n",
    "    # using the output of policy network, pick action stochastically\n",
    "    def get_action(self, state):\n",
    "        mu, sigma_sq = self.actor.predict(np.reshape(state, [1, self.state_size]))\n",
    "        # sigma_sq = np.log(np.exp(sigma_sq + 1))\n",
    "        epsilon = np.random.randn(self.action_size)\n",
    "        # action = norm.rvs(loc=mu, scale=sigma_sq,size=1)\n",
    "        action = mu + np.sqrt(sigma_sq) * epsilon\n",
    "        action = np.clip(action, -2, 2)\n",
    "        return action\n",
    "\n",
    "    # update policy network every episode\n",
    "    def train_model(self, state, action, reward, next_state, done):\n",
    "        self.exp_replay.append((state, action, reward, next_state, done))\n",
    "        \n",
    "        (state, action, reward, next_state, done) = random.sample(self.exp_replay,1)[0]\n",
    "      \n",
    "        target = np.zeros((1, self.value_size))\n",
    "        advantages = np.zeros((1, self.action_size))\n",
    "\n",
    "        value = self.critic.predict(state)[0]\n",
    "        next_value = self.critic.predict(next_state)[0]\n",
    "\n",
    "        if done:\n",
    "            advantages[0] = reward - value\n",
    "            target[0][0] = reward\n",
    "        else:\n",
    "            advantages[0] = reward + self.discount_factor * (next_value) - value\n",
    "            target[0][0] = reward + self.discount_factor * next_value\n",
    "\n",
    "        self.optimize_actor([state, action, advantages])\n",
    "        self.optimize_critic([state, target])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "dg0doG69-4ix"
   },
   "outputs": [],
   "source": [
    "def run_experiment():\n",
    "  # In case of CartPole-v1, maximum length of episode is 500\n",
    "  env = gym.make('Pendulum-v0')\n",
    "  # get size of state and action from environment\n",
    "  state_size = env.observation_space.shape[0]\n",
    "  action_size = env.action_space.shape[0]\n",
    "\n",
    "  # make A2C agent\n",
    "  agent = A2CAgent(state_size, action_size)\n",
    "\n",
    "  scores, episodes = [], []\n",
    "\n",
    "  for e in range(EPISODES):\n",
    "      done = False\n",
    "      score = 0\n",
    "      state = env.reset()\n",
    "      state = np.reshape(state, [1, state_size])\n",
    "\n",
    "      while not done:\n",
    "          if agent.render:\n",
    "              env.render()\n",
    "\n",
    "          action = agent.get_action(state)\n",
    "          next_state, reward, done, info = env.step(action)\n",
    "          reward /= 10\n",
    "          next_state = np.reshape(next_state, [1, state_size])\n",
    "          # if an action make the episode end, then gives penalty of -100\n",
    "          agent.train_model(state, action, reward, next_state, done)\n",
    "\n",
    "          score += reward\n",
    "          state = next_state\n",
    "\n",
    "          if done:\n",
    "              # every episode, plot the play time\n",
    "              scores.append(score)\n",
    "              episodes.append(e)\n",
    "              print(\"episode:\", e, \"  score:\", score)\n",
    "\n",
    "              # if the mean of scores of last 10 episode is bigger than 490\n",
    "              # stop training\n",
    "              if np.mean(scores[-min(10, len(scores)):]) > -20:\n",
    "                  print('Solved Pendulum-v0 after {} iterations'.format(len(scores)))\n",
    "                  return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 39026
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3726464,
     "status": "error",
     "timestamp": 1526814016426,
     "user": {
      "displayName": "Jannes Klaas",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "115843019312458301179"
     },
     "user_tz": -120
    },
    "id": "3XlAsQku-_hX",
    "outputId": "004a1239-d6bd-4651-83ac-4769b034805d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_4 (InputLayer)            (None, 3)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 30)           120         input_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, 1)            31          dense_16[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_18 (Dense)                (None, 1)            31          dense_16[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_7 (Lambda)               (None, 1)            0           dense_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_8 (Lambda)               (None, 1)            0           dense_18[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 182\n",
      "Trainable params: 182\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_4 (InputLayer)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 30)                120       \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 151\n",
      "Trainable params: 151\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "episode: 0   score: [-159.58044646]\n",
      "episode: 1   score: [-143.58861741]\n",
      "episode: 2   score: [-152.80103168]\n",
      "episode: 3   score: [-155.49705716]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-90bc8ec86775>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mscrs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m   \u001b[0mscrs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_experiment\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-9-90b99e7c60d7>\u001b[0m in \u001b[0;36mrun_experiment\u001b[0;34m()\u001b[0m\n\u001b[1;32m     26\u001b[0m           \u001b[0mnext_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate_size\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m           \u001b[0;31m# if an action make the episode end, then gives penalty of -100\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m           \u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnext_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m           \u001b[0mscore\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-8-fd47eeceface>\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(self, state, action, reward, next_state, done)\u001b[0m\n\u001b[1;32m    131\u001b[0m             \u001b[0mtarget\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreward\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdiscount_factor\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mnext_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 133\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize_actor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0madvantages\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    134\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize_critic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2476\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2477\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2478\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2479\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2480\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    887\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 889\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    890\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    891\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1118\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1119\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1120\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1121\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1122\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1315\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1316\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1317\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1318\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1319\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1321\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1322\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1323\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1324\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1325\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1300\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[1;32m   1301\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1302\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1303\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1304\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "scrs = []\n",
    "for i in range(10):\n",
    "  scrs.append(run_experiment())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "olYdDyoh_Cbt"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "colab": {
   "collapsed_sections": [],
   "default_view": {},
   "name": "A2c w exp replay.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}