walker.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6edb1dbe-20ad-49d3-aabf-27a0438da7e0",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3b1d6d12-e770-47ce-a9eb-fb93b7a2b755",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\dmitry\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:246: DeprecationWarning: `np.bool8` is a deprecated alias for `np.bool_`.  (Deprecated NumPy 1.24)\n",
      "  np.bool8: (False, True),\n",
      "c:\\Users\\dmitry\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:326: DeprecationWarning: `np.bool8` is a deprecated alias for `np.bool_`.  (Deprecated NumPy 1.24)\n",
      "  np.bool8: (False, True),\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import gymnasium as gym\n",
    "import tensorflow as tf\n",
    "import tqdm.notebook as tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "from gymnasium.utils.save_video import save_video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "011e03a4-a0ea-4ce2-9f00-e09b9b4c2e04",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n",
      "GPU enable\n"
     ]
    }
   ],
   "source": [
    "gpus = tf.config.list_physical_devices('GPU')\n",
    "print(gpus)\n",
    "if gpus:\n",
    "    try:\n",
    "        tf.config.experimental.set_memory_growth(gpus[0], True)\n",
    "        print('GPU enable')\n",
    "    except Exception as e:\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e580a6f-a987-40ec-9864-2e296bdd9ba6",
   "metadata": {},
   "source": [
    "# Environment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "120abf50-431d-4c8c-bc71-e760aa7c08b4",
   "metadata": {},
   "source": [
    "Create the [environment](https://gymnasium.farama.org/environments/box2d/bipedal_walker/)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4c9241b5-510e-4a18-bcde-8bf3c5ac2b8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('BipedalWalker-v3', hardcore=False)\n",
    "eval_env = gym.make('BipedalWalker-v3', hardcore=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6828a718-db96-42f6-890f-265163fdedb9",
   "metadata": {},
   "source": [
    "# Replay Buffer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "037a88e3-0524-4d4f-9bd5-eb035d80fece",
   "metadata": {},
   "source": [
    "Create a replay buffer to hold game history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4a761715-1ef6-4ffd-b710-1758f292888c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer:\n",
    "\n",
    "    def __init__(self, max_size: int, observation_space: gym.spaces.Space, action_space: gym.spaces.Space, seed: int | None = None):\n",
    "        \"\"\"Stores the replay history with a maximum of `max_size` entries, removing old entries as needed.\n",
    "\n",
    "        Parameters:\n",
    "            max_size: maximal number of entries to keep\n",
    "            observation_space: specification of the observation space\n",
    "            action_space: specification of the action space\n",
    "            seed: seed to initialize the internal random number generator for reproducibility\"\"\"\n",
    "        self.max_size = max_size\n",
    "        self.done = np.zeros(max_size)\n",
    "        self.step = 0\n",
    "        self.rng = np.random.default_rng(seed=seed)\n",
    "        self.len = 0\n",
    "\n",
    "        self.current_state = np.zeros((max_size, *observation_space.shape))\n",
    "        self.action = np.zeros((max_size, *action_space.shape), dtype=int)\n",
    "        self.reward = np.zeros(max_size)\n",
    "        self.next_state = np.zeros((max_size, *observation_space.shape))\n",
    "        \n",
    "    def add(self, current_observation: np.ndarray, action: np.ndarray, reward: float, next_observation: np.ndarray, done: bool) -> None:\n",
    "        \"\"\"Add a new entry to the buffer.\n",
    "\n",
    "        Parameters:\n",
    "            current_observation: environment state observed at the current step\n",
    "            action: action taken by the model\n",
    "            reward: reward received after taking the action\n",
    "            next_observation: environment state obversed after taking the action\n",
    "            done: whether the episode has ended or not\"\"\"\n",
    "        self.current_state[self.step] = current_observation\n",
    "        self.action[self.step] = action\n",
    "        self.reward[self.step] = reward\n",
    "        self.next_state[self.step] = next_observation\n",
    "        self.done[self.step] = done\n",
    "        self.step = (self.step + 1) % self.max_size\n",
    "        self.len = min(self.len + 1, self.max_size)\n",
    "        \n",
    "    def sample(self, n_samples: int, replace: bool = True) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:\n",
    "        \"\"\"Randomly samples `n_samples` from the buffer.\n",
    "\n",
    "        Parameters:\n",
    "            n_samples: number of samples to select\n",
    "            replace: sample with or without replacement\n",
    "\n",
    "        Returns:\n",
    "            current observations, actions, rewards, next observations, done\"\"\"\n",
    "        indicies = self.rng.choice(self.len, size=n_samples, replace=replace)\n",
    "        return (\n",
    "            self.current_state[indicies], \n",
    "            self.action[indicies], \n",
    "            self.reward[indicies], \n",
    "            self.next_state[indicies], \n",
    "            self.done[indicies]\n",
    "        )\n",
    "\n",
    "    def clear(self) -> None:\n",
    "        \"\"\"Clears the buffer\"\"\"\n",
    "        self.step = self.len = 0\n",
    "\n",
    "    def __getitem__(self, index: int) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:\n",
    "        \"\"\"Gets a sample at `index`\n",
    "\n",
    "        Parameters:\n",
    "            index: index of the sample to get\n",
    "\n",
    "        Returns:\n",
    "            current observation, action, reward, next observation, done\"\"\"\n",
    "        return (\n",
    "            self.current_state[index], \n",
    "            self.action[index], \n",
    "            self.reward[index], \n",
    "            self.next_state[index], \n",
    "            self.done[index]\n",
    "        )\n",
    "        \n",
    "    def __len__(self) -> int:\n",
    "        \"\"\"Returns the number of entries in the buffer\"\"\"\n",
    "        return self.len"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "669485c5-5787-4ffa-82f2-58f6f0acc151",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc62c869-195b-4323-bdb4-dd53879455c9",
   "metadata": {},
   "source": [
    "Implement your model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a572a086-e9ce-4194-8809-54a8e05477d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_name(prefix: str | None = None, suffix: str | None = None, separator: str = '/') -> str | None:\n",
    "    return prefix and prefix + separator + suffix or suffix or None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "025c3007",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model(\n",
    "    input_features: tuple | int, \n",
    "    features: int,\n",
    "    out_features: tuple | int,\n",
    "    blocks: int, \n",
    "    activation: str | tf.keras.layers.Activation | None = 'silu',\n",
    "    dropout: float = 0.,\n",
    "    multiply_freq: int = 1,\n",
    "    kind_of_model: str | None = None,\n",
    "    name: str | None = None\n",
    ") -> tf.keras.Model:\n",
    "    if kind_of_model == 'policy':\n",
    "        inputs = x = tf.keras.layers.Input((input_features, ), name=get_name(name, 'input'))\n",
    "    else:\n",
    "        input1 = tf.keras.layers.Input((input_features, ), name=get_name(name, 'observation_input'))\n",
    "        input2 = tf.keras.layers.Input((4, ), name=get_name(name, 'action_input'))\n",
    "        x = tf.keras.layers.concatenate([input1, input2])\n",
    "        inputs = [input1, input2]\n",
    "\n",
    "    for i in range(blocks):\n",
    "        x = tf.keras.layers.Dense(features, activation=activation, name=get_name(name, f'dense_{i}'))(x)\n",
    "        if dropout > 0:\n",
    "            x = tf.keras.layers.Dropout(dropout, name=get_name(name, f'dropout_{i}'))(x)\n",
    "\n",
    "        if multiply_freq > 0 and (i + 1) % multiply_freq == 0:\n",
    "            features *= 2\n",
    "    if kind_of_model == 'policy':\n",
    "        x = tf.keras.layers.Dense(out_features, activation='tanh', name=get_name(name, 'prediction'))(x)\n",
    "    else:\n",
    "        x = tf.keras.layers.Dense(out_features, name=get_name(name, 'prediction'))(x)\n",
    "    return tf.keras.Model(inputs=inputs, outputs=x, name=name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d2cc1a3-3e45-4aac-8936-d5d45cc256f1",
   "metadata": {},
   "source": [
    "# Play the game"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02aa11b5-0ca1-4bfb-91a0-33a4f8922ae9",
   "metadata": {},
   "source": [
    "Implement interacting with the environment and storing entries to the replay buffer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b333e6b3-853a-4cb7-aea3-8c501d0247d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def play_game(model: tf.keras.Model, buffer: ReplayBuffer | None, env: gym.Env, max_steps: int, observation: np.ndarray | None = None) -> np.ndarray:\n",
    "    \"\"\"Play game and record\n",
    "\n",
    "    Parameters:\n",
    "        model: the model to get actions with\n",
    "        buffer: replay buffer to store the entries to\n",
    "        env: environment to play\n",
    "        max_steps: maximal number of steps to perform\n",
    "        observation: the observation to resume from\n",
    "\n",
    "    Returns:\n",
    "        the last observation\"\"\"\n",
    "    if observation is None:\n",
    "        observation, _ = env.reset()\n",
    "\n",
    "    buffer = buffer if buffer is not None else ReplayBuffer(1)\n",
    "\n",
    "    for i in range(max_steps):\n",
    "        a = model(observation[None], training=False).numpy()[0] # Observe state `s` and select action `a`\n",
    "        \n",
    "        new_observation, score, done, terminated, _ = env.step(a) # Execute `a` in the environment\n",
    "        \n",
    "        buffer.add(observation, a, score, new_observation, done) # Store `(s, a, r, s', d)` in buffer\n",
    "\n",
    "        if done or terminated: # If `s'` is terminal, reset environment state\n",
    "            observation, _ = env.reset()\n",
    "            continue\n",
    "            \n",
    "        observation = new_observation\n",
    "\n",
    "    return observation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "971ad3a7-0487-49d3-9ca4-6b6bbc3fa450",
   "metadata": {},
   "source": [
    "# Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f9775cbc-77a8-4b02-ab0a-3fbb22b3e958",
   "metadata": {},
   "outputs": [],
   "source": [
    "def ddpg_loss(\n",
    "    current_observation: tf.Tensor, \n",
    "    action: tf.Tensor, \n",
    "    reward: tf.Tensor, \n",
    "    next_observation: tf.Tensor,\n",
    "    done: tf.Tensor,\n",
    "    q_model: tf.keras.Model,\n",
    "    policy_model: tf.keras.Model,\n",
    "    target_q_model: tf.keras.Model,\n",
    "    target_policy_model: tf.keras.Model,\n",
    "    gamma: float\n",
    ") -> tuple[tf.Tensor, tf.Tensor]:\n",
    "    \"\"\"Computes Deep Deterministic Policy Gradient.\n",
    "\n",
    "    Parameters:\n",
    "        current_observation: observations at the current time step\n",
    "        action: actions taken at the current time step\n",
    "        reward: rewards at the current time step\n",
    "        next_observation: observations at the next time step\n",
    "        done: whether the episode has ended or not\n",
    "        q_model: q-function model\n",
    "        policy_model: action prediction model\n",
    "        target_q_model: target q-function model\n",
    "        target_policy_model: target action prediction model\n",
    "        gamma: discount\n",
    "\n",
    "    Returns:\n",
    "        Computed losses for q-function and policy models\"\"\"\n",
    "    q_current = q_model((current_observation, action))\n",
    "    q_next = target_q_model((next_observation, target_policy_model(next_observation)))\n",
    "\n",
    "    q_ref = reward + gamma * (1. - done) * q_next\n",
    "    q_loss = tf.math.reduce_mean(tf.square(q_current - q_ref))\n",
    "\n",
    "    policy_loss = -tf.math.reduce_mean(q_model((current_observation, policy_model(current_observation))))\n",
    "\n",
    "    return q_loss, policy_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78458a1f-054a-463a-934f-ac3f669c0e27",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fb0aaa7-6ae5-45dc-9c52-04c474ec3ecc",
   "metadata": {},
   "source": [
    "Create models, replay buffers, optimizer. Implement training loop, show training progress and perform model evaluation once in a while"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "139138b8-f63c-4a2f-8793-4f96d25a353a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"nogi\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " nogi/observation_input (InputL  [(None, 24)]        0           []                               \n",
      " ayer)                                                                                            \n",
      "                                                                                                  \n",
      " nogi/action_input (InputLayer)  [(None, 4)]         0           []                               \n",
      "                                                                                                  \n",
      " concatenate_7 (Concatenate)    (None, 28)           0           ['nogi/observation_input[0][0]', \n",
      "                                                                  'nogi/action_input[0][0]']      \n",
      "                                                                                                  \n",
      " nogi/dense_0 (Dense)           (None, 16)           464         ['concatenate_7[0][0]']          \n",
      "                                                                                                  \n",
      " nogi/dropout_0 (Dropout)       (None, 16)           0           ['nogi/dense_0[0][0]']           \n",
      "                                                                                                  \n",
      " nogi/dense_1 (Dense)           (None, 16)           272         ['nogi/dropout_0[0][0]']         \n",
      "                                                                                                  \n",
      " nogi/dropout_1 (Dropout)       (None, 16)           0           ['nogi/dense_1[0][0]']           \n",
      "                                                                                                  \n",
      " nogi/dense_2 (Dense)           (None, 32)           544         ['nogi/dropout_1[0][0]']         \n",
      "                                                                                                  \n",
      " nogi/dropout_2 (Dropout)       (None, 32)           0           ['nogi/dense_2[0][0]']           \n",
      "                                                                                                  \n",
      " nogi/dense_3 (Dense)           (None, 32)           1056        ['nogi/dropout_2[0][0]']         \n",
      "                                                                                                  \n",
      " nogi/dropout_3 (Dropout)       (None, 32)           0           ['nogi/dense_3[0][0]']           \n",
      "                                                                                                  \n",
      " nogi/dense_4 (Dense)           (None, 64)           2112        ['nogi/dropout_3[0][0]']         \n",
      "                                                                                                  \n",
      " nogi/dropout_4 (Dropout)       (None, 64)           0           ['nogi/dense_4[0][0]']           \n",
      "                                                                                                  \n",
      " nogi/dense_5 (Dense)           (None, 64)           4160        ['nogi/dropout_4[0][0]']         \n",
      "                                                                                                  \n",
      " nogi/dropout_5 (Dropout)       (None, 64)           0           ['nogi/dense_5[0][0]']           \n",
      "                                                                                                  \n",
      " nogi/dense_6 (Dense)           (None, 128)          8320        ['nogi/dropout_5[0][0]']         \n",
      "                                                                                                  \n",
      " nogi/dropout_6 (Dropout)       (None, 128)          0           ['nogi/dense_6[0][0]']           \n",
      "                                                                                                  \n",
      " nogi/dense_7 (Dense)           (None, 128)          16512       ['nogi/dropout_6[0][0]']         \n",
      "                                                                                                  \n",
      " nogi/dropout_7 (Dropout)       (None, 128)          0           ['nogi/dense_7[0][0]']           \n",
      "                                                                                                  \n",
      " nogi/dense_8 (Dense)           (None, 256)          33024       ['nogi/dropout_7[0][0]']         \n",
      "                                                                                                  \n",
      " nogi/dropout_8 (Dropout)       (None, 256)          0           ['nogi/dense_8[0][0]']           \n",
      "                                                                                                  \n",
      " nogi/dense_9 (Dense)           (None, 256)          65792       ['nogi/dropout_8[0][0]']         \n",
      "                                                                                                  \n",
      " nogi/dropout_9 (Dropout)       (None, 256)          0           ['nogi/dense_9[0][0]']           \n",
      "                                                                                                  \n",
      " nogi/dense_10 (Dense)          (None, 512)          131584      ['nogi/dropout_9[0][0]']         \n",
      "                                                                                                  \n",
      " nogi/dropout_10 (Dropout)      (None, 512)          0           ['nogi/dense_10[0][0]']          \n",
      "                                                                                                  \n",
      " nogi/dense_11 (Dense)          (None, 512)          262656      ['nogi/dropout_10[0][0]']        \n",
      "                                                                                                  \n",
      " nogi/dropout_11 (Dropout)      (None, 512)          0           ['nogi/dense_11[0][0]']          \n",
      "                                                                                                  \n",
      " nogi/prediction (Dense)        (None, 1)            513         ['nogi/dropout_11[0][0]']        \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 527,009\n",
      "Trainable params: 527,009\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = get_model(24, 16, 1, 12, name='nogi', dropout=0.1, multiply_freq=2, activation='swish')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "f5d9d794",
   "metadata": {},
   "outputs": [],
   "source": [
    "target_model = get_model(24, 16, 1, 12, name='target_nogi', multiply_freq=2, activation='swish')\n",
    "target_model.trainable = False\n",
    "target_model.set_weights(model.get_weights())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "f82a7f13",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"policy_model\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " policy_model/input (InputLa  [(None, 24)]             0         \n",
      " yer)                                                            \n",
      "                                                                 \n",
      " policy_model/dense_0 (Dense  (None, 16)               400       \n",
      " )                                                               \n",
      "                                                                 \n",
      " policy_model/dense_1 (Dense  (None, 16)               272       \n",
      " )                                                               \n",
      "                                                                 \n",
      " policy_model/dense_2 (Dense  (None, 32)               544       \n",
      " )                                                               \n",
      "                                                                 \n",
      " policy_model/dense_3 (Dense  (None, 32)               1056      \n",
      " )                                                               \n",
      "                                                                 \n",
      " policy_model/dense_4 (Dense  (None, 64)               2112      \n",
      " )                                                               \n",
      "                                                                 \n",
      " policy_model/dense_5 (Dense  (None, 64)               4160      \n",
      " )                                                               \n",
      "                                                                 \n",
      " policy_model/dense_6 (Dense  (None, 128)              8320      \n",
      " )                                                               \n",
      "                                                                 \n",
      " policy_model/dense_7 (Dense  (None, 128)              16512     \n",
      " )                                                               \n",
      "                                                                 \n",
      " policy_model/dense_8 (Dense  (None, 256)              33024     \n",
      " )                                                               \n",
      "                                                                 \n",
      " policy_model/dense_9 (Dense  (None, 256)              65792     \n",
      " )                                                               \n",
      "                                                                 \n",
      " policy_model/prediction (De  (None, 4)                1028      \n",
      " nse)                                                            \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 133,220\n",
      "Trainable params: 133,220\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "policy_model = get_model(24, 16, 4, 10, name='policy_model', multiply_freq=2, kind_of_model='policy') # Предсказывает действие по состоянию среды\n",
    "policy_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "787e12e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "target_policy_model = get_model(24, 16, 4, 10, name='target_policy_model', multiply_freq=2, kind_of_model='policy')\n",
    "target_policy_model.trainable = False\n",
    "target_policy_model.set_weights(policy_model.get_weights())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "1dc75d26",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_buffer = ReplayBuffer(10000, observation_space=env.observation_space, action_space=env.action_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "a2eff362",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_buffer = ReplayBuffer(100, observation_space=eval_env.observation_space, action_space=eval_env.action_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "2c250ba8",
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.keras.optimizers.Adam(1e-4, clipnorm=5, decay=2e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "a1c708e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 10000\n",
    "save_loss_frequency = 100\n",
    "batch_size = 1024\n",
    "update_frequency = 128\n",
    "eval_frequency = 512\n",
    "steps_per_epoch = 32\n",
    "eval_steps = 1000\n",
    "initial_samples = 1000\n",
    "n_evals = 5\n",
    "eval_threshold = 400\n",
    "polyak = 0.95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "f871b46d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mulpiply_weights(model: tf.keras.Model, target_model: tf.keras.Model, number: float | int) -> list[np.ndarray]:\n",
    "    return [number * target_weights + (1. - number) * model_weights for target_weights, model_weights in zip(target_model.get_weights(), model.get_weights())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "fefb8c9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d54caab27076474eba4ab23d52146920",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "q_losses = []\n",
    "p_losses = []\n",
    "total_q_loss = 0\n",
    "total_p_loss = 0\n",
    "eval_score = 0\n",
    "all_q_loss_saver = []\n",
    "all_p_loss_saver = []\n",
    "\n",
    "s, _ = env.reset()\n",
    "pbar = tqdm.trange(epochs)\n",
    "for i in pbar:\n",
    "    \n",
    "    s = play_game(policy_model, train_buffer, env, steps_per_epoch, observation=s) # Select action, play and store in buffer\n",
    "    \n",
    "    vals = train_buffer.sample(batch_size) # Randomly sample a batch of transitions\n",
    "\n",
    "\n",
    "    with tf.GradientTape(watch_accessed_variables=False) as q_g, tf.GradientTape(watch_accessed_variables=False) as p_g:\n",
    "        q_g.watch(model.trainable_weights)\n",
    "        p_g.watch(policy_model.trainable_weights)\n",
    "        q_loss, policy_loss = ddpg_loss(*vals, model, policy_model, target_model, target_policy_model, 0.99) # MSBE and mean score from Policy\n",
    "\n",
    "    q_gradient = q_g.gradient(q_loss, model.trainable_weights)\n",
    "    optimizer.apply_gradients(zip(q_gradient, model.trainable_weights))\n",
    "\n",
    "    p_gradient = p_g.gradient(policy_loss, policy_model.trainable_weights)\n",
    "    optimizer.apply_gradients(zip(p_gradient, policy_model.trainable_weights))\n",
    "    \n",
    "        \n",
    "    q_losses.append(q_loss.numpy())\n",
    "    p_losses.append(policy_loss.numpy())\n",
    "    \n",
    "    total_q_loss += q_losses[-1]\n",
    "    total_p_loss += p_losses[-1]\n",
    "\n",
    "    if (i + 1) % update_frequency == 0:\n",
    "        target_model.set_weights(mulpiply_weights(model, target_model, polyak))\n",
    "        target_policy_model.set_weights(mulpiply_weights(policy_model, target_policy_model, polyak))\n",
    "\n",
    "    if (i + 1) % eval_frequency == 0:\n",
    "        eval_score = 0\n",
    "\n",
    "        for i in range(n_evals):\n",
    "            eval_buffer.clear()\n",
    "            play_game(policy_model, eval_buffer, eval_env, eval_steps)\n",
    "            eval_score += eval_buffer.reward[:len(eval_buffer)].sum()\n",
    "\n",
    "        eval_score /= n_evals\n",
    "        if eval_score >= eval_threshold:\n",
    "            break\n",
    "    if (i + 1) % save_loss_frequency == 0:\n",
    "        all_q_loss_saver.append(total_q_loss / (i + 1))\n",
    "        all_p_loss_saver.append(total_p_loss / (i + 1))\n",
    "\n",
    "    pbar.set_description(f'Qloss: {q_losses[-1]:.5f}; AllQloss: {total_q_loss / (i + 1):.5f}; Ploss: {p_losses[-1]:.5f}; AllPloss: {total_p_loss / (i + 1):.5f}; E: {eval_score:.5f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "85c440fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8ZklEQVR4nO3deXxU5d3///fsWWdCAmSRhEURRFR2iHhbq2mptdpWtOrXVuvNT+/WoCLWKu2tte3tDa1V0Jba3t4We3/Vm0oVlX5dqqi4FBCiUHBBFDRhSVgzk30mM+f3xywQCJBJZuYkmdfz8TiPzJxz5sxnjph557qucx2LYRiGAAAAUsRqdgEAACC9ED4AAEBKET4AAEBKET4AAEBKET4AAEBKET4AAEBKET4AAEBKET4AAEBK2c0u4EihUEi7du1Sbm6uLBaL2eUAAIAuMAxDDQ0NKikpkdV6/LaNXhc+du3apdLSUrPLAAAA3VBTU6MhQ4Ycd59eFz5yc3MlhYt3u90mVwMAALrC5/OptLQ09j1+PL0ufES7WtxuN+EDAIA+pitDJhhwCgAAUorwAQAAUorwAQAAUqrXjfkAAKAngsGgAoGA2WX0Sw6HQzabrcfHIXwAAPqNxsZG7dixQ4ZhmF1Kv2SxWDRkyBDl5OT06DiEDwBAvxAMBrVjxw5lZWVp0KBBTFSZYIZhaO/evdqxY4dGjhzZoxYQwgcAoF8IBAIyDEODBg1SZmam2eX0S4MGDdLnn3+uQCDQo/DBgFMAQL9Ci0fyJOrcEj4AAEBKET4AAOhj7rnnHo0bN87sMrqN8AEAAFKK8AEAAFIqbcLHHl+r7v1/H2rBix+bXQoAADFNTU265pprlJOTo+LiYt1///0677zzNGfOnC4fIxQK6Re/+IWGDBkil8ulcePG6aWXXopt9/v9mj17toqLi5WRkaGhQ4dq/vz5ksKX0N5zzz0qKyuTy+VSSUmJbr755kR/zA7S5lLbhrZ2PfLWduVm2HXnhaPNLgcAkGSGYaglEDTlvTMdti5fGXL77bdr1apVeu655zR48GD95Cc/0XvvvRfXmI4HH3xQ999/v/74xz9q/Pjx+tOf/qRLLrlEH3zwgUaOHKmHHnpIzz//vJ566imVlZWppqZGNTU1kqSnn35aCxcu1NKlS3X66aertrZWGzdu7M7H7rK0CR/ZzvBHbfYHZRgGl2IBQD/XEghqzN0vm/LeH/5ihrKcJ/6KbWxs1KOPPqrHH39cF1xwgSTpz3/+s4YMGRLX+/3mN7/RHXfcoSuvvFKS9Ktf/Uqvv/66Fi1apMWLF6u6ulojR47UOeecI4vFoqFDh8ZeW11draKiIlVUVMjhcKisrExTpkyJ6/3jlTbdLtmu8GQowZChtvaQydUAACB99tln8vv9mjp1amxdfn6+Ro0a1eVj+Hw+7dq1S9OnT++wfvr06froo48kSd///ve1YcMGjRo1SjfffLP+/ve/x/a7/PLL1dLSohEjRuj666/X8uXL1d7e3sNPdnxp0/JxeAJtamtXhqPnN8YBAPRemQ6bPvzFDNPeuzeZMGGCtm/frhdffFGvvvqqvvOd76iiokJ//etfVVpaqi1btujVV1/VK6+8ohtvvFH33XefVq1aJYfDkZR60qblw2a1xP4xNPvN6QMEAKSOxWJRltNuytLVrv2TTz5ZDodDa9euja07ePCgPvnkky5/TrfbrZKSEr3zzjsd1r/zzjsaM2ZMh/2uuOIKPfLII/rLX/6ip59+WgcOHJAkZWZm6uKLL9ZDDz2kN954Q6tXr9amTZu6XEO80qblQwp3vbQEgmpsS25zEgAAXZGTk6NZs2bp9ttvV0FBgQYPHqyf/vSnslrjaxu4/fbb9bOf/Uwnn3yyxo0bpyVLlmjDhg164oknJEkPPPCAiouLNX78eFmtVi1btkxFRUXKy8vTY489pmAwqKlTpyorK0uPP/64MjMzO4wLSbQ0Cx927Wv0q9lP+AAA9A733XefGhsbdfHFFys3N1e33XabvF5vXMe4+eab5fV6ddttt2nPnj0aM2aMnn/+eY0cOVKSlJubq1//+tfaunWrbDabJk+erBdeeEFWq1V5eXlasGCB5s6dq2AwqDPOOEMrVqxQQUFBMj6uJMliGIaRtKN3g8/nk8fjkdfrldvtTuixL3zwLX2026c//+sUfenUQQk9NgDAXK2trdq+fbuGDx+ujIwMs8vpkfPOO0/jxo3TokWLzC6lg+Od43i+v9NmzIck5USueGmm2wUAANOkVfiIXvHCmA8AAMyTZmM+uNoFAND7vfHGG2aXkFRp1fKRTcsHAACmS6/w4YpOsU74AADALGkWPsLdLk1tdLsAAGCWtAof0QGnTXS7AABgmrjCxz333COLxdJhGT360O3pW1tbVVlZqYKCAuXk5GjmzJmqq6tLeNHdlRPpdmmi2wUAANPE3fJx+umna/fu3bHl7bffjm279dZbtWLFCi1btkyrVq3Srl27dOmllya04J7IctLtAgCA2eK+1NZut6uoqOio9V6vV48++qiefPJJnX/++ZKkJUuW6LTTTtOaNWs0bdq0nlfbQzkMOAUA9AP33HOPnn32WW3YsMHsUrol7paPrVu3qqSkRCNGjNDVV1+t6upqSVJVVZUCgYAqKipi+44ePVplZWVavXp14irugSxX9FJbWj4AADBLXC0fU6dO1WOPPaZRo0Zp9+7d+vnPf65/+Zd/0ebNm1VbWyun06m8vLwOryksLFRtbe0xj9nW1qa2trbYc5/PF98niENsenVaPgAAME1cLR8XXnihLr/8cp155pmaMWOGXnjhBdXX1+upp57qdgHz58+Xx+OJLaWlpd0+1olwtQsAoLc577zzNHv2bM2ePVsej0cDBw7UXXfdpXju+xoKhfSLX/xCQ4YMkcvl0rhx4/TSSy/Ftvv9fs2ePVvFxcXKyMjQ0KFDNX/+fEmSYRi65557VFZWJpfLpZKSEt18880J/5yH69H06nl5eTr11FP16aef6itf+Yr8fr/q6+s7tH7U1dV1OkYkat68eZo7d27suc/nS1oAyY6FD7pdAKDfMwwp0GzOezuyJIuly7v/+c9/1qxZs/Tuu+9q/fr1uuGGG1RWVqbrr7++S69/8MEHdf/99+uPf/yjxo8frz/96U+65JJL9MEHH2jkyJF66KGH9Pzzz+upp55SWVmZampqVFNTI0l6+umntXDhQi1dulSnn366amtrtXHjxm597K7qUfhobGzUZ599pu9973uaOHGiHA6HVq5cqZkzZ0qStmzZourqapWXlx/zGC6XSy6XqydldFl0krGWQFDBkCGbtev/MAAAfUygWfrPEnPe+ye7JGd2l3cvLS3VwoULZbFYNGrUKG3atEkLFy7scvj4zW9+ozvuuENXXnmlJOlXv/qVXn/9dS1atEiLFy9WdXW1Ro4cqXPOOUcWi0VDhw6Nvba6ulpFRUWqqKiQw+FQWVmZpkyZEt/njVNc3S4/+tGPtGrVKn3++ef6xz/+oW9/+9uy2Wy66qqr5PF4NGvWLM2dO1evv/66qqqqdN1116m8vLxXXOkiHZpeXWLcBwCg95g2bZosh7WUlJeXa+vWrQoGT9xS7/P5tGvXLk2fPr3D+unTp+ujjz6SJH3/+9/Xhg0bNGrUKN188836+9//Htvv8ssvV0tLi0aMGKHrr79ey5cvV3t7cr8j42r52LFjh6666irt379fgwYN0jnnnKM1a9Zo0KBBkqSFCxfKarVq5syZamtr04wZM/T73/8+KYV3h8tulc1qUTBkqKktqNwMh9klAQCSxZEVboEw6717kQkTJmj79u168cUX9eqrr+o73/mOKioq9Ne//lWlpaXasmWLXn31Vb3yyiu68cYbdd9992nVqlVyOJLzPRlX+Fi6dOlxt2dkZGjx4sVavHhxj4pKFovFoiynTQ2t7cxyCgD9ncUSV9eHmdauXdvh+Zo1azRy5EjZbLYTvtbtdqukpETvvPOOvvSlL8XWv/POOx26T9xut6644gpdccUVuuyyy/S1r31NBw4cUH5+vjIzM3XxxRfr4osvVmVlpUaPHq1NmzZpwoQJifuQh+nRmI++KMdlD4cPrngBAPQS1dXVmjt3rv7t3/5N7733nn7729/q/vvv7/Lrb7/9dv3sZz/TySefrHHjxmnJkiXasGGDnnjiCUnSAw88oOLiYo0fP15Wq1XLli1TUVGR8vLy9NhjjykYDGrq1KnKysrS448/rszMzA7jQhIt7cIHU6wDAHqba665Ri0tLZoyZYpsNptuueUW3XDDDV1+/c033yyv16vbbrtNe/bs0ZgxY/T8889r5MiRkqTc3Fz9+te/1tatW2Wz2TR58mS98MILslqtysvL04IFCzR37lwFg0GdccYZWrFihQoKCpL1cWUx4rmQOAV8Pp88Ho+8Xq/cbnfCj//N372tjTu8+u9rJqliTGHCjw8AMEdra6u2b9+u4cOHKyMjw+xyuuy8887TuHHjtGjRIrNLOaHjneN4vr/jnl69r4tNNMaYDwAATJF24SN6uS3dLgAAmCPtxnxkc38XAEAv8sYbb5hdQsrR8gEAAFIq/cJH9GoXWj4AADBF2oUP7mwLAP1bL7uIs19J1LlNu/CR4yJ8AEB/FJ0N1O/3m1xJ/xU9t12ZefV40m7AaZYr2u3CmA8A6E/sdruysrK0d+9eORwOWa1p9/d1UoVCIe3du1dZWVmy23sWH9IufNDyAQD9k8ViUXFxsbZv364vvvjC7HL6JavVqrKysg534O2OtAsfhyYZo+UDAPobp9OpkSNH0vWSJE6nMyEtSmkXPqLzfNDyAQD9k9Vq7VPTq6ejtOsQy460fDQTPgAAMEX6hY/ImI9GwgcAAKZIw/ARnV49yLXgAACYIA3DR7jloz1kqK09ZHI1AACkn7QLH1mOQxOjNHPFCwAAKZd24cNus8plD39srngBACD10i58SIdNNMbN5QAASLm0DB+xKdbb6HYBACDV0jJ8ZHNnWwAATJOe4SPS7dJMtwsAACmX1uGjkW4XAABSLj3DhzM60RgtHwAApFp6hg+mWAcAwDTpGT6iLR90uwAAkHLpGT5o+QAAwDRpHT4Y8wEAQOqlZ/hwMskYAABmScvwkcX06gAAmCYtwwcznAIAYJ70DB/c2wUAANOkafhgwCkAAGZJz/DhZHp1AADMkp7hw8X06gAAmCVNw0e02yWoUMgwuRoAANJLeoaPSLeLJDUH6HoBACCV0jJ8ZDisslrCj7ncFgCA1ErL8GGxWJjrAwAAk6Rl+JAOjftgrg8AAFIrbcNHVnSiMa54AQAgpdI2fNDtAgCAOdI3fMRaPuh2AQAgldI3fNDyAQCAKdI3fLgIHwAAmCGNw0d0inW6XQAASKX0DR90uwAAYIq0DR9Z0W4XLrUFACCl0jZ85ESvdmGSMQAAUiptw0cW3S4AAJgibcNHDt0uAACYokfhY8GCBbJYLJozZ05sXWtrqyorK1VQUKCcnBzNnDlTdXV1Pa0z4bKcdLsAAGCGboePdevW6Y9//KPOPPPMDutvvfVWrVixQsuWLdOqVau0a9cuXXrppT0uNNFymOcDAABTdCt8NDY26uqrr9YjjzyiAQMGxNZ7vV49+uijeuCBB3T++edr4sSJWrJkif7xj39ozZo1CSs6EaJXuzDPBwAAqdWt8FFZWamLLrpIFRUVHdZXVVUpEAh0WD969GiVlZVp9erVnR6rra1NPp+vw5IK2ZFul0ZaPgAASCl7vC9YunSp3nvvPa1bt+6obbW1tXI6ncrLy+uwvrCwULW1tZ0eb/78+fr5z38ebxk9lh1r+SB8AACQSnG1fNTU1OiWW27RE088oYyMjIQUMG/ePHm93thSU1OTkOOeSHSG00DQUFs7XS8AAKRKXOGjqqpKe/bs0YQJE2S322W327Vq1So99NBDstvtKiwslN/vV319fYfX1dXVqaioqNNjulwuud3uDksqZEUmGZOkZq54AQAgZeLqdrngggu0adOmDuuuu+46jR49WnfccYdKS0vlcDi0cuVKzZw5U5K0ZcsWVVdXq7y8PHFVJ4DDZpXTbpW/PaQmf7sGZDvNLgkAgLQQV/jIzc3V2LFjO6zLzs5WQUFBbP2sWbM0d+5c5efny+1266abblJ5ebmmTZuWuKoTJMdl14F2P3N9AACQQnEPOD2RhQsXymq1aubMmWpra9OMGTP0+9//PtFvkxBZTpsONDHLKQAAqdTj8PHGG290eJ6RkaHFixdr8eLFPT100jHRGAAAqZe293aRmGIdAAAzpHX4yKblAwCAlEvv8OFkojEAAFItrcNHdK6PRrpdAABImbQOH55MhySpvsVvciUAAKSPtA4fA3NckqR9DYQPAABSJa3Dx6DccPjY29hmciUAAKSP9A4fsZYPwgcAAKmS1uEj1u1CywcAACmT1uEj2u2yv8mvUMgwuRoAANJDWoePgpzwnWyDIUMHmxl0CgBAKqR1+HDYrMrLCl9uu6+R8AEAQCqkdfiQDg063cugUwAAUiLtwweDTgEASC3CRy7hAwCAVEr78EG3CwAAqZX24WNgbviKF2Y5BQAgNQgfsTEfXO0CAEAqpH34iN3fhW4XAABSgvDB1S4AAKRU2oePaLfLgSa/gkyxDgBA0qV9+GCKdQAAUivtw4fDZtWA2BTrdL0AAJBsaR8+pMOueGmg5QMAgGQjfOiwK14aW02uBACA/o/wIVo+AABIJcKHuLkcAACpRPgQE40BAJBKhA9JA3O4vwsAAKlC+JA0MJf7uwAAkCqEDx2aYp1uFwAAko/woUNjPg40tTHFOgAASUb4kJSfHR7zETLEFOsAACQZ4UPhKdajAYSuFwAAkovwERG94oW5PgAASC7CRwQTjQEAkBqEjwgmGgMAIDUIHxGHWj4YcAoAQDIRPiIO3VyOlg8AAJKJ8BER63ZhzAcAAElF+IiI3d+Flg8AAJKK8BHBmA8AAFKD8BExmCnWAQBICcJHRH62UxZLeIr1A020fgAAkCyEjwi7zaoBWcxyCgBAshE+DjMoh4nGAABINsLHYQbm0vIBAECyET4Ow/1dAABIPsLHYeh2AQAg+QgfhxmYy1wfAAAkG+HjMHS7AACQfISPw8Tu70K3CwAASUP4OEz0/i60fAAAkDxxhY+HH35YZ555ptxut9xut8rLy/Xiiy/Gtre2tqqyslIFBQXKycnRzJkzVVdXl/CikyU64HR/k1/twZDJ1QAA0D/FFT6GDBmiBQsWqKqqSuvXr9f555+vb37zm/rggw8kSbfeeqtWrFihZcuWadWqVdq1a5cuvfTSpBSeDAU5LtmsFhkGg04BAEgWi2EYPbqLWn5+vu677z5ddtllGjRokJ588klddtllkqSPP/5Yp512mlavXq1p06Z16Xg+n08ej0der1dut7snpXXL9AWvaWd9i57+4dmaOHRAyt8fAIC+KJ7v726P+QgGg1q6dKmamppUXl6uqqoqBQIBVVRUxPYZPXq0ysrKtHr16mMep62tTT6fr8NippPyMiVJu+pbTK0DAID+Ku7wsWnTJuXk5MjlcukHP/iBli9frjFjxqi2tlZOp1N5eXkd9i8sLFRtbe0xjzd//nx5PJ7YUlpaGveHSKSSvAxJ0k7CBwAASRF3+Bg1apQ2bNigtWvX6oc//KGuvfZaffjhh90uYN68efJ6vbGlpqam28dKhBJaPgAASCp7vC9wOp065ZRTJEkTJ07UunXr9OCDD+qKK66Q3+9XfX19h9aPuro6FRUVHfN4LpdLLpcr/sqThPABAEBy9Xiej1AopLa2Nk2cOFEOh0MrV66MbduyZYuqq6tVXl7e07dJmeiYj531rSZXAgBA/xRXy8e8efN04YUXqqysTA0NDXryySf1xhtv6OWXX5bH49GsWbM0d+5c5efny+1266abblJ5eXmXr3TpDWj5AAAgueIKH3v27NE111yj3bt3y+Px6Mwzz9TLL7+sr3zlK5KkhQsXymq1aubMmWpra9OMGTP0+9//PimFJ0t0wKm3JaDGtnbluOLumQIAAMfR43k+Es3seT4k6Yx7XlZDa7teufVcjSzMNaUGAAD6kpTM89GfHRr3QdcLAACJRvjoxKGJxhh0CgBAohE+OsGgUwAAkofw0QnCBwAAyUP46ARTrAMAkDyEj07Exnx4CR8AACQa4aMT0W6X3fWtCoZ61ZXIAAD0eYSPTgzOdclmtag9ZGhvQ5vZ5QAA0K8QPjpht1lV5GbcBwAAyUD4OIbooFOueAEAILEIH8fA5bYAACQH4eMYCB8AACQH4eMYDt3fhSnWAQBIJMLHMZxEywcAAElB+DiGEiYaAwAgKQgfxxC92qW+OaCmtnaTqwEAoP8gfBxDboZDuRl2SdJuWj8AAEgYwsdxMOgUAIDEI3wcB5fbAgCQeISP44iO+9h5kPABAECiED6Og5YPAAASj/BxHIfGfBA+AABIFMLHcTDXBwAAiUf4OI5o+Kj1tioYMkyuBgCA/oHwcRyFuS7ZrBYFgob2NbaZXQ4AAP0C4eM47DarityRK14Y9wEAQEIQPk4gerktV7wAAJAYhI8T4HJbAAASi/BxAtHwwURjAAAkBuHjBIbmZ0mStu9vNrkSAAD6B8LHCQwfmC1J2r6v0eRKAADoHwgfJzBiUI4kacfBFrUGgiZXAwBA30f4OIGBOU7lZthlGNIXdL0AANBjhI8TsFgsGkHXCwAACUP46IJo18tne5tMrgQAgL6P8NEF0UGn2wgfAAD0GOGjC0YMotsFAIBEIXx0wYiB4W6Xbfto+QAAoKcIH10wbGB4orH65oAONvlNrgYAgL6N8NEFWU67SjzhG8xto+sFAIAeIXx00fDIuA+ueAEAoGcIH10UHfexnXEfAAD0COGji6JXvGzbS7cLAAA9QfjoIub6AAAgMQgfXXRyZJbTL/Y3KxgyTK4GAIC+i/DRRSV5mXLarfIHQ9p5sMXscgAA6LMIH11ks1o0rCA83weX2wIA0H2EjzjEZjpl3AcAAN1G+IhDdK4PWj4AAOg+wkccRgyM3mCOlg8AALqL8BGHEYPodgEAoKcIH3GItnzs9raq2d9ucjUAAPRNcYWP+fPna/LkycrNzdXgwYP1rW99S1u2bOmwT2trqyorK1VQUKCcnBzNnDlTdXV1CS3aLAOynRqQ5ZBE1wsAAN0VV/hYtWqVKisrtWbNGr3yyisKBAL66le/qqamQ1/Et956q1asWKFly5Zp1apV2rVrly699NKEF24WZjoFAKBn7PHs/NJLL3V4/thjj2nw4MGqqqrSueeeK6/Xq0cffVRPPvmkzj//fEnSkiVLdNppp2nNmjWaNm1a4io3yYhBOXqvup6WDwAAuqlHYz68Xq8kKT8/X5JUVVWlQCCgioqK2D6jR49WWVmZVq9e3ekx2tra5PP5Oiy9GTeYAwCgZ7odPkKhkObMmaPp06dr7NixkqTa2lo5nU7l5eV12LewsFC1tbWdHmf+/PnyeDyxpbS0tLslpUR00Ok2Wj4AAOiWboePyspKbd68WUuXLu1RAfPmzZPX640tNTU1PTpeskUvt92+t0mGwQ3mAACIV1xjPqJmz56tv/3tb3rzzTc1ZMiQ2PqioiL5/X7V19d3aP2oq6tTUVFRp8dyuVxyuVzdKcMUZflZslikhrZ27W1o02B3htklAQDQp8TV8mEYhmbPnq3ly5frtdde0/DhwztsnzhxohwOh1auXBlbt2XLFlVXV6u8vDwxFZssw2GLXfGyeZfX5GoAAOh74mr5qKys1JNPPqnnnntOubm5sXEcHo9HmZmZ8ng8mjVrlubOnav8/Hy53W7ddNNNKi8v7xdXukSNLx2gbXub9H51vc4fXWh2OQAA9ClxtXw8/PDD8nq9Ou+881RcXBxb/vKXv8T2Wbhwob7xjW9o5syZOvfcc1VUVKRnnnkm4YWbaXxZniTp/ep6U+sAAKAviqvloysDLDMyMrR48WItXry420X1dhPKBkiSNtTUKxgyZLNaTK4IAIC+g3u7dMOphTnKctrU2NauT/cw3wcAAPEgfHSD3WbVmUM8kqT3qw+aXA0AAH0L4aObxke6Xhj3AQBAfAgf3TS+NE+S9H4NLR8AAMSD8NFN0ZaPrXsa5WsNmFwNAAB9B+GjmwblulSanynDkDbW1JtdDgAAfQbhowfGlzLuAwCAeBE+euDQZGOM+wAAoKsIHz0Qu+Klpp473AIA0EWEjx4YU+yW025VfXNAn+9vNrscAAD6BMJHDzjtVp1xUniysfe+oOsFAICuIHz0EPN9AAAQH8JHDzHTKQAA8SF89FD0ipePaxvU7G83txgAAPoAwkcPFXsyVOh2KRgytGmH1+xyAADo9QgfPWSxWDQh0vXyHl0vAACcEOEjAaJdL+s/P2BuIQAA9AGEjwQ4++SBkqR3PtunFn/Q5GoAAOjdCB8JcHqJWyflZao1ENI7n+4zuxwAAHo1wkcCWCwWVZw2WJL0yod1JlcDAEDvRvhIkIoxhZKklR/XKRjiPi8AABwL4SNBpg4vUK7Lrn2Nfm2oqTe7HAAAei3CR4I47VadN5quFwAAToTwkUBfiXS9vPJhrcmVAADQexE+EuhLpw6S3WrRZ3ubtG1vo9nlAADQKxE+EsiT6dC0EQWSpFc/ousFAIDOED4SLNr18uqHe0yuBACA3onwkWAXROb7WP/FAR1o8ptcDQAAvQ/hI8GGDMjSmGK3Qoa0kq4XAACOQvhIgljXC+EDAICjED6SIBo+3vxkn1oD3GgOAIDDET6S4PQSt0o8GWoJBLXyIwaeAgBwOMJHElgsFl06YYgk6Ym1X5hcDQAAvQvhI0mumlomq0X6x2f79RkTjgEAEEP4SJKT8jL15VHhy26fXFttcjUAAPQehI8k+u60oZKkv1btYOApAAARhI8kOvfUQRoyIFPeloBWbNxldjkAAPQKhI8kslkt+j9TyyRJj9P1AgCAJMJH0n1nUqkcNos21tRr806v2eUAAGA6wkeSDcxx6cKxxZK47BYAAInwkRLRgafPvr9LvtaAydUAAGAuwkcKTB42QKcW5qglENTy93aaXQ4AAKYifKSAxWLR1VPDrR9L3tmu9mDI5IoAADAP4SNFLps4RAXZTn2+v1l/rdphdjkAAJiG8JEi2S67bvzyKZKkB1duZdIxAEDaInyk0NVTy1TiydBub6seX8OVLwCA9ET4SKEMh01zKk6VJC1+/VM1cOULACANET5S7NIJJ2nEoGwdbA7ov9/abnY5AACkHOEjxew2q277yihJ0n+/tU37G9tMrggAgNQifJjgwrFFGnuSW03+oH7/xmdmlwMAQEoRPkxgtVp0+4zRkqT/u+YL7axvMbkiAABSh/BhknNHDtS0Efnyt4d017ObZRiG2SUBAJAShA+TWCwW/fKbY+W0WfXax3v07AamXQcApIe4w8ebb76piy++WCUlJbJYLHr22Wc7bDcMQ3fffbeKi4uVmZmpiooKbd26NVH19isjC3N1S8VISdI9z3+oPQ2tJlcEAEDyxR0+mpqadNZZZ2nx4sWdbv/1r3+thx56SH/4wx+0du1aZWdna8aMGWpt5Yu1MzecO0JjT3LL2xKg+wUAkBYsRg++7SwWi5YvX65vfetbksKtHiUlJbrtttv0ox/9SJLk9XpVWFioxx57TFdeeeUJj+nz+eTxeOT1euV2u7tbWp/y4S6fLvnd22oPGfrd/xmvb5xZYnZJAADEJZ7v74SO+di+fbtqa2tVUVERW+fxeDR16lStXr2609e0tbXJ5/N1WNLNmBK3KiP3fbn7uQ+Y+wMA0K8lNHzU1tZKkgoLCzusLywsjG070vz58+XxeGJLaWlpIkvqMyq/fIpGF+XqQJNfdz1H9wsAoP8y/WqXefPmyev1xpaamhqzSzKF027VfZedJZvVohc21erRt5l6HQDQPyU0fBQVFUmS6urqOqyvq6uLbTuSy+WS2+3usKSrM4Z49O8XnSZJ+s8XPtLbW/eZXBEAAImX0PAxfPhwFRUVaeXKlbF1Pp9Pa9euVXl5eSLfqt/6/tnDdNnEIQoZ0uz/fU81B5rNLgkAgISKO3w0NjZqw4YN2rBhg6TwINMNGzaourpaFotFc+bM0X/8x3/o+eef16ZNm3TNNdeopKQkdkUMjs9iseg/vjVWZw3xqL45oOv/Z72a/e1mlwUAQMLEHT7Wr1+v8ePHa/z48ZKkuXPnavz48br77rslST/+8Y9100036YYbbtDkyZPV2Niol156SRkZGYmtvB/LcNj0h+9N1MAclz6ubdDtf/0nA1ABAP1Gj+b5SIZ0nOfjWNZ/fkBXPbJGgaChyi+fHLsZHQAAvY1p83wgsSYNy9cvvzlWkrT49c/025VMUw8A6PsIH73clVPK9NOvh6+Auf+VT/TIm9tMrggAgJ4hfPQB1587Qrd95VRJ0r0vfKT/Wf25uQUBANADhI8+4qYLRqryyydLCk/BvvTdapMrAgCgewgffciPvjpKs84ZLkm685lNeviNz7gKBgDQ5xA++hCLxaJ/v+g0/X+RAPKrlz7Wvz+7We3BkMmVAQDQdYSPPsZisejfvzFGd39jjCwW6Ym11brh/1apqY2JyAAAfQPho4/613OG6+GrJ8plt+q1j/foyv9aoz2+VrPLAgDghAgffdjXxhbpf2+Ypvxspzbt9OrrD72tdz7lZnQAgN6N8NHHTSgboGd+eLZGFeZqX2ObvvvoWi169RMFQwxEBQD0ToSPfmDYwGw9WzldV0wqlWFIi17dqu89ulZ7GuiGAQD0PoSPfiLTadOvLjtTD3znLGU6bPrHZ/v19Qff0oubdptdGgAAHRA++plLJwzRipumR7ph/PrhE+/ph49X0QoCAOg1CB/90CmDc/X8TdN10/mnyG616MXNtfrKA2/qr1U7mJQMAGA6wkc/5bLbdNtXR+n52efojJM88rYE9KNlG3XVI2v0wS6v2eUBANIY4aOfG1Pi1vIbz9adF46Wy27Vmm0H9I3fvq07n/4nXTEAAFNYjF7WDu/z+eTxeOT1euV2u80up1/ZcbBZv3ppi1Zs3CVJynbadOOXT9F104cpy2k3uToAQF8Wz/c34SMNVX1xQL9Y8aE27gh3vxRkO/VvXxqh704bSggBAHQL4QMnFAoZenbDTi16dauqDzRLkgbmOPVv556s704bqkynzeQKAQB9CeEDXRYIhrT8vZ367etbVXOgRZKUl+XQd6cO1TVnD9Xg3AyTKwQA9AWED8QtGkJ+9/qnsZYQp82qS8aVaNY5w3VaMf8tAADHRvhAtwVDhv7+Qa0eeWub3quuj62fPGyArppSpq+fUawMB10yAICOCB9IiKovDurRt7fp5Q/qYjeq82Q6dOmEk3TF5FKNLuK/DwAgjPCBhKrzteqpdTVauq5GO+tbYutPK3br2+NLdMlZJ6nIw9gQAEhnhA8kRTBk6K2te7X03Rqt/LhOgWD4n47FIpWPKNBFZxbrq2OKNCjXZXKlAIBUI3wg6eqb/fp/m3br2fd3at3nB2PrrRZp8rB8XTi2SDPGFqnYk2lilQCAVCF8IKVqDjRrxT936aXNtfrnjo73jRlT7NYFpw3Wl0cP1llD8mSzWkyqEgCQTIQPmGbHwWa9tLlWL22uVVX1QR3+r6sg26lzRg7UOacM1L+MHMQ4EQDoRwgf6BX2N7Zp1Sd7tfLjPXrzk71qaG3vsP2UwTk655SBmjYiX1OGFyg/22lSpQCAniJ8oNcJBEOq+uKg3t66T299uk//3FGvI//ljSrM1dQR+Zo8LF8Thw5QSR7jRQCgryB8oNerb/brH5/t1+rP9mvNtv3auqfxqH2KPRmaOHSAJpQN0LiyPI0pdjPBGQD0UoQP9Dn7G9v07vYDWrv9gKq+OKgPd/tiE5tF2a0WjS7O1VlD8nTmEI9OL/Ho1MJcOe1Wk6oGAEQRPtDnNfvbtaGmXlWfH9SGmnpt3FGvfY3+o/Zz2Cw6tTBXp5e4dVpxeBldlKu8LMaPAEAqET7Q7xiGoV3eVm2sqdfGmnpt3uXV5p0+eVsCne5f7MnQqKJcnVqYq5GDc8I/C3OU5bSnuHIASA+ED6QFwzC042CLPtjl1Qe7fPpod4M+rvVpx8GWY77mpLxMjRiUrVMG5+jkQTkaMShbIwbmqNDtksXCHCQA0F2ED6Q1X2tAH+9u0Cd1Ddpa16BP6hq1dU9Dp902UZkOm4YNzNbwgVkaWpCtoflZKivI0rCCbBW5M2RlcjQAOC7CB9CJg01+fba3MbI06dM9jdq+r0nVB5qPGtx6OKfNqiEDMjUkP0ulAzJVmp+lIQMydVJepk4akKlBObSaAEA83990gCNtDMh2alJ2viYNy++wPhAMqeZAsz7f36Rte8Nh5Iv9zfpif5N2HGyRPxjStn1N2ravqdPjOu1WnZSXqWJPhkryMlXiyVBx5HmxJ1NF7gy5M+0EFACIoOUDOI72YEi7va2qOdisHQdaVHOwWdUHmrXzYIt21reozteq4zSaxGQ6bCryZKjQ7VKhO0NF7gwNdmdocK4rvEQeZ7v4ewBA30TLB5AgdptVpflZKs3Pkk4+ensgGFKtt1U7DrZot7dFu72t2lnfot314ce1vlbVNwfUEghq+74mbT9G60lUttOmQbkuDcxxxX7mZzs1IMuhvCynPFkOeTIdynXZleWyK9tpU5bTLofNokDQUHsoFP4ZDP/0t4fkD4YUiCztIUOhkKH2kKFgyFBbe1At/pBaAkG1Rpa29lDsZ1sgKJvVqhyXLfZ+2S673BkOebIc4Z+Z4SXDYaV1B0CXED6AHnAcHk6OocUfVJ2vVbu9rdrT0Ko6X6tqvW2q87Vqb0Ob9jS0ak9Dm5r9QTX5g2ra36zP9zen8FMkhtNulSfTobxMh/IiIcmdeSicRBd3Rni9O9Mee5zttBFcgDRC+ACSLNMZvpJm2MDs4+7X2NauPb5W7Wv0a19jm/Y2hJcDzX55mwOqb/Grvjmg+uaAmv3tavIH5W8PHfN4DptFDptVDptVTrtVDqtFNptFdqtVNqtFNotFLodVGQ6bMh02ZUQeZ9gPPXbarWoPGWpqa1dTW1BNbe1qbGtXQ2tA3paAfK3t8rYEFAyFW1miNcfLapHcmQ7lZtiV64r8zDgUUHJc9ti6nIzIY5ddORn28DaXQ9kum+w2ZrsF+gLCB9BL5LjsyhmUoxGDuv6aQDCkZn9Q7cGQ7DarHJFw4bBZUtaSYBiGmv1B1bcEVB8LSuFwcuTiiy6R0OJrCYS7ggzFgpV07HlaTiTDYQ2fR5dd2ZElx2VXltN2xLpwd9Xh22LdWC67shw2ZblsctroSgKSgfAB9GEOm1WeTHP/2rdYLLEv9ZPivBOxYRhqDYTkaw1EWlPCrSoNre2RdeHnja3t4ceRVpemtmCkBSb8vC3SAtQaCKk14D/unC7xsFktynLalOW0KdtpV2bkcaYzHFSiz7OcdmU6Dj0//HG0ZSm6T4bTGt7uoKUG6YvwAcA0FotFmZEv8UJ3RrePEwiG1BQJI03+djW2hruHol1FDW3t4a4jf3usC6mxrV0t/qCa/O1qbgvGtjX7g7EwEwwZkYDTLin+7qQTcdgs4W4u56Gur0yHTS5HNLQc0RXmDD92OayRdYe6yFx2a+y5y37op+uw57TkoLcgfADo8xw2q/KynAm7oWB7MKTmQFDNbUE1+8OBpDkSVFoij1si426a/eGrhJr97ZErh9oj24NqCQRj+0evJmoOBBWd4CAQNBQIhsNRqrjs1vASCSzhJRJOIo9d9kOPnbF9rJ2ud0YX26HHLrtVTpvt0LbDtrvs4XFINmYNTmuEDwA4gt1mldtmlTvDkfBjG4YhfzCkFn9QrYFQLKC0BIJqCwQjlz13vPy5NbKuNRBUa/uhx7HLogOhyPojL5UOrz98Nqe29lC4Zac1dYGnMzarRc7IOCVnJPCEH1s7DJQ+tE9kXXQAdWQfhz16nOjS8RiHD7zuymO7zSKHNXzcVI+fSieEDwBIIYvFEmldsKXk/aJhx98eUmsgpLbDwkt0fTSwRB+HfwY7PO647eh9osc66nF0/2DHK7OCIUMtoaDCN6Y2NwidiN1qORRMjhjYbbdZO263hn/abeErzKLr7VZLh0Hh0WPZrBY5Itti+0WObY1clRb9abNaZLGE/w1ZLZJFkeeSDuWjo4OSxRL+DDbroffOdtp1xhBPKk9jB4QPAOjHDg87ud0fVtNjhmGEJ747LJgEgqGjnx8eXoIhtR/xmuhEeofvHz6OEXveHgrJ3x6ddC+6T+R5uxF73/bgoceByPP2TqYsbg8Zag8FpYAJJy5JRgzM1ms/Os+09yd8AACSzmKxyGkPd4nIZXY1xxYNSbGgEjoUUqKzBAeCHWcSbu+wzxEzDYfCP6PBpsO6kBFZf2j/YMhQIGQoGDlmyAjPRhw0pFDIkCFDoZAUMgwZhmTIiNQdqV9Ht32EjPDrY8cPho47MWIqED4AAIiIhSRZpcSMX0YnuMgcAACkVNLCx+LFizVs2DBlZGRo6tSpevfdd5P1VgAAoA9JSrfLX/7yF82dO1d/+MMfNHXqVC1atEgzZszQli1bNHjw4GS85Ym11Evv/Y9kc0hW+2E/neHF7govNpdkz4g8P/ynM/zT5jx8WDEApKfYIIPoAE2j4+Mjt3V43sn+R+4b2984+vGRx+r0OMep43jH7laNnRyzs/ft9HkiajxGvcc7jj1DOmmCzGIxjA6fIiGmTp2qyZMn63e/+50kKRQKqbS0VDfddJPuvPPO477W5/PJ4/HI6/XK7XYnrqh9W6XfTUrMsWzOcEixOQ6FF5tdskYCjdV26KfFJlmskceWQ89j66ySLJFtkfUWS2Td4Y8tHfeLDimyqON2qZPHOvbzDut09LZj7tOZE+3ThX9qnf5zNMzbp9P9jvEL4JiH6cov4hPse8LXH+eX5Al/SXf2+LCfXX6tOt+3w7ou/CI95mc90Xk84pfwcevozrFP9EVyovq7epx4vpA6O04n79PtL/ZjvBZ938BTpdnrEnrIeL6/E97y4ff7VVVVpXnz5sXWWa1WVVRUaPXq1Uft39bWpra2Q9MW+3y+RJcU5syWzrxSCgWkYEAKtYeXoF9q90vBNqm9TWpvDT9vbw0/D0bWHS7oDy8AgF6uK394dbLPif5wO+4feYe9d1x/AHb1fTvZ93jv2dnr8spkpoSHj3379ikYDKqwsLDD+sLCQn388cdH7T9//nz9/Oc/T3QZR3OXSJf+sXuvNYxISGntGFSCgUiY8XcMNKF2KRQ89NMISUZQCoUijyPPjVB4e/SvCyN09F+dx1oXrevIv1C7+tfX4Z+t44ft/POfaJ8eOaLFpLutLN1qwTnOL6LjHup4xz7yl8Sx1p3g/eP6ZRlZZznOL5tEP+7s8xzvF/KJPmNPfkl3KONYXw4nqiPOX/ad7nu8bQn6Yjvm9iMen+i/XVJqPPztj3H+j/v/RYL+e9A13uuZfqntvHnzNHfu3Nhzn8+n0tJSEyvqhMVyaEwIAADokYSHj4EDB8pms6murq7D+rq6OhUVFR21v8vlksvFlzoAAOki4ZfaOp1OTZw4UStXroytC4VCWrlypcrLyxP9dgAAoI9JSrfL3Llzde2112rSpEmaMmWKFi1apKamJl133XXJeDsAANCHJCV8XHHFFdq7d6/uvvtu1dbWaty4cXrppZeOGoQKAADST1Lm+eiJpM3zAQAAkiae72/u7QIAAFKK8AEAAFKK8AEAAFKK8AEAAFKK8AEAAFKK8AEAAFKK8AEAAFKK8AEAAFLK9LvaHik655nP5zO5EgAA0FXR7+2uzF3a68JHQ0ODJKm0tNTkSgAAQLwaGhrk8XiOu0+vm149FApp165dys3NlcViSeixfT6fSktLVVNTw9TtSca5Th3OdepwrlOHc506iTrXhmGooaFBJSUlslqPP6qj17V8WK1WDRkyJKnv4Xa7+cecIpzr1OFcpw7nOnU416mTiHN9ohaPKAacAgCAlCJ8AACAlEqr8OFyufSzn/1MLpfL7FL6Pc516nCuU4dznTqc69Qx41z3ugGnAACgf0urlg8AAGA+wgcAAEgpwgcAAEgpwgcAAEiptAkfixcv1rBhw5SRkaGpU6fq3XffNbukPm/+/PmaPHmycnNzNXjwYH3rW9/Sli1bOuzT2tqqyspKFRQUKCcnRzNnzlRdXZ1JFfcfCxYskMVi0Zw5c2LrONeJs3PnTn33u99VQUGBMjMzdcYZZ2j9+vWx7YZh6O6771ZxcbEyMzNVUVGhrVu3mlhx3xQMBnXXXXdp+PDhyszM1Mknn6xf/vKXHe4NwrnuvjfffFMXX3yxSkpKZLFY9Oyzz3bY3pVze+DAAV199dVyu93Ky8vTrFmz1NjY2PPijDSwdOlSw+l0Gn/605+MDz74wLj++uuNvLw8o66uzuzS+rQZM2YYS5YsMTZv3mxs2LDB+PrXv26UlZUZjY2NsX1+8IMfGKWlpcbKlSuN9evXG9OmTTPOPvtsE6vu+959911j2LBhxplnnmnccsstsfWc68Q4cOCAMXToUOP73/++sXbtWmPbtm3Gyy+/bHz66aexfRYsWGB4PB7j2WefNTZu3GhccsklxvDhw42WlhYTK+977r33XqOgoMD429/+Zmzfvt1YtmyZkZOTYzz44IOxfTjX3ffCCy8YP/3pT41nnnnGkGQsX768w/aunNuvfe1rxllnnWWsWbPGeOutt4xTTjnFuOqqq3pcW1qEjylTphiVlZWx58Fg0CgpKTHmz59vYlX9z549ewxJxqpVqwzDMIz6+nrD4XAYy5Yti+3z0UcfGZKM1atXm1Vmn9bQ0GCMHDnSeOWVV4wvfelLsfDBuU6cO+64wzjnnHOOuT0UChlFRUXGfffdF1tXX19vuFwu43//939TUWK/cdFFFxn/+q//2mHdpZdealx99dWGYXCuE+nI8NGVc/vhhx8akox169bF9nnxxRcNi8Vi7Ny5s0f19PtuF7/fr6qqKlVUVMTWWa1WVVRUaPXq1SZW1v94vV5JUn5+viSpqqpKgUCgw7kfPXq0ysrKOPfdVFlZqYsuuqjDOZU414n0/PPPa9KkSbr88ss1ePBgjR8/Xo888khs+/bt21VbW9vhXHs8Hk2dOpVzHaezzz5bK1eu1CeffCJJ2rhxo95++21deOGFkjjXydSVc7t69Wrl5eVp0qRJsX0qKipktVq1du3aHr1/r7uxXKLt27dPwWBQhYWFHdYXFhbq448/Nqmq/icUCmnOnDmaPn26xo4dK0mqra2V0+lUXl5eh30LCwtVW1trQpV929KlS/Xee+9p3bp1R23jXCfOtm3b9PDDD2vu3Ln6yU9+onXr1unmm2+W0+nUtddeGzufnf1O4VzH584775TP59Po0aNls9kUDAZ177336uqrr5YkznUSdeXc1tbWavDgwR222+125efn9/j89/vwgdSorKzU5s2b9fbbb5tdSr9UU1OjW265Ra+88ooyMjLMLqdfC4VCmjRpkv7zP/9TkjR+/Hht3rxZf/jDH3TttdeaXF3/8tRTT+mJJ57Qk08+qdNPP10bNmzQnDlzVFJSwrnu5/p9t8vAgQNls9mOGvVfV1enoqIik6rqX2bPnq2//e1vev311zVkyJDY+qKiIvn9ftXX13fYn3Mfv6qqKu3Zs0cTJkyQ3W6X3W7XqlWr9NBDD8lut6uwsJBznSDFxcUaM2ZMh3WnnXaaqqurJSl2Pvmd0nO333677rzzTl155ZU644wz9L3vfU+33nqr5s+fL4lznUxdObdFRUXas2dPh+3t7e06cOBAj89/vw8fTqdTEydO1MqVK2PrQqGQVq5cqfLychMr6/sMw9Ds2bO1fPlyvfbaaxo+fHiH7RMnTpTD4ehw7rds2aLq6mrOfZwuuOACbdq0SRs2bIgtkyZN0tVXXx17zLlOjOnTpx91yfgnn3yioUOHSpKGDx+uoqKiDufa5/Np7dq1nOs4NTc3y2rt+DVks9kUCoUkca6TqSvntry8XPX19aqqqort89prrykUCmnq1Kk9K6BHw1X7iKVLlxoul8t47LHHjA8//NC44YYbjLy8PKO2ttbs0vq0H/7wh4bH4zHeeOMNY/fu3bGlubk5ts8PfvADo6yszHjttdeM9evXG+Xl5UZ5ebmJVfcfh1/tYhic60R59913Dbvdbtx7773G1q1bjSeeeMLIysoyHn/88dg+CxYsMPLy8oznnnvO+Oc//2l885vf5PLPbrj22muNk046KXap7TPPPGMMHDjQ+PGPfxzbh3PdfQ0NDcb7779vvP/++4Yk44EHHjDef/9944svvjAMo2vn9mtf+5oxfvx4Y+3atcbbb79tjBw5kktt4/Hb3/7WKCsrM5xOpzFlyhRjzZo1ZpfU50nqdFmyZElsn5aWFuPGG280BgwYYGRlZRnf/va3jd27d5tXdD9yZPjgXCfOihUrjLFjxxoul8sYPXq08V//9V8dtodCIeOuu+4yCgsLDZfLZVxwwQXGli1bTKq27/L5fMYtt9xilJWVGRkZGcaIESOMn/70p0ZbW1tsH851973++uud/o6+9tprDcPo2rndv3+/cdVVVxk5OTmG2+02rrvuOqOhoaHHtVkM47Cp5AAAAJKs34/5AAAAvQvhAwAApBThAwAApBThAwAApBThAwAApBThAwAApBThAwAApBThAwAApBThAwAApBThAwAApBThAwAApBThAwAApNT/D+8ue62UPx0nAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(list(range(len(all_q_loss_saver))), all_q_loss_saver, label='q loss')\n",
    "plt.plot(list(range(len(all_p_loss_saver))), all_p_loss_saver, label='p loss')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "27b8c6b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_weights(f'./models/nogi_model_{eval_score}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "56570f3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-10.068967595714286"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3bd5b44e-55cd-43bb-89dd-87c28cc10a9c",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "965e5731-8d24-4482-82ed-7226fb18fa5c",
   "metadata": {},
   "source": [
    "Test the model on the environment and get a cool video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "72ec281c-d005-499a-b416-4d0e43437a0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_gameplay(model: tf.keras.Model, render_mode: str = 'human', n_frames: int = 1000, buffer_capacity: int = 1000):\n",
    "    env = gym.make('BipedalWalker-v3', hardcore=False, render_mode=render_mode)\n",
    "    buffer = ReplayBuffer(buffer_capacity, env.observation_space, env.action_space)\n",
    "    play_game(model, buffer, env, n_frames)\n",
    "    # save_video(env.render(), './videos', durations=[1] * len(), fps=24) if you wanna use this line change 'render_mode' -> 'rgb_array_list'\n",
    "    return buffer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec9bccad",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('./models/nogi_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "57e4ea04",
   "metadata": {},
   "outputs": [],
   "source": [
    "buffer = save_gameplay(policy_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "49e78353",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-100.57728555658645"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "buffer.reward.sum()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}