MVARL19_part1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4P3WM-hVOPfo"
   },
   "source": [
    "# Reinforcement Learning in Finite MDPs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "E9_DLZvWQzhb"
   },
   "outputs": [],
   "source": [
    "!git clone https://github.com/rlgammazero/mvarl_hands_on.git > /dev/null 2>&1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wnzUJeyJOPfq"
   },
   "source": [
    "## MDPs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RcWJSw_uOPfr"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, './mvarl_hands_on/utils')\n",
    "import numpy as np\n",
    "from scipy.special import softmax # for SARSA\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "import math\n",
    "from cliffwalk import CliffWalk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RcWJSw_uOPfr"
   },
   "outputs": [],
   "source": [
    "from test_env import ToyEnv1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ym-B_4HaOPfu"
   },
   "source": [
    "Setting up the environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rVR5qYoLOPfv"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Set of states: range(0, 48)\n",
      "Set of actions: [0, 1, 2, 3]\n",
      "Number of states:  48\n",
      "Number of actions:  4\n",
      "P has shape:  (48, 4, 48)\n",
      "discount factor:  0.99\n",
      "\n",
      "initial state:  36\n",
      "reward at (s=1, a=3,s'=2):  0.6\n",
      "\n",
      "random policy =  [3 3 1 3 2 3 3 2 3 0 2 3 1 3 3 2 2 1 1 2 2 0 2 2 2 3 1 2 1 3 3 0 0 3 2 2 0\n",
      " 0 0 1 3 0 2 2 1 2 1 2]\n",
      "(s, a, s', r):\n",
      "36 0 37 0.6\n",
      "37 0 38 0.0\n",
      "38 0 39 0.0\n",
      "39 1 40 0.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "env = CliffWalk(proba_succ=0.98)\n",
    "\n",
    "## Smaller env!\n",
    "#env = ToyEnv1(gamma=0.99)\n",
    "\n",
    "# Useful attributes\n",
    "print(\"Set of states:\", env.states)\n",
    "print(\"Set of actions:\", env.actions)\n",
    "print(\"Number of states: \", env.Ns)\n",
    "print(\"Number of actions: \", env.Na)\n",
    "print(\"P has shape: \", env.P.shape)  # P[s, a, s'] = env.P[s, a, s']\n",
    "print(\"discount factor: \", env.gamma)\n",
    "print(\"\")\n",
    "\n",
    "# Usefult methods\n",
    "state = env.reset() # get initial state\n",
    "print(\"initial state: \", state)\n",
    "print(\"reward at (s=1, a=3,s'=2): \", env.reward_func(1,3,2))\n",
    "print(\"\")\n",
    "\n",
    "# A random policy\n",
    "policy = np.random.randint(env.Na, size = (env.Ns,))\n",
    "print(\"random policy = \", policy)\n",
    "\n",
    "# Interacting with the environment\n",
    "print(\"(s, a, s', r):\")\n",
    "for time in range(4):\n",
    "    action = policy[state]\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "    print(state, action, next_state, reward)\n",
    "    if done:\n",
    "        break\n",
    "    state = next_state\n",
    "print(\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AUlNvT3cOPfx"
   },
   "source": [
    "## Question 1: Value iteration\n",
    "1. Write a function applying the optimal Bellman operator on a provided Q function: $Q_1 = LQ_0, \\; Q_0\\in \\mathbb{R}^{S\\times A}$\n",
    "2. Write a function implementing Value Iteration (VI) with $\\infty$-norm stopping condition (reuse function implemented in 1)\n",
    "3. Evaluate the convergence of your estimate, i.e., plot the value $\\|V_n - V^\\star\\|_{\\infty}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\n",
    "\\newcommand\\calA{\\mathcal A}\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Recall** The Bellman operator operates on an input value function $Q_0$ as\n",
    "$$\n",
    "    LQ_0(s,a) = \\sum_{s'} \\left(r(s, a, s') + \\gamma \\max_{a'\\in\\calA}Q_0(s',a')\\right) p(s'|s, a)\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "R8TLRx6MOPfy"
   },
   "outputs": [],
   "source": [
    "# --------------\n",
    "# Point 1\n",
    "# --------------\n",
    "def bellman_operator(Q0, R, P, gamma):\n",
    "    r\"\"\"\n",
    "    Args:\n",
    "        Q0: input state-action value func\n",
    "        R: pre-computed S x A x S reward function values\n",
    "        P: proba transition matrix\n",
    "    \"\"\"\n",
    "    \n",
    "    Q_max_act = np.max(Q0, axis=1)[None,None,...]\n",
    "    Q1 = np.sum((R + gamma * Q_max_act) * P, axis=2)\n",
    "    greedy_policy = np.argmax(Q1, axis=1)\n",
    "    \n",
    "    return Q1, greedy_policy\n",
    "\n",
    "# --------------\n",
    "# Point 2\n",
    "# --------------\n",
    "def value_iteration(Q0, env, epsilon=1e-5, print_every=100):\n",
    "    # TODO (feel free to change the return argument)\n",
    "\n",
    "    # Precompute the S x A x S reward tensor\n",
    "    import itertools\n",
    "    R_ = np.empty((env.Ns, env.Na, env.Ns))\n",
    "    for s, a, sprime in itertools.product(env.states, env.actions, env.states):\n",
    "        R_[s, a, sprime] = env.reward_func(s, a, sprime)\n",
    "\n",
    "    Q = Q0.copy()\n",
    "    Q_history = []\n",
    "    Q_history.append(Q)\n",
    "    norm = np.inf\n",
    "    its = 0\n",
    "    norm_vals = [norm]\n",
    "    while norm > epsilon:\n",
    "        if its % print_every == 0:\n",
    "            print(\"Iteration #%d -- norm=%.3e\" % (its, norm))\n",
    "        Q_new, greedy_policy = bellman_operator(Q, R_, env.P, env.gamma)\n",
    "        deltaV = np.absolute(Q_new.max(axis=1) - Q.max(axis=1))\n",
    "        norm = np.max(deltaV)\n",
    "        norm_vals.append(norm)\n",
    "        # Set Q as Q_new\n",
    "        Q = Q_new\n",
    "        # add to history\n",
    "        Q_history.append(Q)\n",
    "        \n",
    "        its += 1\n",
    "    \n",
    "    return Q, greedy_policy, Q_history, norm_vals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## SANITY TEST FOR BELLMAN_OPERATOR\n",
    "\n",
    "import itertools\n",
    "\n",
    "R_ = np.empty((env.Ns, env.Na, env.Ns))\n",
    "for s, a, sprime in itertools.product(env.states, env.actions, env.states):\n",
    "    R_[s, a, sprime] = env.reward_func(s, a, sprime)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q0 shape (48, 4)\n"
     ]
    }
   ],
   "source": [
    "## Sanity test\n",
    "Q0_ = np.random.rand(env.Ns, env.Na)\n",
    "print('Q0 shape', Q0_.shape)\n",
    "Q1_, pol_ = bellman_operator(Q0_, R_, env.P, env.gamma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "W_lBe6q6OPf2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration #0 -- norm=inf\n",
      "Iteration #50 -- norm=4.093e-01\n",
      "Iteration #100 -- norm=2.370e-01\n",
      "Iteration #150 -- norm=1.411e-01\n",
      "Iteration #200 -- norm=8.475e-02\n",
      "Iteration #250 -- norm=5.103e-02\n",
      "Iteration #300 -- norm=3.078e-02\n",
      "Iteration #350 -- norm=1.857e-02\n",
      "Iteration #400 -- norm=1.121e-02\n",
      "Iteration #450 -- norm=6.773e-03\n",
      "Iteration #500 -- norm=4.094e-03\n",
      "Iteration #550 -- norm=2.474e-03\n",
      "Iteration #600 -- norm=1.496e-03\n",
      "Iteration #650 -- norm=9.046e-04\n",
      "Iteration #700 -- norm=5.471e-04\n",
      "Iteration #750 -- norm=3.309e-04\n",
      "Iteration #800 -- norm=2.001e-04\n",
      "Iteration #850 -- norm=1.211e-04\n",
      "Iteration #900 -- norm=7.323e-05\n",
      "Iteration #950 -- norm=4.430e-05\n",
      "Iteration #1000 -- norm=2.680e-05\n",
      "Iteration #1050 -- norm=1.621e-05\n"
     ]
    }
   ],
   "source": [
    "# --------------\n",
    "# Point 3\n",
    "# --------------\n",
    "\n",
    "## Load data\n",
    "\n",
    "with open(\"./mvarl_hands_on/data/Q_opts.json\", \"r\") as fp:\n",
    "    Qopts = json.load(fp)\n",
    "Qstar = Qopts[\"{}_{}\".format(type(env).__name__,env.gamma)]\n",
    "\n",
    "Qstar = np.array(Qstar)\n",
    "\n",
    "Q, greedy_policy, Q_history, norm_values = value_iteration(Q0_, env, print_every=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dist_to_opt = np.array([np.max(Qstar.max(axis=1) - Q.max(axis=1)) for Q in Q_history])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "W_lBe6q6OPf2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Value iteration: Convergence of state-action Q')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEICAYAAAAKrmAuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU5fX48c/JvpNAWEIIqyCbIIgsrigq2LpVxYpLlapoXWrV1mr7/SlqW2sXt9alVK24i9oidaOKuCOLC8iigqyRNewkQDLJ+f3x3MCQTJKZkJnMZM779bqvmbs999yZmzm59z73eURVMcYYY6JNQnMHYIwxxgRiCcoYY0xUsgRljDEmKlmCMsYYE5UsQRljjIlKlqCMMcZEJUtQYSQiXUVERSQpDGUvEpGRTV1uCNvvLCK7RCSxuWIwTUtE0kXkvyKyXUReau54IkVEHhWR/9fccZjaLEHVQ0Smi8idAaafKSLrw5F4gqWq/VT1PS+eiSLyTDi3JyIrReQkv+2vVtUsVa0M53a9bYuI/FxEFopIqYgUi8hLInJYuLcdZ84F2gNtVHVssCs15h+xmsdTpIjIpSLykf80Vb1KVe8K0/b6isg0L+nvFJF3RWR4OLbVElmCqt+TwMUiIjWmXww8q6q+yIfU9Joz0QbpAeB64OdAa6AXMBX4YXMG5S8GPsNgdAG+bSnHdXMTkR7Ax8BXQDegI+64fVtEhjZnbDFDVW2oYwDSge3AcX7T8oA9wEBv/IfAF8AOYA0w0W/ZroACSd74SuAkv/kTgWf8xocDnwDbgPnAyHpiWwmcBIwByoEKYBcw35vfCngcWAd8D/wOSPTmXYr7w7kP2OLN6wG8C2wGSoBngVxv+aeBKmC3t42bA+xbR2CaV94y4Ioa+zkFeArYCSwChgT5HfQEKoGh9SzTyit7E7AK+D8gwW9fPwL+AmwFVgCnevPOB+bVKOsGYJr3PtVbbzWwAXgUSPfmjQSKgV8D64Gnvek3e5/5WuBy7zM6JITybgI2emWMr3Es/tXbv+3ePlWvG8px0wd4z1t2EXCGN/0ODjyOLguw7lBgHu5Y3wDc601f7e3nLm8YQYjHUyP2o87yvflFwL+9Y2Iz8Hdv3/fgjqddwDZv2SeB3/mtewXuGN6CO6Y7+s1T4CpgKe54egiQOmJ8GngjwPRHgJnN/fsWC0OzBxDtA/BP4DG/8SuBL/3GRwKH4c5GB3h/uGd587oSZIICCr0/pB94ZZ3sjbetI659ZVEj0XnTpgL/ADKBdsAc4Epv3qWAD7gOSML9+B3ibTMVaAt8ANwfaHt17Nv7wMNAGnC498Mwyi++Pd6+JQJ3A5/6lfUw8HAd+3kVsKqB7+gp4FUg24vrW7wfWG9fK7wfnUTgZ7jkIUAGLmH29CtrLnC+9/5+3A9Ua6/s/wJ3+33vPuAe7zNLx/2zsB7o55X9NAcmqGDKuxNI9j6rMiDPm/8QLrEUevtxlLfdoI8br9xlwG+AFOBEb/8Pres4qrH+LOBi730WMDzQseBNC/V4CvX4r7N87/OZj/sHLBN3TB7jdzx8VKOsJ/ESlPeZlACDvbL/Bnzgt6wCrwG5QGfccT6mjhjX4/dPht/0E7zvOq25f9+ifWj2AKJ9AI7B/cda/d/qx8AN9Sx/P3Cf9/6AP9wAf5T7fhBw/4k/XaOs6cAldWxnX1k1f1hw9xH2VsfsTRuH91+b90e6uoH9Pgv4ItD2au4b7r/VSiDbb/7dwJN+8b3jN68vsDvIz/+3+CWzAPMTvX3t6zftSuA9v31d5jcvw4u7gzf+DHCb974n7gc7A5fASoEefuuOAFZ470fizjjS/OY/gZdwvPFDvG0dEmR5uznwR34j7qwiwZs3MMD+B33cAMfifjQT/KY9j3fWX/M4CrD+B7gzrfwa0/cdCwdxPIV0/NdXvve5bgoUDw0nqMeBP/nNy8L9g9PVG1e8ZOeNTwFuqSMmHwGSF9DbK6djMPsWz4Pdg2qAqn6EO9jPFJHuwJHAc9XzRWSYiMwUkU0ish33H39+IzbVBRgrItuqB1xyLGhkWcnAOr+y/oE7k6q2xn8FEWknIi+IyPcisgP3wx3sfnQEtqjqTr9pq3D/FVdb7/e+DEgL8r7NZur/DPJxZwOrgtm2qpZ5b7O81+dwyRvgAmCqt0xbXKL6zO8zfMubXm2Tqu7xG+/IgZ+r//tgytusB97/KfPizMedBXxXe/dDOm46AmtUtcpvWs3Pqj6X4e7/fS0ic0XktLoWbMTxVOd+iMixXo3RXSKyKIjyi3Bn3Y25l9YRv2NJVXfhjsH6juUsAish8PdQgEtQJY2IL65YggrOU8BPcJUj/qeqG/zmPYe7bFOkqq1w9xVqVqqoVor7karWwe/9Gtx/kLl+Q6aq/jGI+LTG+BrcWUW+X1k5qtqvnnXu9qYNUNUc4KIa+1FzeX9rgdYiku03rTPu3tfBmgF0EpEhdcwvwf2H26WR2/4fkC8ih+MSVfU/HyW4s5Z+fp9hK1X1/zGq+ZmsAzr5jRfViLOh8upSgrtE2iPAvFCOm7VAkYj4/90H/Vmp6lJVHYf7R+ce4GURySTwsRHq8VTnfqjqh+pqjGb5HcP1lb8G6FzHP0D1HcfgPqN9x5K3f21o3LH8DhCoNuR5uKsC5Y0oM65YggrOU7gKCVcAk2vMy8adPezxauZcUE85XwLni0iy94N7rt+8Z4DTRWS0iCSKSJqIjBSRToGLOsAGoGv1D4+qrsP98P5VRHJEJEFEeojI8fWUkY1341hECoFfBdhG90Arquoa3M3tu724B+D+2342iNjrpapLcfeonvc+jxRvG+eLyC3qqrlPAX4vItki0gW4Efd5BlO+D3gZ+DPu3tDb3vQq3P3H+0SkHYCIFIrI6HqKmwKMF5E+IpIB3Oa3ncaU57/uE8C9ItLROz5GiEgqoR03s3H/JN3sHYMjgdOBFxqKwYv3IhFp68WzzZtcibvCUMWBx0eox1Oox3995c/B/bPwRxHJ9Mo62m+7nUQkpY5yn8N9h4d7n+8fgNmqurKO5etzB3CUiPxeRFp7x+d1wHjg9kaUF3+a+xpjrAy4G9RbgdQa08/FXRLYibt5+nf231fqyoH3oLrjfiR2Aa8DD3LgvaNhuMoGW3B/9K8DneuIZyX770G1wdXq2gp87k1rhastVIy7h/YF+2/+X0rt6/D9gM+82L7E1SYr9pt/Jq621jbglwH2rZO3/1twl6Ku8lt3Yo39rLnuo8Cj9Xz2gqtmvgh3SeV74EXc2Qi4mpXPeJ/ZGlxiSKhnX/dVXPDGj/WmPVRjuTTcD9RyXM21JcDPvXkj/T8fv3VuxV0CWourkKG4s+uQy6vxHafj7m9+732fH7D/vmgox00/b9ntwGLgR3V9TwHWfQZ3X2yX912c5TfvTm/b23D3zUI6nhq5H/WV3xlXUai6lt+D3vQUr9wtQIk37UkOrMV3Fe4Y3oI7pjvVc+wcsG6AOPt7ZezAJfNteLVIbWh4EO9DNMY0MRHpAyzE/VNjzxbFOe9s8FPgdlV9vLnjiQV2ic+YJiQiP/IuQ+bh7tP815KTAVDVYuBUXMWPYO49xj07gzKmCYnIW7hqzpW4y1VXq7snaIwJkSUoY4wxUcku8RljjIlKMdfAZUJCgqanpzd3GMYYE1PKyspUVWPqpCTmElR6ejqlpaXNHYYxxsQUEdnd3DGEKqayqTHGmPhhCcoYY0xUCluCEpEnRGSjiCysY/6FIrLAGz4RkYHhisUYY0zsCecZ1JO4/nHqsgI4XlUHAHcBk8IYizHGmBgTtkoSqvqBiHStZ/4nfqOfcmAr0MYYY+JctNTiuwx4s66ZIjIBmACQklJXI8TGGGNakmZPUCJyAi5BHVPXMqo6Ce8SYGZmpjV9YYwxcaBZa/F5/QY9BpypqpvDua09FZXc+79v+GzVlnBuxhhjTBNptgQlIp2BfwMXq+q34d5e5axPOfanZ7Pig3nh3pQxxpgmELZLfCLyPK4TtnwRKcb1IJkMoKqP4jqVawM8LCIAPlWtq1vvg5bgq+DI7xezcaM1LG2MMbEgnLX4xjUw/3Lg8nBtvybJbwtA6vatkdqkMcaYgxA/LUm0zQcgbUtJMwdijDEmGHGToCS/DaXJaWSvW9PcoRhjTHQSyUXkZUS+RmQJIiMQaY3I24gs9V7zIhVO3CSohIQE3ut+BLtzIvbZGmNMrHkAeAvV3sBAYAlwCzAD1Z7ADG88IuInQYlwzVm3Mueia5o7FGOMiT4iOcBxwOMAqJajug04E5jsLTUZOCtSIcVRgnKvVdbFvTEmDuVDEiLz/IYJNRbpDmwC/oXIF4g8hkgm0B5VV/3ZvbaLVMxxk6BEhGNXfM5l446Db75p7nCMMSaiSsCH6hC/oWYD3UnAYOARVAcBpUTwcl4gcZOgAHalZZJdsgG+DftzwcYYE2uKgWJUZ3vjL+MS1gZECgC8142RCiiuEtSq1oXujSUoY4w5kOp6YA0ih3pTRgGLgWnAJd60S4BXIxVSszcWG0k7M7Ipy8klY+nS5g7FGGOi0XXAs4ikAMuB8bgTmSmIXAasBsZGKpi4SlAiwpbCrmTYGZQxxtSm+iUQqMm5UZEOBeIsQSUIfHPUyXTqlNHcoRhjjGlAnCUo4dMfXcqoH/Zt7lCMMcY0IK4qSQhQpUBFBZSXN3c4xhhj6hFXCSpBhDarlkF6OrwasYooxhhjGiGuEpQIbMsvgMpKsJp8xhgT1eIqQSUkCHvT0qFjR3sWyhhjolx8JSgRdw+qVy9LUMYYE+XiKkG5ShLqEpRd4jPGmKgWV9XMRQQFOPts6NHD3YtKTGzusIwxxgQQVwkqQUBVYfRoNxhjjIlacXWJL0GEqipAFdasgQ0bmjskY4wxdYizBOXdgyovh27d4G9/a+6QjDHG1CGuEpRU1+JLTXX3oJYsae6QjDHG1CHOEhQoXpfv/frBokXNG5Axxpg6xVWCShBBvfxE376wbBns3dusMRljjAksbAlKRJ4QkY0isrCO+SIiD4rIMhFZICKDwxVLtX33oMAlKGvyyBhjolY4z6CeBMbUM/9UoKc3TAAeCWMsgF9LEgDHHw/PPuuaPTLGGBN1wpagVPUDYEs9i5wJPKXOp0CuiBSEKx4A/M+gCgvhggugdeuwbtIYY0zjNOc9qEJgjd94sTetFhGZICLzRGSez+dr9AYTXC2J/ebPhxkzGl2eMcaY8GnOliQkwDQNMA1VnQRMAsjMzAy4TDAOuAcFMHEifPMNLF7c2CKNMcaESXOeQRUDRX7jnYC14dyguwfll6D69nWVJKx3XWOMiTrNmaCmAT/xavMNB7ar6rpwblD8K0mAS1A+n9XkM8aYKBS2S3wi8jwwEsgXkWLgdiAZQFUfBd4AfgAsA8qA8eGKZV9MeI3FVuvXz70uXrz/vTHGmKgQtgSlquMamK/ANeHafiAJCeCfnzj0UNe8xKJFMHZsJEMxxpjoI7IS2AlUAj5UhyDSGngR6AqsBM5DdWskwom7liQOuAeVng6ffALXX998QRljTHQ5AdXDUR3ijd8CzEC1JzDDG4+IuEpQte5BAQwfDnl5zRKPMcbEgDOByd77ycBZkdpwXCWoRIHKmhnqq6/gzjuhoqJ5gjLGmAjIhyRE5vkNEwIspsD/EPnMb357qiuwudd2EQo5vnrUTUtOZE9F5YETv/wSbr/d3YPq06d5AjPGmDArqb6nVL+jUV2LSDvgbUS+jkRsdYmrM6j05ET2+GokqP793etXX0U+IGOMiSaqa73XjcB/gKHABqqboXOvGyMVTlwlqLTkRHaX10hQfftCUpJr9sgYY+KVSCYi2fvewynAQtwzq5d4S10CvBqpkOLwEl/VgRNTU6F3b0tQxph41x74DyLgcsNzqL6FyFxgCiKXAauBiD2TE2cJKqH2PSiAgQNh7tzIB2SMMdFCdTkwMMD0zcCoiMdDnCWo9ECVJAAeegiysiIfkDHGmDrFVYJKS05kd0UlqoqIX2PqrVo1X1DGGGMCiqtKEnmZKVQpbN9d45mn8nK48kqYMqV5AjPGGFNLXCWo9jmpAGzYsffAGSkpMHUqvPlmM0RljDEmkDhLUGkAbNixp/bMgQOtJp8xxkSRuEpQ7bLdGdTGnXtrzxw40LVqbk0eGWNMVIizBNXAGVR5uesC3hhjTLOLqwSVnpJIq/Rk1m8PkKAOPxy6d4fNmyMfmDHGmFriqpo5QKe8dNZsLas9o39/+O67yAdkjDEtgUgmqqWIZKG6qymKjKszKICivAzWbAmQoIwxxhyMPESuBY5pqgLjL0G1Tqd4625Ua/ZcCNx/Pxx5ZOSDMsaY2DcKuBTo7nXXcdDiMEFlsNdXxaZANfkA5s2D9esjG5QxxsS+OcBPgVVedx0HLf4SVF4GQOD7UEcc4V7nzYtgRMYY0wKoLkF1AaqvN1WR8ZegWqcDsGbL7tozBw0CEUtQxhgTBeKwFp93BhWookRWluv23RKUMcY0jkhH3P2oDOBrVN9vbFFxl6DSkhNpm50a+BIfwAUXuAd2jTHGhEbkFGAy8B6wF7gKkQxgPKqfhFpcWBOUiIwBHgASgcdU9Y815nfG7Uyut8wtqvpGOGMCKMpLD3yJD+C3vw335o0xpqX6HXAsqsv2TREZAfzT65G3FNWFwRYWtntQIpIIPAScCvQFxolI3xqL/R8wRVUHAecDD4crHn9FrTPqPoMC1x7fzp2RCMUYY1qSlAOSE4DqLOBs4Gng+VAKC2cliaHAMlVdrqrlwAvAmTWWUSDHe98KWBvGePYpystg3fY9+Cqras+sqIA2beCeeyIRijHGtCR7EGlba6rqt0AlIXYdH84EVQis8Rsv9qb5mwhcJCLFwBvAdYEKEpEJIjJPROb5fL6DDqyodTqVVcq6QG3yJSe7NvmsooQxxoTqz8BUr6LEfiL5wN5Qn48K5z0oCTCtZvMN44AnVfWv4q5TPi0i/VX1gFMbVZ0ETALIzMwM0AREaIpau5p8qzaX7Xt/gCFDXAeGqq7auTHGmIapvoJIKjALkc+A+UAKcB7u/lRIwnkGVQwU+Y13ovYlvMuAKQDqrlOmAflhjAmA7vlZAKwoqaM9wyFDXKvmq1aFOxRjjGlZVJ8D+gCv4W7dVAAXoDo51KLCeQY1F+gpIt2A73GVIC6oscxq3DXJJ0WkDy5BbQpjTIDr+j0jJZHlJaWBFxgyxL3Omwddu4Y7HGOMaVlUy4AnDraYoM6gRKRWjYFA0/ypqg+4FpgOLMHV1lskIneKyBneYjcBV4jIfFztjks1YCuuTUtE6JafyYq6EtRhh8Fdd7kuOIwxJl6IJCLyBSKveePdEJmNyFJEXkQkJaLhBJMPRORzVR1cY9oCVR0QtsjqkJmZqaWldSSWEFz73Od89f123v/VCU0QlTHGRDcRKVPVzAYWuhEYAuSgehoiU4B/o/oCIo8C81F9JALhAg2cQYnIz0TkK+BQEVngN6wAFkQmxPDonp/Jmi1llPsCVDUH2LEDpk931c6NMaalE+kE/BB4zBsX4ETgZW+JycBZkQypoXtQzwFvAncDt/hN36mqW8IWVQR0a5tJlcLqLWUc0i6r9gJvvgnnnw9z5+6/J2WMMTEqH5IQ8X9+ZhKuhnS1+4GbgWxvvA2wDXe7BgI/KlQ3kYHAsd7Yh6jODzXmes+gVHW7qq5U1XFecBW4quJZXjNFMatrG3emu2pzHZcLR4xwr7NmRSgiY4wJnxLwoTrEb9ifnEROAzai+pnfKsE8KhSYyPXAs0A7b3gGkYDPudYnqFp84rrxnQhsAKqviSkQ8XtQTSU7LRmA0vLKwAsUFUHHji5BXRfy52qMMbHkaOAMRH6Aq02dgzujykUkyTuLCvSoUF0uA4ah6s4AXKW6WcDfQgkq2OegfgEcqqr9VPUwb4jZ5ASQmuR2fW9FHQlKxJ1F2RmUMaalU70V1U6odsU9EvQuqhcCM4FzvaUuAV4NskTBNW1UrZLAZ2T1CjZBrQG2h1p4NNuXoOqqJAEuQa1caV3AG2Pi1a+BGxFZhrsn9XiQ6z0BzEZkIiITgU9DWHefYB/UXQ68JyKv4/r4AEBV7w11g9EiNSkRoO5afOD6hho1CtrWbvvQGGNaJNX3cP05gepyXMPfoeoAjAeOwZ05jcedmYUk2AS12htSvCHmpSYHcQZVUOAGY4wxoTgZ1V8Dn++bIjIZd0YWtKASlKre4cqXTK2+6RXjUhJdgqr3DArgf/+DhQvhxhsjEJUxxsQwkZ8BVwPdEfF/VjYb+DjU4oJt6miEiCzGNVmEiAwUkYh0LhguCQlCcqKw11dHJYlqb74J//d/9sCuMcY07DngdGCa91o9HIHqRaEWFmwlifuB0cBmAHUPXB0X6saiTUpiQv2X+MBVlNi9G+aH/IyZMcbEF9XtqK5EdRyqq/yGRjXsEHR3G6q6psakBk49ol9qcmLDZ1BHH+1eP/ww/AEZY4zZJ+hq5iJyFKAikiIiv8S73BfL8rNSWB+oV11/hYXQowe8/35kgjLGGAMEn6CuAq7BtcNUDBzujce03h1yWLJuZ8MLHn+8ex7KGGNMxARbi68EuDDMsURcn4Icps1fy/ayClplJNe94EMPQVpa5AIzxphY5lpCvxDojuqduLZbO6A6J5Rigm2LrxtwHdDVfx1VPaOudWJBnwLXaO+S9TsY3r1N3QtacjLGmFA8jGu39UTgTmAn8ApwZCiFBPug7lRcMxX/ZX9jsTGvb0EOAEvWNZCgAG66CcrK4JGI9dVljDGxahiqgxH5AgDVrY3pjTfYBLVHVR8MtfBo1zY7lTaZKSxZt6PhhTdvhtdfd5f7EoKu/GiMMfGoApFEqrvnEGlLI05ugv2lfUBEbvce2B1cPYS6sWgjIvQpCKGiREkJLF4c/sCMMSa2PQj8B2iHyO+Bj4A/hFpIsGdQhwEX464n+vcHdWKoG4w2fQqymTxrFb7KKpIS68nXxx/vXt9/H/r3j0xwxhgTi1SfReQzYBSusdizUA350aRgE9SPgO6qWh7qBqJdn4Icyn1VrCgppWf77LoX7NbNdWL4/vtwTczXsDfGmPARuQF4CdWHDqaYYC/xzQdyD2ZD0aqPV1FicUP3oUTgkkugd+8IRGWMMTEtB5iOyIeIXINI+8YUEuwZVHvgaxGZy4H9QcV0NXOAHm2zSE4UFq/bwZmHF9a/8F13RSYoY4yJZa4HjDsQGQD8GHgfkWJUTwqlmGAT1O2hxhcrUpISOKRddnAVJQB8PtiyBdq1C29gxhgT+zYC63ENjYf8oxnUJT5VfR9YCSR77+fi3xFVHURkjIh8IyLLROSWOpY5T0QWi8giEXkuhNibTL+OOSz6fjuq2vDCRx/tLvUZY4wJTORniLwHvAvkA1egOiDUYoLtD+oK4GXgH96kQtzDu/Wtkwg8BJwK9AXGiUjfGsv0BG4FjlbVfsAvQoq+iQzs1IrNpeV8v213wwsPG+YqSuxpoJFZY4yJX52B61Hti+rtqDbq+ZxgK0lcAxwN7ABQ1aU0fLo2FFimqsu92n8vAGfWWOYK4CFV3eqVuzHYwJvSgE6u/seC4u0NL3zKKa5/qE8+CXNUxhgTY0Q+8t5dB3yIyA5v2IlIEC0iHCjYBLXXv4q5iCRR/YRw3QoB/z6kir1p/noBvUTkYxH5VETGBBlPk+pdkE1KYgLzi7c1vPDIkZCc7LqCN8YYs5/qMd5rFqo5fkM2qjmhFhdsgnpfRH4DpIvIycBLuHb56iMBptVMaklAT2AkMA54TERqVWcXkQkiMk9E5vl8viBDDl5qUiJ9CrKZvyaIBJWVBUcdZQnKGGPqInJPUNMaEGyCugXYBHwFXAm8AfxfA+sUA0V+452AtQGWeVVVK1R1BfANLmEdQFUnqeoQVR2SlBRsxcPQDOiUy8Lvd1BVFURFidtug7/8JSxxGGNMC3BygGmnhlpIsAnqTOApVR2rqueq6j+14Spvc4GeItJNXCu25wPTaiwzFTgBQETycZf8lgcfftMZ0KkVu/b6WLG5tOGFTzzRDcYY01KIpCEyB5H5iCxC5A5vejdEZiOyFJEX622V3NXe+wo4FJEFfsMKYEGoIQWboM4AvhWRp0Xkh949qHqpqg+4FpiO6x5+iqouEpE7RaT6Ad/pwGYRWQzMBH6lqptD3Ymm0D7H9fm0tTTI1pw+/RReeCGMERljTETtBU5EdSCu1/QxiAwH7gHuQ7UnsBW4rJ4yngNOx52MnO43HIHqRaEGJEE9+wOISDLuFO3HwDHA26p6eagbPFiZmZlaWhrEWU6I5q7cwthHZ/H0ZUM5tmfbhle4+GJ46y1Yvx4SE5s8HmOMaUoiUqaqmUEunIFrgfxnwOu43nB9iIwAJqI6OnyR7hd0x0aqWgG8iasu/hm1q4zHtPRkl2R2l1cGt8Jpp7nuN2bPDmNUxhjTNPIhCZF5fsOEWguJJCLyJa4FiLeB74BtuCtiELg2dmAieYgMReS4fUOIgu3yfQzuHtIJwHvAY8B5oW4smqWneAmqIsgENXq0O3N67TVXq88YY6JYCfhQHVLvQqqVwOG42tT/AfoEWqrBjYlcDlyPqxz3JTAcmEWIXTQFewZ1Ka5CQy9VvURV39D9GbVFCPkMKjcXjj0W/ttQbXtjjIkxqttwJyPDgVz21zsIVBs7kOuBI4FVqJ4ADMLVBA9JsG3xnY/LfieLyGki0uJaSt2XoII9gwI4/XQoLnaNxxpjTCwTaUv1c6gi6cBJuApuM4FzvaUuAV4NorQ9qO7xykpF9Wvg0FBDCrYtvrHAHGAs7tLebBE5t/61YktGqktQ28oqgl/pqqtg40Zo3TpMURljTMQUADMRWYB7TOhtVF8Dfg3ciMgyoA3weBBlFXvJbirwDiKvEtyZ1wGCqsUnIvOBk6vbyhORtsA76qojRlS4avEBjL7vA9rlpPL0ZcPCUr4xxjSXkGrxNe2Gj8d1YPgWrrJd0IK9B5VQoyHXzSGsGzNG9GjD3JVbKPdVBb/Sa6/BwIGwM8j+pIwxpqWqbhT2wAZi/4t7PirkZ1yDTTJvich0EblURC7F1Yt/I9SNRbsRPdqwp6KKL4Npk69abhBzMQwAAB8SSURBVC4sWACvvx6+wIwxJhZUNwrr30BsuBuLVdVf4fqCGgAMBCap6q9D3Vi0G96tDSLwyXclwa901FHQoQO88kr4AjPGmFgiIohchMj/88aLEBkaajENJigRSRSRd1T136p6o6reoKr/aUTIUa9VRjL9O7bik+9COBNNSICzz4Y33oCysvAFZ4wxseNhYARwgTe+C9eBbUgaTFDqHtwqE5FWoRYei0b0aMOXq7cF/zwUwDnnuOT01lvhC8wYY2LHMFSvAVxVc9cpbd2NzNYh2HtQe4CvRORxEXmwegh1Y7FgRI82lFdW8dmqrcGvdNxxcOmlUFAQtriMMSaGVCCSSHWrE67mdwi1z5xgO1d63RtavCO7tiYpQfjkuxKO6Zkf3EpJSfCvf4U3MGOMiR0P4ppKaofI73EP+jbUh2AtQSUoVZ3s9enUG5cRv/HvAr4lyUpN4vCiXD5aVsLNoa68dCmoQq9e4QjNGGNig+qziHwGjML1rn4WqktCLSbYliR+gGvV9kHg78AyEQm5d8RYcXyvtiwo3k7Jrr3Br+TzwbBhcMcd4QvMGGOinavBV4Tq16g+hOrfG5OcIPh7UPcCJ6jqSFU9Hteq+X2N2WAsGHmoa2rwg29DaNswKQnGjoWpU2HXrjBFZowxUc41TzS1KYoKNkFtVNVlfuPLcf2FtEj9OuaQn5XCe9+E2PjuhRe62nyvBtOWojHGtFifInLkwRYSbIJaJCJveC1JXIJrumKuiJwtImcfbBDRJiFBOL5XOz5YuonKquB6HAbgmGOgc2d45pnwBWeMMdHvBGAWIt8hsgCRr7xGaEMSbC2+NGADcLw3vglojetrXoF/h7rhaDe4Sy6vfF7Mxp17KGiVHtxKCQnuLOr++13bfNnZ4Q3SGGOiU5PUUQi2Ft/4pthYLMnLcM+Ubd9dEXyCArjxRvjlLy05GWPil+qqpigm2C7fuwHXAV3911HVM5oiiGiUm54MhNg/FEB+kM9OGWOMqVew96CmAiuBvwF/9RtarFYZLkFtLW3E417z58PRR8OSRtWsNMaY2FVdzbwJBN3Ukao+qKozVfX96qEpAohWXdpkkpqUEFrDsdUKCmDOHHg8mI4njTGmBWmGauYPiMjtIjJCRAZXD00RQLTKSk3ilH4d+O+Ctez1hdBwLEC7dnDmmTB5MpS3yAY3jDGmPhGtZn4YcAXwR/Zf3vvLwW482p0zuJBtZRXM/LoRj3xdfjmUlNgzUcaYeNQk1czFnY01sJDI18CAUNvfE5ExwANAIvCYqv6xjuXOBV4CjlTVefWVmZmZqaWlpaGE0Wi+yiqO+uO7DCzK5Z8/GRLaypWV0L079O4N06eHJ0BjjAmSiJSpamaENtYl4PQQa/cF+xzUfCCXEFqPENfU+kPAyUAx7sHeaaq6uMZy2cDPgdnBlh0pSYkJnDWokCc+WsGmnXtpm50a/MqJiTBxomujzxhj4onqKkQGAsd6Uz5EdX6oxQR7ia898LWITBeRadVDA+sMBZap6nLvzOsF4MwAy90F/Inqjq2izHlDivBVKa98Xhz6yuPHwxVXNH1QxhgTzUSuB54F2nnDM4hcF2oxwZ5B3R5qwUAhsMZvvBgY5r+AiAwCilT1NRH5ZV0FicgEYAJASkrInTIelEPaZTG0W2temLOaCcd2JyFBQitg507X9NH48ZCWFp4gjTHmYLmq4U8BHXCdC05C9QFEWgMv4p6DXQmc5/WQW5/LcL3qlnpl3wPMwj2qFLSgzqD8q5aHUM080C/5vhteIpKAaxH9piC2P0lVh6jqkKSkYHNq07lwWGdWbi7j0+WNqHL+6adw9dUwZUrTB2aMMU3HB9yEah9gOHANIn2BW4AZqPYEZnjjDRHAv/pzJYFzQr3qTVAislNEdgQYdorIjgbKLgb8H9bqBKz1G88G+gPvichK3AcyTURCrI0QfqP7dSA3I5nn5qwOfeWTToI+feDBB11nhsYYE41U16H6ufd+J7AEdyXsTGCyt9Rk4KwgSvsXMBuRiYhMBD4FQn4wtN4EparZqpoTYMhW1ZwGyp4L9BSRbl5vvOcD++5bqep2Vc1X1a6q2tXbgTMaqsXXHNKSEzlncCemL1rP5lA6MQQQgeuug88+c2dTxhjTDPIhCZF5fsOEOhcW6QoMwlVea4/qOgDvtV29GxIRXK3s8cAWYCswHtX7Q4052EoSIVNVH3AtMB2Xiaeo6iIRuVNEYq4Nv3FDi6iobGRliYsvhlat4G8hXX41xpgmUwI+VIf4DZMCLiiSBbwC/ALVhq6U1VbdkoTq56g+iOoDqH7RmJjDlqAAVPUNVe2lqj1U9ffetNtUtVYNQK+33qg7e6p2SLtshnZtzfNz1hDMs2MHyMqCn/4UNm50z0cZY0w0EknGJadnUa3uRmkDIgXe/AKCe9wooi1JGGDcsCJWlJTy6fItoa98zz3wzjvu+ShjjIk27tLc48ASVO/1mzMNuMR7fwkQTPM4kWtJIppEsiWJmvZUVDLsDzM4rldb/jZuUOMKWbfOVTfPy2va4Iwxph4NtiQhcgzwIfAVrpo5wG9w96GmAJ2B1cBYVOv+L90lumOB2q1GhKklCcP+yhJPf7qSDTv60D4nxOeaNm1yzR/96ldw553hCdIYYxpD9SPqrgo+KoRyFJH7UD3iYEOyS3whuvSorlRWKZM/WRn6ym3bwpgx8Pe/uwd4jTGmZbJ7UM2hc5sMRvfrwLOzV1NW3oh29m69FbZuhUmBK9AYY0wLcAIuSdk9qEj7bNUWznlkFned2Y+LR3QNvYBRo2DRIvjuO8iMTOPCxpj4FoutmdsZVCMM7pzH4UW5PP7RCiqrGpHg77zT3Y+aObPpgzPGmOYicjNQnYiGorpq3wBXhlqcJahGEBEuP7YbKzeXMWPJhtALOPpoWLkSTjutyWMzxphmdL7f+1trzBsTamGWoBppTL8OFOam89hHKxpXQJHXTOGmTU0XlDHGNC+p432g8QZZgmqkpMQExh/dlTkrtvD56oZanq/D/fdDjx6WpIwxLYXW8T7QeIMsQR2EcUM70zozhfve/rZxBYwZA6Wl8LvfNW1gxhjTPAYisgORncAA7331+GGhFmYJ6iBkpiZx9cgefLi0hNmN6Suqd2+4/HJ4+GFYsqTpAzTGmEhSTUQ1B9VsVJO899XjyaEWZwnqIF00vAvtslP56/++Db0RWXBnT5mZcOONTR+cMcbEMEtQByktOZHrTjyEOSu38MHSktALaNsWbrsNPv4YVoX0iIAxxrRo9qBuE9jrq+Tkez8gPTmR139+DEmJIeb98nLXukT79uEJ0BgT9yL6oG4TsTOoJpCalMitp/bmmw07eXHemtALSElxyUkVFoTcGogxxrRIlqCayJj+HRjarTX3/u9bduypaFwhd9wBw4bBsmVNG5wxxsQgS1BNRES47bS+bCkr5+/vNjLBTJjgzqauvNKdTRljTByzBNWE+he2YuwRnXjioxUsXrsj9AI6doQ//QnefReefLLJ4zPGmFhilSSa2NbSck6+7wPaZafy6rVHkxxqhYmqKhg5EhYudEPHjmGJ0xgTX6yShCEvM4U//Kg/i9ft4KGZjbjUl5AAjz3mKk2sW9f0ARpjTIywM6gw+cULX/DagnVMu/YY+nbMCb2AqiqXrIwxpgnYGZTZ5/bT+5GbkcJNL81nr68y9AISEtzzUb/5DXz1VdMHaIwxUc4SVJjkZaZwzzmHsWTdDv7weiPb2du2DR5/HC64wDUqa4wxcSSsCUpExojINyKyTERuCTD/RhFZLCILRGSG1NVNcIwa1ac9lx/TjcmzVvHGV424n9SuHTz1lOse/qqrrOq5MSauhC1BiUgi8BBwKtAXGCcifWss9gUwRFUHAC8DfwpXPM3l5jG9GViUy69fXsDqzWWhFzB6tHuA95ln4JFHmj5AY4yJUuE8gxoKLFPV5apaDrwAnOm/gKrOVNXqX+1PgU5hjKdZpCQl8PdxgxCBnz37GWXlvtAL+e1v4Yc/hFtvdW32GWNMUxN5ApGNiCz0m9YakbcRWeq95kUypHAmqELAv2G6Ym9aXS4D3gw0Q0QmiMg8EZnn8zXiB76ZFbXO4IHzB7F43Q5+9dKC0LvlSEiAp5+G99+HvIgeH8aY+PEkMKbGtFuAGaj2BGZ44xETzgQVqP/5gL/MInIRMAT4c6D5qjpJVYeo6pCkpKQmDDFyTujdjlvG9Ob1r9bx4IxGPB+VlweHH+7ev/CCq0BhjDFNRfUDYEuNqWcCk733k4GzIhlSOBNUMVDkN94JWFtzIRE5CfgtcIaq7g1jPM1uwnHdOXtQIfe98y1vNqbSBMDy5fCTn8DYsVDRyEZpjTFxJx+SEJnnN0wIYrX2qLofK/faLqxB1hDOBDUX6Cki3UQkBTgfmOa/gIgMAv6BS04bwxhLVBAR/nD2YQzqnMuNU+bz2apG3E/q3h3++U945x2XqCob8YyVMSbulIAP1SF+w6TmjqkhYUtQquoDrgWmA0uAKaq6SETuFJEzvMX+DGQBL4nIlyIyrY7iWoy05EQmXTyE9jmpjP/XHL5e34hGZS+5BO65x13qmzDBtTphjDFNbwMiBQDea0RPJKypo2ayZksZYx+dRaUqL105gq75jWiB5Lbb4K67YOZM18CsMcbUIaimjkS6Aq+h2t8b/zOwGdU/4p5lbY3qzWEOdX84lqCaz7KNOxn76CwyU5N46aoRFLRKD60AVZg9G4YPD0+AxpgWo8EEJfI8MBLIBzYAtwNTgSlAZ2A1MBbVmhUpwsYSVDNbULyNC/45m9yMZJ69fBhd2jSyLcf334fnnoOHHoIYrelojAkfayzWhGxAp1yeu2IYpXt9jH10Ft9u2Nm4gmbPhkmTXO2+PXuaNkhjjGkGlqCiwIBOuUy5cgQA5/1jFrOXbw69kJtvhgcegKlT4eSTYdOmJo7SGGMiyxJUlOjZPpuXrzqK1pkpXPT4bF6cuzr0Qn7+c1ezb948GDYM1q9v+kCNMSZCLEFFkc5tMvjP1Uczokc+v37lK+747yJ8lSFWIf/xj939qB/8wLWGbowxMcoqSUQhX2UVf3jja574eAXDu7fmwfMH0S4nrXGFffed61Nq4kRISWnSOI0xscMqSZgmkZSYwG2n9+WvYwcyf812fvDgh3y0tKRxhU2bBnffDcceCytWNG2gxhgTRnYGFeWWbtjJ1c9+ztKNu7j0qK7cPOZQMlJCrEb+yitw2WWuxYm//hUuvxwkUFu+xpiWKhbPoCxBxYDd5ZX8afrX/OvjlXRpk8Gfzx3I0G6tQytk5Ur46U9dqxN/+xtce21YYjXGRCdLUBEQjwmq2qfLN3PzywtYs7WMi4Z14aZTepGbEcJ9paoq14X82LGQmQnLlkFREaSmhi9oY0xUsAQVAfGcoABK9/r48/RveGrWSlqlJ/PL0Ydy/pGdSUwI8ZKdzwf9+7vW0O+9F047zS77GdOCxWKCskoSMSYzNYmJZ/Tj9Z8fS6/22fz2Pws54+8f8d43G0PrqTcpyT3Ym5QEZ5wBp5ziWqMwxpgoYWdQMUxVeW3BOv745td8v203Q7rkceMpvTiqR37whVRUuPb7fv97KCmBt9+Gk04KX9DGmGYRi2dQlqBagHJfFS/OW8Pf313Khh17GdqtNVcc251RvduREOylv507YfJk+NnPIDHRtUiRnw+jRtmlP2NaAEtQEWAJqm57Kip5bvZqHvtwOWu376Frmwx+ekw3zhnciczUEKqmq8Lhh8OCBTBgANxwg2uhIj3E7kCMMVHDElQEWIJqmK+yijcXruexj1Ywf802MlMSOX1gR8YOKWJw51wkmDOiPXvg+eddBYqFCyE3F+67Dy69NOzxG2OaniWoCLAEFTxV5fPV23hhzmpeW7CO3RWVHNIui3MGd+KHhxXQuU1GMIW4Z6cef9w94HvCCfDtt/Dqq3DuudCtW/h3xBhz0CxBRYAlqMbZtdfHa/PX8uK8NXyxehsA/QtzOLV/AT84rIBuoXQ5f++9cNNN7v0RR7jnqs4+G3r2DEPkxpimYAkqAixBHbw1W8p4a+F6Xv9qHV+uccmqa5sMjuvVluN6tmVEjzYN37Navtw1ofTSSzB3rnvwd8sW1yDtkiXQqRNkZ0dgb4wxwbAEFQGWoJrW99t28/ai9bz/7SY+Xb6F3RWVJCcKgzvnMaRrHkO6tGZw5zxaZSTXXcjKlfDVV3D66W788MNh8WIYOhSOPhqOOQaOOgratInIPhljarMEFQGWoMJnr6+SeSu38sG3m5i1fDOL1u6gskoRgV7tshncJY/DClvRt2MOvTtkk5acGLigmTPhrbfgww9d54kVFXDeefDii+6e1mOPQb9+MHCgO/MyxoSdJagIsAQVOWXlPr5cvY15q7Yyb9VWvli9lZ17fAAkJgg92mbSr2MrDu2QTff8TLq3zaJLmwySE/0aKNm9210CzMiAIUNg7VooLHTzRKBXLxg0CCZMcBUwfD43pDWy/ytjTECWoCLAElTzUVWKt+5m0drtLFq7g8Vrd7Bo7Q7W79izb5nEBKFz6wy652fSLT+TTnnpFOZl0DE3jU65GeSkJSLffw9ffHHg8LvfwcUXu2Q2fDh07Qq9e0P37u79Oee418pKSEiwh4eNCZElqAiwBBV9duypYPmmUpZv2uVeS9zripJS9voO7LI+KzWJwtx0CnLTaJuVSn52KvlZqbTNSiE/O5WCLespmPoiqd8tRb7+2nWyuGMHzJgBJ57oKmb85CfQpQt07AgdOrjhF79wFTM2bIBNm9y0vDzXKoYxxhJUrcJFxgAPAInAY6r6xxrzU4GngCOAzcCPVXVlfWVagoodqsrm0nK+37qb77ftZu223RR779dt303JznJKdu3FV1X7GExKEHLSk2mVnkyB7iEtJ4vMnEx6r/2WYR+/SZvN68neuomMLSWklmxk4WszSezTh4InHqHNbbfuL6hVK2jdGj76yCW0V19198fy8tz0nBxX2/DccyE5GYqLXbNPWVluelaWa1DXmBgXVIKq8ZtNjd/sSAtbghKRROBb4GSgGJgLjFPVxX7LXA0MUNWrROR84Eeq+uP6yrUE1bJUVSnbd1dQsmsvm3btZdPOvZTsKmfzrr1s312xb9jh93777goOyGnVx7AIRdvWM2DdUtqWbiVv907yyktpU1HKfT+6Ac3K5ML3X+TsmS+QXbaTxKrKfUX86unZJKan8aMn/siwN54/IMaKtHSee2cRiYkJDH7kHgo/fpeq1DSqUlPR1FR8eW1Ycu8/SEwQCl98isylSyAtDUlLg9RUNL8Nu8ZfQYIImf97k6QNG5DkREhOQZKToE0+FSeOQkRImTeHhLIyJDkZSfGGnBzodSgikLByJQm+ChISE5HEBCQx0d3fa9fOBbtpk/s8EhPdpdCEBFf1v7qZqr1790+3S6VxpcEEVcdvNn6/2ZEWzgQ1ApioqqO98VsBVPVuv2Wme8vMEpEkYD3QVusJyhKUqapSdpX72F5WQVl5JaXlPsr2VlJW7ts3vru8klJvWvX8vb4q9vq814pKEkpLSdq1k8SyXXyXV0h5pdJ1zVKKNq4ieU8ZGXv3kFGxm+TKSu479kIAfvLZfzlq9QJSfeWk+ipIqaygLDmNn/z4LgD+8vp9jFo2h9RKNz9Rq1iZW8DIK/8JwHPP/4ajVi84YH8WtevOD8c/CMDUp27g8HVLD5g/p1NfzrvwTwC888+rOGRL8QHz3+t+BJf/+E4SRPjgbxfTYefmA+a/2fc4fnXubwCYdc/ZZO8tO2D+vweN4Y4zb0AEZv3uDEQV9UtcU4aewYOjryCtYg/T/3T+AeuqCM8cM5YnT7iQvF1befH+y/ZNB1CEJ0ZdzL+P+hEFW9bxj0eu2zdfvPmTTr2ct44YTfcNK/jT47fUWv/hM67mw8OOo/fqJdz2zJ375isCAveefSOf9zqCgd99yS9f/kut4+WeH9/Coq79GbZkFldPe7jW/Dsunsjyjj04fsF7/PStJ2rNv/Wyu1mbX8jouW9xwbvP1Zp/w8/uZ2tOa874eCo/+ujfteZfff2j7E7LYOx7L3LqnDdqzb/iV/+iKiGRi6c/ycj5Mw+YV56Uws9udMfOiZ+/Q7srx3P5sd1rlRGMIBLUCGAi3m823m82fr/ZkRbOaxeFwBq/8WJgWF3LqKpPRLYDbYAS/4VEZAIwASAlJYQeZE2LlJAg5KQlk5NWz7NZjba/qxFfZRV7fVX4qpSLqxRfZRW+qhOprFJ8VUplVRUVlUpalTK1enzCcBb7za/cW0Glz8cDicmowpZTn+X13Xtc1XufDyoqqExI4neduqCqrOl7P+t27ACfD/FVIL5K9mRm8+vDeqMoX7e6je927kSrqlwPyVWV7GzdjiuP6E6Vwty9vyS5dBeiClVVSFUV2zt24cdHFqEKsy64hqSKPUiVgrr55d37cNbhHQGYd/qFSJWi4BIVSmLfwYzp34HEinK+POWcfWes4r2mDRzCCYe2I6U0g4XH/7DW/KyB/dzD322Eb4afiFT/++nNzzn0EAZ3ziUvvT3LBx21/4zYe83r3pnDClvRTjqwuv8QRBVBQRVR6NClA4cVtqKgogPreg2o9Y0WdG4Pha3ILy1gQ6/+teYXFeWT1j6HvK0d2HBI31rzuxS2Ibd1DtmdOwac371jLqXZOWR0LmRjjz615h/SsRUVqemkFRVSsunQWvN7tc9BExJIKipk87YDW2PxJSVzaHv3wHurju3Iz2p879f5kITIPL9Jk1Cd5DcezG92RIXzDGosMFpVL/fGLwaGqup1fsss8pYp9sa/85bZHKhMsDMoY4xpjCDOoMYCo/F+s/F+s/H7zY60cPaoWwwU+Y13AtbWtYx3ia8VsCWMMRljjAksmN/siApngpoL9BSRbiKSApwPTKuxzDTgEu/9ucC79d1/MsYYEzZzgZ6IdKPu3+yICts9KO+e0rXAdFyVxSdUdZGI3AnMU9VpwOPA0yKyDHfmdH7dJRpjjAkbVR81frNRXdScIdmDusYYEwdi8UHdcF7iM8YYYxrNEpQxxpioZAnKGGNMVLIEZYwxJirFXCUJEakCdjdy9STA14ThRJuWvH8ted+gZe+f7Vt0SFfVmDopibkEdTBEZJ6qDmnuOMKlJe9fS943aNn7Z/tmGiumsqkxxpj4YQnKGGNMVIq3BDWp4UViWkvev5a8b9Cy98/2zTRKXN2DMsYYEzvi7QzKGGNMjLAEZYwxJirFTYISkTEi8o2ILBORW5o7nlCJSJGIzBSRJSKySESu96a3FpG3RWSp95rnTRcRedDb3wUiMrh596BhIpIoIl+IyGveeDcRme3t24tety2ISKo3vsyb37U54w6GiOSKyMsi8rX3HY5oKd+diNzgHZMLReR5EUmL5e9ORJ4QkY0istBvWsjflYhc4i2/VEQuCbQtU7+4SFAikgg8BJwK9AXGiUjtvpujmw+4SVX7AMOBa7x9uAWYoao9gRneOLh97ekNE4BHIh9yyK4HlviN3wPc5+3bVuAyb/plwFZVPQS4z1su2j0AvKWqvYGBuP2M+e9ORAqBnwNDVLU/rpuG84nt7+5JYEyNaSF9VyLSGrgd12X6UOD26qRmQqCqLX4ARgDT/cZvBW5t7rgOcp9eBU4GvgEKvGkFwDfe+38A4/yW37dcNA643jtnACcCrwEClABJNb9DXH81I7z3Sd5y0tz7UM++5QArasbYEr47oBBYA7T2vovXgNGx/t0BXYGFjf2ugHHAP/ymH7CcDcENcXEGxf4/omrF3rSY5F0WGQTMBtqr6joA77Wdt1is7fP9wM1AlTfeBtimqtXNyPjHv2/fvPnbveWjVXdgE/Av7xLmYyKSSQv47lT1e+AvwGpgHe67+IyW891VC/W7ipnvMJrFS4KSANNisn69iGQBrwC/UNUd9S0aYFpU7rOInAZsVNXP/CcHWFSDmBeNkoDBwCOqOggoZf8lokBiZv+8y1ZnAt2AjkAm7rJXTbH63TWkrv1pafvZLOIlQRUDRX7jnYC1zRRLo4lIMi45Pauq//YmbxCRAm9+AbDRmx5L+3w0cIaIrARewF3mux/IFZEkbxn/+Pftmze/FbAlkgGHqBgoVtXZ3vjLuITVEr67k4AVqrpJVSuAfwNH0XK+u2qhflex9B1GrXhJUHOBnl7NohTcTdxpzRxTSEREgMeBJap6r9+saUB1DaFLcPemqqf/xKtlNBzYXn2JItqo6q2q2klVu+K+m3dV9UJgJnCut1jNfave53O95aP2v1NVXQ+sEZFDvUmjgMW0gO8Od2lvuIhkeMdo9b61iO/OT6jf1XTgFBHJ884yT/GmmVA0902wSA3AD4Bvge+A3zZ3PI2I/xjcJYIFwJfe8APc9fsZwFLvtbW3vOBqLn4HfIWrZdXs+xHEfo4EXvPedwfmAMuAl4BUb3qaN77Mm9+9ueMOYr8OB+Z5399UIK+lfHfAHcDXwELgaSA1lr874Hnc/bQK3JnQZY35roCfevu5DBjf3PsVi4M1dWSMMSYqxcslPmOMMTHGEpQxxpioZAnKGGNMVLIEZYwxJipZgjLGGBOVLEEZY4yJSpagjDHGRKX/DzB6Q3TkScztAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(norm_values, label=\"Improvement at each step\")\n",
    "plt.ylabel(\"Improvement\")\n",
    "plt.twinx()\n",
    "\n",
    "plt.plot(dist_to_opt, label=\"Distance to $Q^*$\", c='r', ls='--')\n",
    "plt.tick_params(colors='r',grid_color='r')\n",
    "\n",
    "\n",
    "plt.xlabel('Iteration')\n",
    "plt.ylabel('Error relative to $Q^*$', c='r')\n",
    "plt.title(\"Value iteration: Convergence of state-action Q\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3, 3, 3, 3, 3, 3, 3,\n",
       "       3, 3, 0, 0])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "greedy_policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FbzFs3tDvcJy"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|\u001b[43mS\u001b[0m:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|\u001b[43m_\u001b[0m: : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (up)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| :\u001b[43m_\u001b[0m: : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : :\u001b[43m_\u001b[0m: : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : :\u001b[43m_\u001b[0m: : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : :\u001b[43m_\u001b[0m: : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : :\u001b[43m_\u001b[0m: : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : :\u001b[43m_\u001b[0m: : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : :\u001b[43m_\u001b[0m: : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : :\u001b[43m_\u001b[0m: : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : :\u001b[43m_\u001b[0m: : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : :\u001b[43m_\u001b[0m: |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : :\u001b[43m_\u001b[0m|\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:\u001b[42mG\u001b[0m|\n",
      "+-----------------------+\n",
      "  (down)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|\u001b[43mS\u001b[0m:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|\u001b[43m_\u001b[0m: : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (up)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| :\u001b[43m_\u001b[0m: : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : :\u001b[43m_\u001b[0m: : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : :\u001b[43m_\u001b[0m: : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : :\u001b[43m_\u001b[0m: : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : :\u001b[43m_\u001b[0m: : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : :\u001b[43m_\u001b[0m: : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : :\u001b[43m_\u001b[0m: : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : :\u001b[43m_\u001b[0m: : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : :\u001b[43m_\u001b[0m: : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : :\u001b[43m_\u001b[0m: |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : :\u001b[43m_\u001b[0m|\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:\u001b[42mG\u001b[0m|\n",
      "+-----------------------+\n",
      "  (down)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|\u001b[43mS\u001b[0m:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|\u001b[43m_\u001b[0m: : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (up)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| :\u001b[43m_\u001b[0m: : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|S:\u001b[41mx\u001b[0m:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| :\u001b[43m_\u001b[0m: : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (up)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : :\u001b[43m_\u001b[0m: : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : :\u001b[43m_\u001b[0m: : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : :\u001b[43m_\u001b[0m: : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : :\u001b[43m_\u001b[0m: : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : :\u001b[43m_\u001b[0m: : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : :\u001b[43m_\u001b[0m: : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : :\u001b[43m_\u001b[0m: : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : :\u001b[43m_\u001b[0m: : : |\n",
      "| : : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : :\u001b[43m_\u001b[0m: : |\n",
      "| : : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : :\u001b[43m_\u001b[0m: |\n",
      "| : : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : :\u001b[43m_\u001b[0m|\n",
      "| : : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : :\u001b[43m_\u001b[0m|\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (down)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:\u001b[42mG\u001b[0m|\n",
      "+-----------------------+\n",
      "  (down)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|\u001b[43mS\u001b[0m:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|\u001b[43m_\u001b[0m: : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (up)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| :\u001b[43m_\u001b[0m: : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : :\u001b[43m_\u001b[0m: : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "+-----------------------+\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : :\u001b[43m_\u001b[0m: : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n"
     ]
    }
   ],
   "source": [
    "state = env.reset()\n",
    "env.render()\n",
    "for i in range(50):\n",
    "    action = greedy_policy[state]\n",
    "    state, reward, done, _ = env.step(action)\n",
    "    env.render()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "azfXmRzZOPf4"
   },
   "source": [
    "## Question 2: Q learning\n",
    "Q learning is a model-free algorithm for estimating the optimal Q-function online.\n",
    "It is an off-policy algorithm since the samples are collected with a policy that is (potentially) not the one associated to the estimated Q-function.\n",
    "\n",
    "1. Implement Q learning with $\\epsilon$-greedy exploration.\n",
    "  - Plot the error in Q-functions over iterations\n",
    "  - Plot the cumulative sum of rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_t3WqIt-OPf5"
   },
   "outputs": [],
   "source": [
    "# ---------------------------\n",
    "# Q-Learning\n",
    "# ---------------------------\n",
    "# suggested interface\n",
    "# you can change it!\n",
    "class QLearning:\n",
    "    \"\"\"\n",
    "    Q learning with epsilon-greedy exploration\n",
    "    \"\"\"\n",
    "    def __init__(self, Ns, Na, env, epsilon=0.3):\n",
    "        self.Ns = Ns\n",
    "        self.Na = Na\n",
    "        self.env = env\n",
    "        self.Q = np.random.rand(Ns, Na)  # S x A initial \"guess\" for Q function\n",
    "        self.epsilon = epsilon\n",
    "    \n",
    "    def sample_action(self, state):\n",
    "        \"\"\"\n",
    "        Take the best short-term (greedy) action with proba 1 - epsilon\n",
    "        or take a random action.\n",
    "        \n",
    "        Args\n",
    "            state:  state for which we want to take an action\n",
    "        \"\"\"\n",
    "        u = np.random.rand()\n",
    "        if u < 1 - self.epsilon:\n",
    "            idx = np.argmax(self.Q[state])\n",
    "            return self.env.action_sets[state][idx]\n",
    "        else:\n",
    "            # Choose an action uniformly\n",
    "            return np.random.choice(self.env.action_sets[state])\n",
    "    \n",
    "    def update(self, state, action, next_state, reward, alpha):\n",
    "        \"\"\"\n",
    "        Given the state, action, next state and reward trajectory slice,\n",
    "        update the Q function.\n",
    "        \"\"\"\n",
    "        delta = reward + self.env.gamma * np.max(self.Q[next_state]) - self.Q[state, action]\n",
    "        self.Q[state, action] = self.Q[state, action] + alpha * delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CKTc5nWIOPf6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration #0 -- norm=inf\n",
      "Iteration #100 -- norm=2.459e-01\n",
      "Iteration #200 -- norm=8.710e-02\n",
      "Iteration #300 -- norm=3.148e-02\n",
      "Iteration #400 -- norm=1.144e-02\n",
      "Iteration #500 -- norm=4.169e-03\n",
      "Iteration #600 -- norm=1.522e-03\n",
      "Iteration #700 -- norm=5.563e-04\n",
      "Iteration #800 -- norm=2.034e-04\n",
      "Iteration #900 -- norm=7.441e-05\n",
      "Iteration #1000 -- norm=2.722e-05\n",
      "Iteration #1100 -- norm=9.963e-06\n",
      "Iteration #1200 -- norm=3.646e-06\n",
      "Iteration #1300 -- norm=1.334e-06\n",
      "Iteration #1400 -- norm=4.884e-07\n",
      "Iteration #1500 -- norm=1.788e-07\n",
      "Iteration #1600 -- norm=6.543e-08\n",
      "Iteration #1700 -- norm=2.395e-08\n"
     ]
    }
   ],
   "source": [
    "# --------------\n",
    "# Point 1\n",
    "# --------------\n",
    "# Number of Q learning steps\n",
    "# max_steps = int(1e5)  \n",
    "max_steps = 60000\n",
    "\n",
    "Q0 = np.zeros((env.Ns, env.Na))\n",
    "# Use the previous code to verify the correctness of q learning\n",
    "Q_opt, pi_opt, *_ = value_iteration(Q0, env, epsilon=1e-8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------+\n",
      "| : : :\u001b[43m_\u001b[0m: : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "| : : : : : : : : : : : |\n",
      "|S:x:x:x:x:x:x:x:x:x:x:G|\n",
      "+-----------------------+\n",
      "  (right)\n",
      "Opt. policy: [0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 3\n",
      " 3 3 3 3 3 3 3 3 3 0 0]\n",
      "Est. policy: [0 1 0 1 1 1 0 1 1 0 1 1 0 0 0 0 1 0 0 0 1 1 1 1 3 3 0 0 0 0 0 0 0 0 0 1 3\n",
      " 3 3 3 3 3 3 3 3 3 0 0]\n"
     ]
    }
   ],
   "source": [
    "ql = QLearning(env.Ns, env.Na, env, epsilon=.5)\n",
    "\n",
    "# main algorithmic loop\n",
    "norm_values = []\n",
    "t = 0\n",
    "state = env.reset()\n",
    "# num_visits = np.zeros((env.Ns, env.Na), dtype=int)\n",
    "cum_rewards = [0.]\n",
    "\n",
    "while t < max_steps:\n",
    "    action = ql.sample_action(state)\n",
    "    observation, reward, done, info = env.step(action)\n",
    "    # num_visits[state, action] += 1\n",
    "    \n",
    "    alpha = 1.9 / np.log(np.log(t+1)+2)\n",
    "    \n",
    "    ql.update(state, action, observation, reward, alpha)\n",
    "    norm_values.append(np.abs(ql.Q - Q_opt).mean())\n",
    "\n",
    "    t += 1\n",
    "    state = observation\n",
    "    cum_rewards.append(env.gamma * cum_rewards[-1] + reward)\n",
    "\n",
    "norm_values = np.array(norm_values)\n",
    "\n",
    "env.render()\n",
    "print(\"Opt. policy:\", pi_opt)\n",
    "greedy_policy = np.argmax(ql.Q, axis=1)\n",
    "print(\"Est. policy:\", greedy_policy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final norm: 0.07518181555593262\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAArUAAAE1CAYAAAD5+BgEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3hUZfbA8e+ZdCD0UAOE3nsHQezdtWEv+FMXuy7rrtjWrthXV1fFvnYQG4KIjaJIEwSkt9AFpJcAKe/vj3tnMplMDXMzmcz5PM88ydy5987JncydM+897/uKMQallFJKKaXimSvWASillFJKKXW0NKlVSimllFJxT5NapZRSSikV9zSpVUoppZRScU+TWqWUUkopFfc0qVVKKaWUUnFPk1qllFJKKRX3NKlVSimllFJxT5NapZRSSikV9zSprcREZIqIPKJxqIpAROqKyDciclBEcmMdj1KqJBEZJiIbj3IfQ0TEiEhytOKKZyKyUUSGxTqORKFJbRwQkbYiMkZE/hSRQyKySERuERGJdWxhOg8YVd5PKiI5IvK2iGy2j9sKEXlBRLLLOxYFwI1AY6AL0DvYiiJyvIh8LSI7RSRPRGaKyOnlEqVSDqss5yYReUREpvgsngE0NMYUxCAkleA0qa3gRKQjMAvrtToVaAs8BdwP/DeGoSEiaeGsZ4zZaYzZ73Q83kSkLTAXqANcBLQBrgKSgb+VZyyRCPeYxqkWwK/GmFXGmO2BVhKRu4HxwDRgENAL+An4SkROKZdIlXJIvJ6bwmWMOWKM+aO8nk9EXLFuFRaRlDhqZKrcjDF6q8A34AdgJiA+y08HDHBMkG2nAI943U8CHgY2Avvsx7t4PX4N8BtwAFhnr5vss7+ngdeAvcCL9rIngVftfeYCF4eII+g2QG3gcyAPWAGcYf+tQyI4bt9hfRkQP4/VtH9WBV4HdgH7gXFAfT+x+40V6wNoic/6KcBO4Pwwj3mpYxrOMQhzv6Fel2TgIWA9cAhYCpwVzv4DHPOAx9Pe3njd3g6wj3OBAn+vtf2aTov1e1JvejuaW5jnpreB93wem0LJ86gB/g/rMyIPmI31xXEI8Lt9PnkXSPfZ5kSv+zn2slb2/WHARq/Hz8b6/NkHbMZqSKnqta7xueXYz2/s80t3oBCr5db7b/kK+I/X/VuBNcBBYI6/97+fmC+w/+YjQC/7sYuBJfbx+B24wGu7hcDVXvfXAjO87t8JTLJ/bwdMBP4Edtu/N/da1/03ngosts9ZdYFUYLR9/tsAXIF1Dh0W6/+7RLnFPAC9BXlxrDeJwScZ8Xp8GfBCkO19T4IPAb9itX61Ah4FtgLV7cevA04EmgOnAVuAG332tw8YaW/f3F62ByvBawU8YJ9Q6gWJI+g2wHv2Cakf0BfrA8A7oXsAyA1x3IqAi0Ic39HASmAw0APr5D3ZzzH0GyvWpfQioLPX+qfbxygjzGNe6piGeQzC2W+o1+VR+zU+D2hpv+anhrP/SI8nVpL+CfAx0ACo4Wd7l7396AD7/xewO9bvS73pray3CM5NbxNeUrsGOAtoj3XZfyZW0twLGIiVlN3is00kSe2F9v5b2O/rpcCT9mMZwHP28zawb0l4JbX2est9YqgJHMZukMFKzFdjJYgtgFuwktucAMfGHfNS4GT7/FQDOB7YDgy193OpvZ9+9nYvAW/av2djJasHKT5XfwXcY//ey46rHdAJq4FhllcM7r/xF2CAffxTsa6gbrHj6mq/ZnloUlt+77FYB6C3IC+OlcwYoFuAx78AJgbZ3nMSBNLtN3Ann3VWAJcH2H4k8IPP/n708xwTve4nY7X0nukvjlDb2CenfEqeeE+mZEJ3M/B9WY+bvU6m/Tyney1rZ2/XMdy/D+sSufff9jb2h1E4xzzAMQ16DCLYb7C4M7BaZy/wc2zK8r8S8nhiJepvh/G69Qrw+OPA9vJ6/+lNb9G+hXNustfznEe8lvmeRw3wT6/7F/u+f4BXgHE+24Sd1PqJ62Jgjdf9R4ApPusMoWRS+xDwk9fjw7BaMcW+vwavzwt72WTg3gAxuGO+ymf5D8DNPstGA697xb7S/v1S4EusVuHjsL5Q7wIGB3jOBvZzNvX5G4/1WW8rcL3Xffc5cFis//cS5aa9Eyu2cGp08kXkMqzLzG4djDHrfdZriZXIzPQp/cnA+laLiAzAatHriJVYJWOdfLzN9xPDIvcvxpgCEfkTqyUzmEDbNLef91evded6b2iMeRGr9OFotLCfZ6bXfpeJyG6suuXFYcQK8BFWa+i9IpIK/AW40n4s5DG3+R7TUMcg3P0Gi7sVkIb1Qekr3P17i+R4BtIVqxVrUYDHewR5TKlE5P1+2Gr/XOyzrENZdy4iHbCu0vQEamG9xyPNGz7COj82McZswGr9HWOMMSJSDet897GIGK9t0rAu2wfje97sDPQXEe9OyalY9fgA04FWItIA6wrUNKCh/ftOoApWOQMiUgN4DKsxoT7F/Y+aYJVrlYrB3qaeex/gOQfuC/F3qCjSpLZiW2X/bI9V6+rLXffzJdblabfNftatZv8cgnXZxdtOEckEJgBjsC7z7sT6NjvMZ92Dfvad73PfELoTYqBtxOt+Wa22t2+L/+MG4X1hcAv2930CvCAiPbDKEQC+sX8GPeZev/se01DHINz9Bos72N8f7v69RaOTRCpW/Z1v3IhIc6zSmL9G4XmUipVwzk1gfbnzfU+l+FnP+71iAixz+dz33q+/fXr7EqsW9TJgG1YJwugQ25RgjFkiIouBoSLyFtb7+H774ar2z0sp/cU3VDLoe96sBtxB8fnXLc+OY5OIrMVKYgdjtYY3xLrytwuYY4w5ZG/zDFbp1+1YtbfJwAJKHy/vGKLx2aWOkia1FZgx5k8R+RG4TUQ+Mvb1DAB7eKM2wJXGmH2EPgEsxSqob2iMmev7oIj0wqp1utMYs9te1iRKf0ok1mAlNj2B7+1lPSPZgddxu11ExngfN/B8o16NVdzfD+uLASLSDusYLIvgubbZz3UR0Aj4zBhzxH446DEPItQxKOt+va3EqmsbgpWYeyvL/qNxPBdhfWj0wavFV0RSsD5I5wDvhLkvpSqccM5Nxpg9WLWhvbyWp2Ilwj8eZQjbsS6lu3UOtKKI1MW6anOBMeY3e9mFPqvlY9XRhvIR1jlyD7DBGDPHXr4N+APrsv4XYf0FgS0AWhhjVgVZZzrW1bSmWFfCamB1pttnP+bWD6tsYQKAiAwK9eTGmN0isg3r/DXf3q4tVmmWKiea1FZ8twA/A2Psyyp/Yn3LfA6r9+isYBu7GWP2isiLwMv2CXIe1sntLOB9rEsq+cCNIvIR1mWXcwidLEeVMWaP/fzPici1WN9+H3I/DCAiNwPnGmNOCLKrm7GO23ci8gRWPWh94HLgiDHm7yLyJvBv+/LQAayevd8aY5ZEGPbHwH1YCdxQr78l6DE3xvi9JB/qGJR1vz7PkSciz2C1MhdhnYRbAy5jzKRI92+M2ReF4zkN6zV7R0Ruw+pg0ha4F+vS57FGx75U8S/ouQn4O9Z7YYSInIfVm/+fWFcyjtY0rEaS+VhDit0bZN1d9u06+1zRGxjus846oK39BfZPAl/J+RhrNJUq9u+AdTITkceAh0Vkvx1fLazW3NnGmB8i+Nsew/qc3Ih11TEDq1V2uzHG/ZzTseqMp9jnkh0ish7r3Paa175WAxeIyGSsTq5PhRnDK8D9IrIa6wvEc1h9F1Q50XFqKzg7gXB3LpiMNTTT/4DnjDG3Rri7f2AlGk9jJQxjsGqEdhhjtmFd2r0Rq8XsZGIwYYLtNqxLPtOwOhc9bS8/bP90tyAEZIxZitXSsRGrdW+ZvS8DPGuv9nesk5x7TNRNWEOwRGoc1mWsI1idFbwFPOYh9hnqGJR1v97uB97Cqk9einUCdre6lGX/R3U87VarM7Bap9/C+mD52t5XD2PMn+HuS6mKKsxz00T791eBqVglAPOi8PR/x2qomGXv/4EgcRZilR2cjFUaMByrNM3bJ1g1pHOwkrimAfa1Civ+TngltfZj/8FK2v+JdR4aj9XauSmSP8wY8yVwCdY5ZxHWKBBnYiXebtOxznHTvJZNw2o4mOG17O/2sl+xrhL5/t2BPIZ1zvoC6zX8H5Gdk9VRcvc+VHHCHmR6HFar2iBjTKV/w4jIMVgno/p28p1wEvEY2K3EP2H1tr441vEopZSq2DSpjUP2rFO3A0vtb6eVioj0xvrGPx+rdfBFYLMxJmFmk9JjYBGRHKzOiq8bY45qTnqllFKVmya1qsIRkb5Yl3xaY3Us+Ba43RgTqF6r0tFjoJRSSkVGk1qllFJKKRX3tKOYUkoppZSKe5rUKqWUUkqpuJeQ49SmpaWZrKysWIehlKrENm3adMQYkxbrOJyi51GllNMiPY8mVFIrIkOBobVr12bjRu1IrZRyjohsj3UMTsrKytLzqFLKUZGeRxOq/MAYM9YYc2FGRkasQ1FKKaWUUlGUUEmtUkoppZSqnBIqqRWRoSIyJi8vL9ahKKWUUkqpKEqopFbLD5RSSimlKqeESmqVUkoppVTlpEmtUkoppZSKewmV1GpNrVJKKaVU5ZRQSa3W1CqllFJKVU4JldSWxa4DR7jzk4W89fPaWIeilFJKJYwd+w+zfsfBEsuMMSzcuJv1Ow7ywJeLOZRfGKPoVEWkSW0IVdOS+X7ZNl6ZupojBUWxDkcppcpERNJE5EURWSkii0XkPXt5axGZISIrRGS2iHSIdawqfhw8UsDVb83m9017It42988DXPDyDHL/PFDqMWMMPR/5jsFP/YgxxrP8/VnrOfvFnxn81I+8PSOXt2fkHk34GGPYsPNg6BUdZoyhqMiEXrEMNu3OY/u+w47su6JJqKS2LDW1qckurh6Yw9a9hxn76wYHo1NKKUeNAoqANsaYjsA/7OWvAqONMW2AJ4E3YhSfigN/7DnEs9+u4HCB1UJ6/Xvz+HH5ds78z08AFBWZEkmor4LCInYfPALAVW/NZu66XYz8dGGJdTbvzuP693713N/mlZC98VPJq6a/rd99VH/Piz+sYtCTP/LDsq1HtR9v63cc5KeVf0a0TdcHJ9Pi7om8OnV11OJwGzjqB3o/+l3U91sRJVRSW9aa2sv7NSMt2cW7v6xz7JuUUko5RUSqAlcDdxs74zDGbBGRekAP4D171XFAcxHJiUWcTgiWYAXim5hF0opmjCG/sKhMzxsPbv5gHi98v5Ixc6xGnmkrtnse23+4gBZ3T2TEmAV+t/1l9Q4uGj2Tbg99y6bdeayzSwu27DlUYr0Bo37gm8XFSebBI8UlBmt9WnUnLf7jqP6eZ75dAcA3v0cnqV28eQ+nPj+Ny9+YxYHDBWFtk3ekkL2HrHUf/3qZ33UOFxRSWIb8Y9eBIyHX+W7JVuat3xX2PrfuPcR1/5vLoo2Rt847LTnWAcSDGhkpXNm/Ga9NX8uXCzZzTvfGsQ5JKaUi0RLYAdwrIicCecADwG5gszGmAMAYY0RkPdAUyI1NqMX2HMznto/nM2X5dpJdQp1qqXx922C27Mnj03mb+HTeRvLyC8mpU5XXruxFk9pVPNu+OnU1L09dzf5DBdx/VgcmL9nKqm37+eZvg6menlLieX7ftIeculUZv2Azz3+3kj/2HuKUjvV56C+d+NvHvzFr7U5PQnFJn6aIQNXUJNbtOMjkJVvp0LA6SS5hy55D7DhwGGOgR9OafHrjQJZs3svWfYc4rm29cj12TrjijVnMXWclPzsP5AOQmZbMvsMFDGxVhxOemQLAZ/M38czQrrhc4tl25dZ9XPLazBL33fq3qOP5femWvaWed+aaHTSvWzXgFwtjDCLi97Fg9nslnXsP5Zd4bMLCLdz0wTxm3nUCDWqks3XvIfILi8iuVcV3Nx47DxzhjBd+8tzPLwyvZNH3uTfuOogxsPtgPlv25HF8u3r0e+x78gsNWZlpTLx1EOt3HqRFVlVSkoK3Tf66rjhZzf3zAM3qVCl1rK7931zr8VFnBN3X5t15HCko4oRnp1JYZPh2yVZWP3Y6s9fupE/z2iS5In8Nok2T2jBdf2xL3vo5l7s/W0TfFrVpWENHUFBKxY0UoAWwxBgzUkS6At8BZwK+mYLfTyYRGQGMcN+vUaNGmYM5cLiAf32xmOHHtqBN/cyA6z3z7XKmLLdaAguKDFv3HubdX9bx3HcrSqy37I99PDh+Ma9f1Zvr/jeXb5eUbHW774vFnt/v/GQhL1/e03N/5podXDx6Jie0q8cPy7dRt1oaAN8s3uppLcyulcHGXVbZ2oez15eKc4mdiDWqkc4pHRowb/0ulmzZS2GR4fQXpgOhEwaA3QePUD09pUQy6O1QfiFpya4yJXDRMN3rknrjWtZnYPOsqizcuIefV+0osW6be79mzj0nUqtqKkCpms65ucXJ1kdzNjDq/C4AnPb89FLPe9enizi/RzYvT/F/af5IYRFpyUlh/x2vT1/Dgo17uPm4Vp5lLbKqAtZrsO9QAXeOs0oipizfxsV9mtL3se+B4K/jep/a3HAaVpf/sQ/fl/OYJ34scX/B/Sez66CV+O4/XMCbP6/lqW+W06d5bZJdwkuX9vAcZ2/fLtnKdXbCCjDk6SncdVo7hh/bssTzh2vAqB9KLXvh+5U8//1KXr2iJ6d0bBD2vpySUOUHR6NOtTTuP6sDB48Ucs5LP7PP55uVUkpVYOuw6mnfBzDGLADWAs2AbBFJBhArW2oClMrcjDHPGmOy3bdq1aqVOZhHJixl3LyNnPzctBKtWcaUvOz/9e+lLy37JrRu3y3dxi0fzi+R0D5/cbdS6/nu85NfNwLw/bJtGANPXdClxON9cmrzxlW9w/ir4D+XdueVK3rSqXENDuUX0fLuiQHXveXD+bz04yru/mwRv23YzZTl2+j20Lf875dcv+v/secQ7e6bxKhJ/i9PR+pIQRHD353LL6t3+H3ssYlL+Xz+Js8y31bHzPRkjhQUsTDAJeiCIsNvG6x614LCohJ1sQAv/riq1DbTV24vtcytw78m8fGc0l8oAL5fui3gdv48MmEp4xds5pR/T/Msq1UllYLCIro99C2DnvzR04pbNS38tr9zXvq5xP0py4PH9e2SrZzy72k8/c3yoOt9Nm9jifvuutvZa3cyY/UORk9f47cV2zuhdZvmc4wnLNoS9LndJgco85i5xvr/mb5yO8aYmOdGCZXUHu3kC5f2bcbgNlls3XuYK9+cHeXolFLKGcaYP4HvgVMARKQZ0ByYDswHLrdXPR/INcbkOhXL/sMFJVo7Hxq/xDMs08NfLaX5XRM5eKSA/3t7Dtv3HSanThUGtqrDrSe0Drnv8Qs2l7jfoWF1z+917JasHk1rsicvn0P5hXw+f5MnqXXr0axWqfsts6pyVtdGIZ+/c+OagP863qGvzGDtnwf4ZfUODuUXMn7BZp76ZjkfzFrPOS/9zLC35gAw0U8iD3Dyc1MBeHXqmpBxhFJYZLjr00V8s3grl7w2kw07D3pGAHh16mra3Ps1o6et4faPf+OmD+YBcPBwyaGzqqYm8/6sdUGfx305+o6xC7j9499CxnXFG4E/VwuKDJt9am/djLGS7pyRE7j7s0V+17n5g3nkjJzAz6v8d+AyBm79aH7p5RAyUcsvLKLAT6nBiDELOJRfyKtTV5eobd198Aiz1+70fKGYvCR4Pe8D45eUuO+uv3X7fdMeWtw9kZyRE4LuB6y65gOHCzxJ+56DxXH9d8oqjnniB7+1u39999dSywBmrd0JwHsz1/PqtDV0fmAyv6zewdi5G/weE6clVFJ7tJMvJLmEt4dZ39jnr9/NpN/D+4ajlFIVwPXAP0VkEfAF8FdjzBZgODBcRFYAI4FrnAyi0/3flLj/7sx1nhauN+3xwJf9sY8fllmtXLk7DvL+tf04plXdiJ9rd15xMnJBz2wA5q3fTdcHJ9PuvkmlEq261VJL1ds2r1uF5CQX/7mku+cStVvvnFpM+8dxnvupydZH6oqt+0vFMid3F8c9PYVLXpvJCc9MDRjz0s17ef67lezYX7Jl0zuReXTCkog6oh3KL+SbxX/wwaz17DmYzzszchnn1fo36MkfGfTkj+T+eaBUR6UJC7ewbe8huj40ucRyg2HJ5tL1r95ydxzgiUnL+Py3zUHXA6vzUVmt2rafqXaZygezSrbm7snLZ8aqP/lqofV5fdnrs/zuY//hAiYuKv2F4tYP59P5gcl+tijW+p6vGfTkj34fe23aGh7/elmJhLnbQ99y4au/sHp76f+TsvAuCwn1f1FkoOP933jehzu8ku0nJy1n46489h8Kr4Obr1H2/84lr83kH58s5N/frSzTfo5GQiW10eByCWOG9yfZJVz/3jzOeGF6mcbnU0qp8mSMWWOMGWKM6WyM6WaM+cxevtwY098Y08YY08sYszjUvsoq0ED5y/7YV+Ly9nn/neH5vVNjq7X1z/2lx9ns2ayW53FfLoHOjYvrfrMy00LG16hm6QYP72W+daE5darStE4V3hzWiw+u6+tZvml38KuBwR7fd7iA575bQc9HAg/B9Nr0tWzdG/64o//8ZCHD3/2Vuz9bRNeHJvNrgJ7uQ56e4nd5H7ue1NvYuRtDtjD+64vFAetgvZ3RuaGnZtXbyNPahdwWrJKUa70utbsTu8IiQ9cHJ3NpgETW2/PfR5aA/bJ6Byc8M8Uzfr3vCA5u7tEV3Innqm3FNazeneWi5dinpnh+r5Iaus7YdzQJAFeUMkN/JSZO06S2DPo0r80nNwygT05tFm/ey00fzItJM7tSSsUT317e3lrf87Xf5c9f3B2AY1qXbKmd/LfBjLthAF/dMoihdiustxoZKaSnFH+oh5PU1shIKbWsZkZxB5x9Pi1Y7n0e364+A1pG3pJ8NAI1pjw5aVmJkooVW/fxpU9ZRjQmEvpywWb25EWnftKU6qto8W01D9eD462W7INHytbiGI5LXpvJ6u0HeHdmcQlGzSqh4919sPiY7TwYeritSK3fedBTWtC+of8vfG63fTSfxX5a2699Zy45Iycw4uPfMMawOcSXtGDm5u5k4cbdbNtX9pb4SGhSW0bdmtRkzPX9ObtrI9btOEj3h77luW/9d2BQSilVOikMR8ssq0Oab4LjPWpCnp8W4Oo+Cap7VINg3C3Jp3r14q6eUbqj0Pk9skl2CRf3bhpyn9Him3Bf66cTEMB/p6zmjrELyLPHdv3XF7+XWmfPwYrV0dlfuQZAekrZUpS3Z+RyzTtzI+rZHwnvS/wPf1Vc71pYGLokxPuLwKF8ZxrDOt3/DQ+NX8IfAVqP3b4IUBbirpP9dP4mmt810e+oB+G64JVfOPvFnxn0hP/yjGjTIb2O0oiT2vDH3kPMXruT579fyZs/ryU9JYn/G9ic1GQXyS7B5RJSXEJ2rSrUrJJCjYyUEuMpKqVUIihrrV4o/soaMtNLfrz5a4X15R6265Ureno63fhrLTy1UwOeubBrwP1kZaZFdVrSPXn5YbWKumf5Amj/r0msffx0Zq7ZWWq92bmll8XSqm0lk9qnh3blx+XbOL1zQ5Zu2ctr09cG2DKwH5Zto6kDn7N78vJ58Ev/FTr7Qky28NbPa3nQp9OXt/QUV9QSXXd9ekVxOApXB8KRUEmtiAwFhtauXTtq+8ypW5Uxw/uz/I99vPDDSn5cto3t+w7zRIhhV3o2q0WD6umkpyRxXLsszuwSumetUkrFs7K01IbDb0utnYy2qleNVdv2+x3H01eDGumllvkmxxC6VnFvlC7L7z2UT/X0lBKzdvla++cBpizfxtUDm5caoaC8JsDs16K23+S5rE5qX9/TsW9Aq7oBk9rW9aqxclvgzlb+6rCP1vuz1vGp11BnkQiW0AJ0b1KLX9aUHmLtaA1oWYcZfoZuq4wSKqk1xowFxmZnZ0f9rd62QSYvXdqDwiLDsj/2cqSgiMIiQ0GRNb3i1n2H2Lb3MBt2HWTcr5uYt34X7isY4+Zt5I6xC+jXog4DWtahZVY1XC7BJUKSCC4XJInQqGaGtvAqpeLW/sPOXPbOOxK4pXbcDQPYvu8wjf10AvO12k+ClOxnxibvWl1/otUq1eWByax+7HS/s2y5HWd37uqdU5s61Uom7gs37o5KHKG8fXUf2t03KeLtHj6nE/d9Xro8ooZXberhAJ0Lk1zCcxd148z//OT3ccAz4kE0OTn7cXKSM5NqhNNhrLJIqKS2PCS5hI6Ngs+088g5nQGrLmfK8u28P2s9U1dsY8ry7Z7ZcwLJykzjnG6NSHK5OLtrIxrXyiAlSUh2uUhJkpjNNKOUUqFEq6X2jM4NS9zP87pk275hdZZu2UuGnXjWyEgJq/QASo//GUioJOGEdvX4ftk2Braqw9zcXUGT3C9uGshffAbt99buvq/JD1Cr6T0u6TszcrnJa4YsgHO9RpFwUnIZp0e9ol8zv0mtt0DHLiMlyVNvfTRcElmLdmqIaWmPxtBeTUoMzxUtFaWVtqCwyO+XxGjSpDaGRITj2tXjuHbWnOC7Dhxh7rpd7MnLp6jIUGgMhUWGImPYm5fP05NXsH3fYc+lmFemlh4qJckldG5cg2SXIALinvGy5A/PYyLQvWlNeufURkRwCbjEWt60dpWg81wrpVQk3LWvLbKqkp6c5JleNpCP/9rP7/K+LUqWkLn3O+KkNmzbd4ilW/aSU7dqqe0+v2lgqVmfyiJUUvvipT1Yt/MA7RpU59YP55cafcBb1yY1qZKaxEE/rc1AwITW15Y9hzhU4H8fTktOcoVVD3p65wYs3bLP7zBSgRwOsM/eObX8Lo9UWnKS3/KVQJxqTQU4uUN9R/Yb6H+rvLW652uWPnQqGQ62HGtSW4HUqprKSUH+qa8b3IId+49QWGQYO3cDOw4coaDQkF9UREGhNT3d0i37PGMgFl8mMZ7fDcU9N4uMPTD16h2A/7EEL+3blNtOaE396qVrzZRSKhJH7ATtifO7hGyhA//jxgJc0qfkqAMpdqJRPT2Zqwe2o3W9TC7q3aTUdt2a1Az6fJf2Ld7vsW2ySnS88pYRovwgIzWJdg2s4ZSCJRQ//P3YoPuJRHpKEte+439EhLIYPkl+8SMAACAASURBVLgFr04rOXtZZloy3ZrW9Nua2KhmBmu2B09WU5NcFBRFVprh7zV4+bIeDGqTRTQuTKYmuyJKatfbM685IS3ZmVbMUFcDypPTF5M1qY0jaclJnpP8iJPbRmWf89fvYtW2/RgDRcYaLbDIGFZu3c/bM3L5YNZ6Ppi1nlev6MkpXsPcKKVUpNwTLKQmuTirayOW/RF8zvtALTopPpcwX7+yN69MW80lfZuSlpzEVQNyQsZyw5CWpSYGeOzczp7f3/m/PhHH5U+wsVJb2JfPHz+vM7d9FHoa2VDP4x69IRpGnNymVFK76MFTGPaW/6ls88MYqz0tOYmCMFue3XzLD8YM70+f5tHr7F2zSkpE4+2+9XNuWOsFqhcO5OQO9R0rH+zapCZdm9RkwYbyqbEOxve9G206Tm2C6960FkN7NeHC3k24uE9TLunTlMv6NuOBszvyxU0DueaY5gDc+P48fl1XsYaBUUrFl3w7QUlJcpFTp3R5gK9wO7g0rVOFx87tTFpy+MlmoMva4QjVUuvtQBiXfs/q0ojHz+sccr2kILWr0aybfPy80sfyYrvl21+nPCCs4bNSkiXscgq3Do1KTiDgm9DWCWNUC7c29UvX4D5wVkduOb64FjkaNbOPntuJIW2yItrGd3KRX+46Pqr1u4+e0+mo95Ge4uIv3ayRmn6998Qy7SPY/3A0JFRSKyJDRWRMXl70vs1WZl2b1OS+Mztwfo9sCosM57/8C1/8timiOceVUsrtiLulNlk4oX09zuraiPE3HxNw/XSfxGrefSfx88jjjyqG7FoZVEtL5khh2esMI+nsEs7sXS6XlCqp8CepnDoC929Rp9Sy0+zOeYFa2l64uDtPXdAl6H7P7Z7NOd0iG77Se6a2IW1LJ4oDWoU/k5u/Lw5dm9Tk715XPt8c1tvze3M/ddmhnNG5IZf1bRZx7e3CjSVniMuqllbmySf8KYzC+G4vX9aT5y/uTu6oM6gTxmQmsZBQSa0xZqwx5sKMjNBDu6hiTw/twjD7ct5tH/3Giz+U/3zOSqn450lqk5JIT0niP5d0p3N24NFiXD6tOrWrpoY1NFcwU/9xHL/966SoTBUbjnAuy4erLElWMNXSiisQz+xSPKJEqp/azgEtrUS3tZ/WToA61dIY2qt0HbPbmsdOp2ezWlw3uEXEcbpf835+ku0nzw+eSHvzbX0efUVPavu09Hr/7VllSNzc/7ORTvHrW9ebnOTyJKLndGt01J3Igv0f9mgavNbczd/segAPnNWhTDE5IaGSWlU2IsL9Z3Xg5ct6APDMtyt4f9a6EFsppVRJ+QXWh3RKcslktWuQxDbaklxCcpIr4trOsopmUjvm+v6ekrBIVPczgQTA3HtP9CQ0WZnFCZy/1lj3Mt/E8qG/dAwrBney5wrQ2nxieytp89dZ6ohXLbavjNQkfvvXSTwcxuV135bPk/30E/G+PF41LfJe+u4G2qppkXVZco+RPPK0dgxsZR3jfDupHdQ6ixcu6R720HTe3rraankOVPaRO+oMzghz8qcu2f6T3yFt60Ucl1M0qVVhERFO69yQW09oDcA9n/3O/hBTAiqllDf3JX/fpCnSOstoaFyruMV37eOnkzvqjJDbtGuQGfHzDPPptHZu98YAXNgrO+h2/oasqpGRwn1nRt4qVq96eqlW3nYNMklPSaLATpy8X5NgtczeHYY/+ms/ruyfE1EsgUoqT2hfr1Qcbu5WdX8tyAA1q6RyRb9mQZ/3h78fS7KrePsr+5dc/zz7dcmpU1wbXCU1/MR0yh1DGNCyTolShmCjGflyf6m4/tiWvH+tNZSdu6U2OUlIT0niq1sCl+oEUtX+G4J9uSoMc0QK39fm85sGMvHWQY6PaBAJTWpVREac1Iazulrf6vo//j3X/W8uBVFsiVBKVV7ullrf5CSS3ufRcuOQVpzYvh5f3jww7F7nE28dxIpHTovoea4e2JzPbhzguf/00K68cVUvHvpL4JbFE9vXZ+z1AwI+HomezWrx0qU9Sg0f5k5Qjng673m3UBbPxhZMOJMunNapAaOv6Om5H+hYu5Nrfx2JejWzEvxQ5RfBLoO3yKpWopXYO8EFeObCrix64OQStaKRjHKRU7cqH1zXr8Ssn0N7Bv/i4q19w+qllnmSWjtW33KccDSqaQ3H6dvhruTzhN6Pv3rmbk1q0qFR9eLx8CsAHdJLReyxczuxNy+fqSu28+2SrTz+9bIytR4opRJLfoDLyO6xtctTRmoSr1/VO/SKXlwuIbUMiYV3cpTkEk5oH7wF79K+gWtTIxUoMXV3ZHK/Jv5aSHs2q0V2rYxSnYz+fVE3Xvh+JZ0ahy4befnyniXu++7r0xsHsGP/EU9y7a81/N8Xd2Nu7i5PXW8gVw3IYeu+w6WGarvMHn/Y++piik9HLhEh06cO9minl42kc1aw0gJ3oh/pf95bw3p7JlCqG6Q+OJyW2huHtAq5jvt5sjLTgk7t7CRtqVURy0xP4Z3/68OC+08G4I2f1upwX0qpkI4ESaAqswjnGyAjxfn2Jvdr4J5a95SODfjPJd15/9q+Jdab9o/j+PnOkiNOnNO9MT/cMYR0P0ObPXdR14ji6NG0Fid1qM+pnRrw8DmdeMnuu+EtMz2F49rVC9miLiLceWq7UssfONuq+/W+BN8iK3Snu9b1M/22oPoKlPxWj6AG1t+69eySBHdNtPef369FbU5sH7yWtVW98KYRLggj+a6XGTgptka4t2RlpvH1bYNi1nkssc4sKqpqZKR4hn45/+Vf2Huo/C8hKqXix5GCIpJc4vhYlRVNuLNouZOXQL3Mo8ldOnBej2zWPHY67RtW56yujRjoM0SWyyURXfY+t3s239w+OOJ4klzCFf2aBW1RLIs3rurlSeBrVilOHC/oGbo1/PwejQnn+9eg1v6HFRvQsg6Pnuu/zCTTp/Oev1KOj4f3546T29DfbqH2Lp9ISXLx18EtS23zF68h08J9nxWFSGqvPaY5zeoEHofYuyb++Yu7ATBsYOQdGqNBk1p1VN67ti+ndLQupc1eo621SqnAJi/ZGpXxMuNNuB3hvrj5GO4/qwMdQrQO+ptEIBzetb0Pnl08akFZajWDaVO/GiNOasPXtw0q9Vidqqn0bV6bJ84PPdlENHhPtewuL2iRVTWshC8tOYkDhwOPZ3zP6e0BePw8/8OKiQiX9S3dgW1I2yxO61Ry5AV/HQOb163Kzce39rRQe0ecmuTyO7NaFZ9SF2+BaqDd01cH6gh575kdgraSe7+n29SPvDNlNGlSq45Kkks833hXb98f42iUUqriCTeRb163KlcPbB7yMvt/vS7RR3K5v3vTWvRoWpO/ndiG1g4mHyLCrSe09nvp3uUSPh7en4t6h55soqxuP7G153fvz6XaVVMZf/MxfHbjwLD2k+QSTz2uP9cNbkHuqDNKjXXrK3fUGbT0Knd4a1hvz5i5fZvXZvGDp9CqXujXw/v/4vTODf2u06pepqcswbdO94VLuvvdZuveQwBs33c4ZAz+VKQvqprUqqPmfkPHqjBcKVU5pCa7uKBnNqPO68yPdwyJdThRE+0RYrwndcyqlh7Rtp/eOJDbvJK+yuj2E9t4fq9fveTx6ZxdI6LxXocNyOG2E1oz/Z/HHVVM3rW+IuIZwqtp7Sphj2nr/V3nvB6NSzx26/GteOXynlzVvxmvXdmLxQ+eUqrm+fTODfluRHFpSN1q1me3eza7u+2W50gVVaBZRhNq9AMRGQoMrV27dJO9KrtOja1v47+sid7c40qpyqdz4xrsPHCk1PLnLurK3z5ewKTbBtEiq2yX1iuyFHsIM38TC5SFd8eeeet3lXhs7eOn0/yuiVF5nsqgY5ChrMKRnOTibye1Cb1iCCk+r/11g1pQJTUp6Cxsvrxran1b8we0qltiYoxAibJ3i7B7H32a12bVo6eRnOTi72MXWNunJnHz8a25uHfo+CpSjXxCtdTqNLnOSEtO4uyujdi69zBv/bw21uEopSqowiLj9wPw3O7Z5I46o1ImtAA9m9bi5uNaMb4Mg+e7ebcUel/unZNb3Jdh0u2Dwh5zt7I71Z4kIj356Iblipb6mSVbjDNSk7h2UIuIWo1NkBbRsiSW3tMAJ/v0iPtL98bcMKQltUKUVoBVi3vrCa2ZeGvpGmooHme4PCRUUquc465henD8Eh6buJTJi/8I+gZUSiWeIuM/qa3sXC7hjlPaRtyJxj3sVL8WtUsM6u99ude7da5dg6NrlaxMXr68ByseOa1MneDC+V7w052RlSO0b5jJyNPalbj8H6nCIJ+pgaYf9qdhDSvBDjbu7wez1oe9PxFhxEltAk7w0NPuBOd+XiclVPmBck6LrGo8em4n7vnsd0ZPW8NooH71NH668/iEG5NSKeVfoJZa5V/LrGqs2X6ALJ9Wvs6Na/DXwS04u2sjdvgp51BWopWaXLb/tYX3n+yZDCIQ96QGkcRz/bGlh+CKRLCR4SJ5W9WplsqWPYdISnL2vXj/WR34aPYGrjmmOVOXby8x4oZTNNtQUXNZ32bMGHk8L11q9czduvcwG3YejHFUSqmKotAYkvTyeNiKPNOklp796u7T29OpcY0K1UmnsshMTykxXa7bkxf4H7qrvPiObQvWuLDtG1YPa5IIN3dy7PR78eqBzfnmb4Opl5nOpNsH07dF8BnhokGTWhVVjWpmcEaXhvzjlLYAHP/MVGas/rPE9IRKqcRUVGSiPiZqZXblgBwALgrSWUfLvMpPqGl6nVY1LZkvbhrIvPtO8iz7S7fGfH3bIL+zuwXi/iIUSclCvNCkVjnigp7ZntnGLn1tFl0fnMwvq3V0BKViRURyRWSZiPxm3y6yl7cWkRkiskJEZouIY/NbFhoT1gxNynJsmyzWPn56ibpZX/4uSbunNI327FyJriJ8f+japGbIcXFDcf8dwb5fjojCiA+xoKcX5Yj61dN56+rePHxOJ/q3qENhkeHf362IdVhKJboLjDHd7NvH9rJXgdHGmDbAk8AbTj15YaGWH0Qq1GgG/soPJtw6iAt6ZjPxtrKPtqBKqyz14O7/mWD/W83rVg34WEWmSa1yTHpKElf0a+aZ/WbW2p1aY6tUBSIi9YAewHv2onFAcxHJceL5Co2WH0Sbv8mcsjLTeHpoV+plOt/bPJE0qpnBnae246ujGJqtIgin/CBeE3hNapXjalVN5YYhVq/PRyYs0RowpWLnfRFZJCKvi0gW0ATYbIwpADDWm3M94MgcpoVFzndOSTTuHv6RjHeqyu6GIS3p1LhGrMM4Kq3tCRiCDbGlSa1SQZzZxZqn+pvFW/lq4ZYYR6NUQhpsjOmK1TK7A3jHXu77LdPvp5mIjBCRje7b/v37Iw6gSFtqo25w6yyGDcjh4+H9Yh2KihOjzu/MY+d2LjXVrrd4/fKpSa0qFx0b1eCRczoBMOn3P2IcjVKJxxiz3v6ZD/wbGARsALJFJBlArCK7Jlittb7bP2uMyXbfqlWLfPavwiKtqY225CQXD5zdUSdeUGGrWSWVS/s2LTWLmDenx7B1iia1qtyc3tlqrZ2waAvvzMiNbTBKJRARqSoiNb0WXQLMN8ZsA+YDl9vLzwdyjTG5TsRRVGRIjtMPS6USSbx++dSkVpWb2lVTPTOK3P/lYvbk5cc4IqUSRn3gRxFZKCKLgGOBK+3HhgPDRWQFMBK4xqkgCo2plGNjKlXZ+E74ES/iPqkVkVoi8oaIbIh1LCq0qwbkcFbXRgDcMXZBjKNRKjEYY9YYY7obY7oYYzobY/7ibo01xiw3xvQ3xrQxxvQyxix2Kg6dJlep+JDvb1iNOFChkloRecEeINyISCefx/wOEG6M2WWMuQZYHpOgVcTctbW/b9oT40iUUuWpSFtqlYoLK7fui3UIZVKhklrgE+AYYJ2fx8ptgHDlrBoZKfTOqcWWPYfIL/QzHY5SqlKyWmpjHYVSKpTDBfH52VyhTi/GmGnGmI2+y8t7gHDlvHrVrfHxVsTpt0GlVGSMMRSZ+B3/UilV8VWopDaIoAOEi8grQDsReUVEWvpuHI3xFVV0nd7JGglh8aa9MY5EKVUeijzzzWtSq1RFVxSnNbXJsQ4gAgEHCDfGXB90Q2OeBZ5138/Ozo7PV6sSaVXPGuPyn+MWMjt3J08P7RrjiJRSTiq0PyS1pVapiq8gTpPaeGmpDXuAcBUfWtWrxuX9rJk4x83bqFPnKlXJeZJabalVqsIritPP5LhIaqM1QLiIDBWRMXl5eVGOUEUqySU8co41TZ8xcCg/PovSlVLhKTTaUqtUvOjRtFasQyiTCpXUishLIrIRyAa+E5FVXg8f9QDhxpixxpgLMzIyohOwOmqZaVYFzK6DR2IciVLKSe6WH62pVari6trEmniwZVbk02BXBBWqptYYcxNwU4DHlgP9yzci5bT01CQAnv9uJU9c0CXG0SilnGLsizGuCtWUopTy9t41fVi5bT9N61SJdShlklCnFy0/qHhuHNIKgI/nbmDLHn1dlKqs3C21oi21SlVYmekpcVt6AAmW1Gr5QcVTIyOFtvUzAZi/fneMo1FKOcXd7URTWqWUUxIqqVUV08jT2gEwN3dXjCNRSjnFaE2tUsphCZXUavlBxeQes/bNn9cydcX2GEejlHKCe9hLzWmVUk5JqKRWyw8qpia1q3DHyW0A+OK3TazbcSDGESmlos2gLbVKKWclVFKrKq7L+jYjySV8Om8Txz41hYe/WsKXCzaTd6Qw1qEppaIgTsdyV0rFEU1qVYVQq2oqX948kCFtswB446e13PrhfI5/ZkrczkGtlCpmtPxAKeWwhEpqtaa2YuvYqAZvX92HBfefzIuXdgdgy55DDHt7jtbaKhXndPIFpZTTKtTkC04zxowFxmZnZ2vTXwVWIyOFM7s0orDIMGLMAqat2M7+Q/kc2yYr1qEppcpIh/RSSjktoVpqVXz5S7fGrHzkNNo1yGTe+t3Mzd0Z65CUUmXkGdLLpWmtUsoZmtSqCs3lEjo2qgHAoxOXxjgapVRZeWpqYxuGUqoS06RWVXhPD+1CtbRk5q/fzdBXZrB93+FYh6SUilBxRzFNa5VSzkiopFY7isUnEWHU+Z0BmJO7iynLt8U4IqVUpNwdxTSnVUo5JaGSWp18IX6d2aUR424YAMA/PlnI75v2xDgipVQktKOYUsppCZXUqvjWJbsG/VrUBmDWWu00plQ80SG9lFJO06RWxY2UJBf3ntEBgLwjBTGORikVCZ18QSnlNE1qVVypkpoEwNOTV3BQE1ul4obx1NRqVquUcoYmtSquNK1dheZ1qwKw7I99MY5GKRUuralVSjktoZJaHf0g/iUnubiyfzMACgp1YjilIiUi94uIEZFO9v3WIjJDRFaIyGwR6eDE87rLD7SmVinllIRKanX0g8ohOcn6t80vLIpxJErFFxHpAfQD1nstfhUYbYxpAzwJvOHEc+uQXkoppyVUUqsqhxR7mk1NapUKn4ikAS8BN2JXA4hIPaAH8J692jiguYjkRPv5dUYxpZTTNKlVccfdUqvlB0pF5CHgPWPMWq9lTYDNxpgCAGP15loPNI32k3uG9HJpWquUcoYmtSrupCRZH4q5Ow7EOBKl4oOI9Ad6A//187Dvt0O/WaeIjBCRje7b/v37ox2mUkodFU1qVdypWSUVgA9nrw+xplLKdizQDlgrIrlANvAN0AnIFpFkALHG22pCyZpbAIwxzxpjst23atWqRRSATr6glHKaJrUq7gxqVTfWISgVV4wxo4wxjYwxOcaYHGAjcIox5h1gPnC5ver5QK4xJjf6MVg/NadVSjkloZJaHdKrcnC5hO5Na7J6+wH2H9YJGJQ6SsOB4SKyAhgJXOPEkxS31Dqxd6WUSrCkVof0qjx6NK0FwMWjf4lxJErFH7vF9nf79+XGmP7GmDbGmF7GmMWOPKf9U3T8A6WUQxIqqVWVx8jT2lE1NYnfN+3lUH5hrMNRSoWg5QdKKadpUqviUkqSi6sHNgdgzNwNMY5GKRWK8Uy+oFmtUsoZmtSquHVO98akJrl4fOIy8o5oa61SFVlx+YFSSjlDk1oVt1rVq8Zl/ZqSl1/Iwo27Yx2OUiqIoiLtKKaUcpYmtSqu9WtRB4CLRs9k/ILNMY5GKRWIp6VWyw+UUg7RpFbFtVM6NuCZoV0BuOXD+YwY85t2HFOqAtIhvZRSTtOkVsW983tmM/ueE2iRVZVP523ilH9P45NfN3oudyqlKoDiptqYhqGUqrw0qVWVQr3MdL6+bRDXHtOcnQeOcMfYBTzxzTJNbJWqINxvRU1plVJO0aRWVRppyUnce2YHpv/zODo0rM6rU9dw7ssz2HngSKxDUyrhGdzlB5rWKqWckVBJrU6TmxhqVknl4+H9uKRPExZs2M3Vb8/RxFapGNPJF5RSTgs7qRWRJBF5wslgnKbT5CaOzPQUHju3M8OPbcGCDbs556Wf+W2DDvulVKxoRzGllNPCTmqNMYVAHwdjUSqqRIS7TmvPw+d0YvPuPM777888OUnrbJWKheLJFzSrVUo5I9Lyg/EicqeIZIlIFffNkciUipIr+jVjwq2DaN+wOv+dsppbPppPoSa2SpUrY7SnmFLKWZEmtU8DjwNbgX3AfvunUhVa2waZjBnen+PaZjFh4RaufHMWm3ZrbbVS5cWd02pHMaWUUyJKao0xLq9bkvunU8EpFU1V05J5+fKenNe9MT+v2sGJz0xlzJwNWo6gVDnQIb2UUk4r0+gHIlJfROpFOxilnJaeksSzF3Xjrat7UzUtiX+OW8i1/5vL7oM6OoJSTnKXH7gSaswdpVR5iuj0IiLtRWQRsAxYLiILRaSdM6Ep5Zzj2tbj+xFDOKdbI35Yto0zXvhJR0dQykHaUUwp5bRIvzP/F3jcGFPLGFMLeAx4OfphKeW8GlVSeO6ibow6rzN/7j/MBS/P4K2f18Y6LKUqJXdLrZbUKqWcEmlSW8sY84H7jjHmI6BmdENSqvyICBf3acqnNw6gfvV0Hhy/hHd/yY11WEpVOsWTL2hWq5RyRqRJbaGIdHDfEZG2QFF0Q1Kq/HVsVIO3r+4NwH1fLGb0tNUxjkipykU7iimlnBZpUns3MFVEJovIN8A04K7oh6VU+WtdP5OvbxsEwGMTl/HR7PUxjkipysPgnlFM01qllDMimiYXOB7oALwAvAh0NMZMdig2pcpd+4bVmX33CTSpncG9n//OTyv/jHVISlUKnpZazWmVUg6JeJpcY8x2Y8xXxpjxxhj9xFeVTr3q6bw1rDdpyS4uf2MWv67bFeuQlIp7niG9NKlVSjlEp8lVyo9W9TIZfWUvAC4e/Qvjft0Y44iUim/GM8eJZrVKKWfE/TS5IpImIm+LyAsi8lIsY1GVy8BWdfngur5UT0/h72MXMOn3LbEOSam4VVxTG+NAlFKVVoWaJtdOTHNFxIhIJ5/HWovIDBFZISKzvUZhOB+Yaoy5FdglIv2jFY9SA1rW5f3r+pKZnszNH8zXcWyVKiMd0ksp5bSIOoqJyBNOBgN8AhwDrPPz2KvAaGNMG+BJ4A17eTMg1/59jX1fqahp16A6Y4b3p2FNaxzbD3VUBKUipkN6KaWcFnFHMQdjwRgzzRhTqnhRROoBPYD37EXjgOYikoOVALsTWfd93+1HiMhG923//v0ORK8qs/YNq/PBtf1oXDODuz5dxD8/WcCevPxYh6VU2OyhGBeKyG8iMl1EutnLA10FiypPR7FIi96UUipM8dJRrAmw2RhTAGCss+N6oClWgjtYRJ4FahtjfvHd2BjzrDEm232rVq1aOYSsKpsmtasw/pZjGNwmizFzN9L1wcl8PEdbbVXcuNAY08UY0w14BnjTXh7oKlhUecoPtK1WKeWQeOooZnzuC4Ax5rAx5v+MMSOMMTeXUywqQdWumsqbV/Xi+mNbAnDnuEU65JeKC8aY3V53awBFIa6CRff57VO4ltQqpZxSoTqKBbEByBaRZACxeho0wWqtDZuIDBWRMXl5eQ6EqBJFcpKLkae1474zrau05788g+krt8c4KqVCE5H/icgG4BHgKoJfBYuqIu0oppRyWMTVTSJST0QG2b8ni0hq9MMqyRizDZgPXG4vOh/INcbkRrifscaYCzMyMqIcoUpE1xzTnP9e1gOAK96Yze6DR2IckVLBGWOuNMY0Ae4FnnIv9lnNb9Z5tH0T3OUHOqSXUsopESW1InIeMBt4117UEfg8WsGIyEsishHIBr4TkVVeDw8HhovICmAkcE20nlepsjq9c0PuOLkNAA9/tdTTGUapiswY8w5wHLCRMK+CHW3fhCL7vaE1tUopp0TaUns30BPYBWCMWUAUh9AyxtxknzCTjTENjDGtvB5bbozpb4xpY4zpZYxZHOn+tfxAOeGvg1syqHVdxs3byA3vzdPEVlU4IlJdRBp53T8X2AFE5SpYONzvCq0+UEo5JdKktsgYs8NnWdxcc9XyA+WE1GQXz13UjfQUF5MW/8Hlb8xi02794qQqlBrA5yKySEQWADcBZ9o1tOVzFczTUquUUs6INKndJyL1sb90i8hx2K22SiWyutXSmHPPiVzQM5ufV+3gzBem8+m8jdpqqyoEY8wGY0wfY0xnY0xXY8yJxpjf7MeO+ipYOD751RqCfN/hAid2r5RSESe1dwITsYZ8mYI1DMwd0Q5KqXiUmZ7C00O78tqVvRARRoxZwEWvzuRQfmGsQ1Mq5hZs3AOgQ+AppRwT6ZBec4HjgUuxBunuaIyZ50RgTtCaWlUeTupQn0m3D6Jbk5rMzt3Jv79bGeuQlKow9OqFUsopEQ/pZYzZY4z52hgz0Wcw7wpPa2pVeamXmc4LF3enbrU0Xp22mndm5MY6JKWUUqpS01m4lXJI0zpVGHt9f5rVrsL9Xy7m2yVbYx2SUjGnky8opZyiSa1SDmpeqWQW4gAAG/VJREFUtyqjr+xFkku47n9z+e+UVaE3UkoppVTEopLUikjzaOzHaVpTq2KhTf1MXr+qFwBPTlrOh7Mjmt1ZqUqlQ8PqsQ5BKVVJRaul9h9R2o+jtKZWxcpxbesx7oYBANz16SImLtoS44iUKl/XDbLaPno0rRXjSJRSlVWZkloRWScik+3bt8CZUY5LqUqnZ7NafHnzQFKTXNz4/jwenbAk1iEpVW5cdi2tltQqpZxS1pbab40xJ9u3k4AJ0QxKqcqqS3ZNJt52DDl1qvDa9LU8NnEpRUU6xJGq/PS/XCnltLImtSUmXDDG3BCFWBynNbWqImhVL5MPrusHwOhpa7jvi98pKCyKcVRKlQ9tqVVKOaVMSa33+LQiMixq0ThMa2pVRdGoZgaLHjiZXs1q8f6s9TwwfrEOSq8qNf3/Vko5LRodxf4VhX0olXAy01N4/7q+dG5cg/dmrmeyjmOrKjF3Tqvj1CqlnBKNpFbPUEqVUVpyEs9f3A2A4e/+yra9h2IckVLOcLfT6geGUsopYSW1IrJaRMaKyF0icoqIZHk9rNeUlDoKLbKq0SW7BgB9Hvue/YcLYhyRUtFX3FIb2ziUUpVXuC21lwNTgNbAE8AmEdkgIl8AtR2KTamE8fmNAz2/d7r/G/KOFMYwGqWUUir+hJXUGmN+Mca8ZIz5P2NMNyATOA/4GhjjZIDRpKMfqIrK5RLWPHY6Z3RuCMA9ny+KcURKRZexL+qJFiAopRySXJaNjDGHgTn2LW4YY8YCY7Ozs7VkQlU4LpfwwiXd2X+4gE/nbaJfizpc2KtJrMNSKiq0/EAp5bRoTZOrlIqCJJfwzIVdycpM477Pf2fK8m2xDkmpqNKcVinlFE1qlapg6lZL48VLupPsEoa9NYeb3p/H4QKtsVVKKaWC0aRWqQqob4s6fDy8PwATFm2h7b2T2LjrYIyjUqrsPJMvaFOtUsohmtQqVUF1alyDVY+eRsdG1QE45okfyRk5gT0H82McmVKRKx6nVrNapZQzNKlVqgJLTnLx1S3HcNdp7TzLhr46I4YRKVU22lFMKeU0TWqVquBEhL8ObsGwATkArNi6nxve+zW2QSkVoeIhvZRSyhkJldTqOLUqXokID5zdkTeu6gXA17//wa0fzi+uU1RKKaUSXEIltcaYscaYCzMyMmIdilJlckL7+nz7t8EAfLlgM/d98bsmtiouFJcfaFutUsoZCZXUKlUZtK6fyZQ7hpCSJLw3cz3HPjWFQ/k65Jeq2Io7iimllDM0qVUqDuXUrcqk260W2/U7D/LC9ytjHJFSwWlHMaWU0zSpVSpOtcyqxopHTgPgi982xzgaVZGJSLqIfC4iK0TkNxGZJCI59mP17PsrReR3ETkmttEqpVTZaFKrVBxLTXZxbJssNu3O4+lvlsc6HFWxjQbaGmO6AV/Z9wFGATONMa2Bq4H3RSQ5+k/vHv1Am2qVUs7QpFapOPf00K60b1idF39cRbv7vo51OKoCMsYcMsZMNMW9CmcCLezfLwRestebA2wFot5aa7SoVinlME1qlYpzWZlpvH11bwAO5Rfxwaz1MY5IxYFbgfEiUgdwGWO2ez2WCzSN9hNqTa1Symma1CpVCdSvns74m63GtQfHL2bWmh0xjkhVVCJyN9AauMde5DsmnN+0U0RGiMhG923//v0RPa8p9TRKKRVdmtQqVUl0zq7B6Ct6UmQMN7w/jy17dJIRVZKI3AGcB5xmjDlojNlhL8/yWq0ZUKq53xjzrDEm232rVq1a2WIo01ZKKRVaQiW1OqOYquxO7tiAFy/twc4DRxjy1BQ+nqOlCMoiIiOAS4CTjDG7vR4aC9xkr9MbaAD8FO3n18kXlFJOS6ikVmcUU4nglI4N+PtJbThcUMSd4xbR+p6JsQ5JxZiIZAPPADWBH+1hvWbZD98JDBCRlcDbwBXGmIJox6D9xJRSTkuopFapRHHLCa09v+cXGt79JTdmsajYM8ZsNMaIMaalMaabfetrP7bVGHOyMaa1MaajMWaqMzFYP7WhVinlFE1qlaqkckedwZ2ntgPgvi8Ws+dgfowjUkoppZyjSa1SldgNQ1p6fp+3YVcMI1GJzujkC0oph2lSq1Ql5x7D9pO5G2MciUpoWn6glHKYJrVKVXI9mtUCYMKiLeSMnMDeQ1qGoMqfjlKrlHKaJrVKVXLV01Po1Li6536XByaTX1gUw4hUIiqeoVcppZyhSa1SCeCrWwaVuN/6nq9ZvT2yGaGUigYtP1BKOUWTWqUSRO6oM3j9yl6e+yc8M1Vbz1S5KR6nVrNapZQzNKlVKoGc2KE+0/95nOd+87t0YgZVPnScWqWU0zSpVSrBNKldhQ+u7eu5f+uH82MYjUoUOqOYUsppmtQqlYAGtKrLZX2bAvDlgs1c+86cGEekKjstdVFKOU2TWqUS1KPndiY12ToFfLd0GzkjJ3DPZ4tiHJWq7ETrD5RSDtGkVqkEtuyhU0vcf3/Wep6ZvDxG0ajKTMsPlFJOi+ukVkRqicgbIrIh1rEoFY9cLmHZwyUT2//8sIpjn/oxRhGpSks7iimlHFbuSa2IvCAiuSJiRKSTz2OtRWSGiKwQkdki0iHYvowxu4wx1wDatKRUGaWnJJE76gx+vfdEumTXAGDdjoMcLiiMcWSqMjE6p5hSymGxaKn9BDgGWOfnsVeB0caYNsCTwBsAItJARKb43P5dfiErVfnVqZbGh9f189x/ffraGEajKiutqVVKOaXck1pjzDRjzEbf5SJSD+gBvGcvGgc0F5EcY8wfxpghPrfbyzNupRJB1bRkZow8HoBnJi+nqEhb11R06OAHSimnVaSa2ibAZmNMAYCxxn9ZDzQNtpGIvAK0E5FXRKRlgHVGiMhG923/fp0eVKlAGtXMAKDIwEs/ropxNKqyMEbraZVSzqpISS1Qqugq5CnQGHO9MSbb/rk6wDrP2utkG2Oyq1X7//buPUqK8szj+PeZGQUVBZGLlx4dYkBFF5SgUWNQvB80Jq7GmMRET3Qlia57wokbNCZxEzchmlVDcNck6ho1m4sxWd0QE1FDIGjAG17wAqitAkEQlZuAzMyzf1RNT88wM9BDv11V3b/POX1OXbq6fjTt68Nbb9XbryxhRarVHV84AoD/mLGQlWs3JZxGqoHG1IpIaGkqat8AcmbWAGDRwKtGot7asjCzT5rZrzds2FCujxSpSuNGDC4s3/PkFqOFRErmrsd5iUhYqSlq3X0F8BRwXrzpLCDv7vkynuNudz9np512KtdHilStF+Jn2E65/8WEk0i10E1iIhJSEo/0usnMlgA54EEzKx60NxGYaGYLgcnAhZXOJyKRnXasLyy/vf79BJNINXDUUysiYSXx9INL4rGtDe6+p7t/sGjfS+5+lLuPcPex7r6gnOfW8AOR0tTFVciY78xINohknm4UE5HQUjP8oBI0/ECkNM93mka31rz3fjNNk6fz0WsfTjpKFdCNYiISVk0VtSJSmr47tA9BWPpubV3hcHdGfvNPALzxdm392UMxDUAQkYBU1IpIj048aCgAH5nyMLfMfiXhNJUz6ddPF5bv/uJRCSapDq5BtSISWE0VtRpTK1K6q88YWVi+ZvoLCSaprN89tbSwfHjTwASTVAfVtCISWk0VtRpTK1K63O47d1hvmjyd5as3JpSmMl5avraw/OzVJyeYpHq45skVkcBqqqgVkd7JTzmtw/qR33uoqouU2x/JF5Z37btDckHKyMymmlnezNzMDinaPtzMHjGzhWY2z8xG9vQ5veXo6QciEpaKWhHZJi9/d0KH9WFX/CGhJNvv9VXv8eLyNSx7dwMbN7d02PfaqvX8Yl40keHcK09IIl4ovwGOAV7rtP3HwE/cfQRwLXBrqAC6UUxEQmpIOkAlmdkngU8OHKjxcSKlqq8zPrTf7jzx2juFbQuWrebgvfsnmKo0Ly5fw6k3zu6w7aPDB3HnhR8urB973czC8tDd+lYqWnDuPgs6zuplZkOAMUDbGIt7gGlm1lTO2Ryj86unVkTCqqmeWo2pFdk+v7r4SM4YvXdh/aKfPZ5gmtLc9tdXtyhoAWYvequwXDyk4kefPqwiuRLWCCxz92YAj76A14F9y32i6h2sIiJpUVM9tSKyfRrq65j66cM4Z2wj5906l79n4Iax1lbnA1f2PFSiafJ0zj28kXfea58O+GNFxXuV61xvdtmfamaTgElt6/37l9ZD7+4afCAiQdVUT62IlMfoxvaCpqU1vX1w+bfW85Hvd5wN7PJTDiA/5TSmfeYw9urfPrzgl4+9wZ8WvAnArMvHVzRngt4AcmbWAGDR2IRGot7aDtz9+niK85y75/r161fyyUzjD0QkIPXUikjJip8IcOejeS74yLDkwnSjuaWV434ws8O2uVeeUBgne/qovTl91N40t7Tywa/fX3jPHV84gn336PgYs2rl7ivM7CngPOB24CwgX+7xtG1U0opISDXVU6vJF0TK59bzxwJw/3PLE07Stef/vqbD+u++fHSXN3411Nfx+FUn8quLj+Tl705g3IjBlYpYUWZ2k5ktAXLAg2a2ON41EZhoZguBycCFIc6vGcVEJLSa6ql197uBu3O5XHqvl4pkxAnx9LlzX3074SRdO2PanMLyjK+MY/jQXbt976B+fRjUr08lYiXG3S8BLuli+0tA8HmAXbeKiUhgNdVTKyJhvLkmvTeMnXjQkB4LWqkMd3XUikhYKmpFZLt9+LsPJR2hg/feby4s//hzYxNMIsV0o5iIhKSiVkR67clvnFRYXpGi3tq1G9uL2vo6FVJpoMkXRCQ0FbUi0msDd9mRMw/bB4Cb//JKwmnatfUcnz5qr4STSBuNqRWR0GqqqNXTD0TK7+qPHQzAbXNe5Z3172/l3ZV1/IFDko4gMY2pFZHQaqqo1TS5IuXXf+f2Z9Ye9p0ZbG5pTTBNRzU0K1gmaEytiIRUU0WtiIRx3dmjCsvDiyYySIK703eHOo4dMZgd6tXEpYUeUysioanFF5Ht1jauNg1efWs9Gze3MmzQLklHkWK6UUxEAlNRKyLbraG+jjmTj086BgCP5aPJIA5vGphwEimmG8VEJDQVtSJSFvsM2IkD94wmOVj93mY2bm5JJMd1f3oJgLFNuydyfumaO2gAgoiEpKJWRMomt3t0E+bobz/Agd/4I3c8mueMaX9l1bpNFTm/u/PWuugJDEN361uRc8q20/ADEQmppopaPdJLJKxLxn+ww/o3713AM0tWc8qNs4Ofe3NLK8Ou+ANAocdY0kM3iolIaDVV1OqRXiJhjc4N6HL7PgP60jR5Onc8mi/r+V5btZ6DvvFH7nt6GafeOKuw/YhhGk+bNu4aUysiYdVUUSsiYdXVGf92xsFbbF+wbA0Q9dx2NmfxW9wyO5qN7IYZC5nx/JvbfL5jr5vJhs0tXPaLpxjUr09he9MeevJB2jgafiAiYamoFZGyOv/oJvJTTuuwrbm1+166z94yl2umv8Dy1Rv54UOL+Kc7HueZJe+WfN5Nze2TPnzhmGElHy9hRTOKqaoVkXBU1IpIEN/7x38o6f1Hfu+hwvIZ0+aUfL75b0SFcOeCWtJDPbUiEpKKWhEJ4tzDG3vcv+jNtTy3dHWvP//e+Ut7faxUnkbUikhoDUkHEJHqZFvpljvphlk97u+Ju/Mvv5y/xfazP5Tr9WdKYO4afCAiQamnVkSC+eG5h5blc5pbWvnsLX/jvqeXsXFzS+HRXZ19++Nb3qQm6RDdKKayVkTCUU+tiATz8UP34S8LV/LbJ9uHCrS2OptbW7s9Zp8BO7F8zUYWvbmWles2cfT+g3h55XrmLF7FnMWr6Nenvdn66efHctLIoUH/DCIikg0qakUkqJMOGtqhqP3p7FfYuU/XTc89Xzqas/7rkei4eHjC+AMG8+eXVhbes25Tc/tnq6DNDHfdKCYiYWn4gYgEdfLBe/Kl4/YvrM9atHKLaXP/56IPk59yGh/ab/ctjq+v67oS2meAJlHJEtetYiISWE0VtZomV6Ty6uuMr516YGF9zuJVdJ5cqqG+vSmaftkxHfY9+MKKLT6zzuDXXzyqvEElKPXUikhoNVXUappckeRcOaG9sP3hQ4sAuOkzY5g47gMc3tTeQ3vw3v27PH7GV8YVlm/41KHqqc0YTb4gIqHVVFErIsm5eNz+W2w7YM9+XDHhoK3eFX/O2BzDh+4aKppUiHpqRSQkFbUiUjGXn3JAh/XuitmvnjwCgMe+fiL5Kadx7dmjARi6Wx8A+jTUB0wpIWhErYiEpqcfiEjF7LHLjh3W39vU0uX7Lj1+OJceP3yL7T/69Bjue3opxx84JEg+Ccc1+YKIBKaiVkQqpvPTDTY1d13UdueIYQM5YtjAckYSwMyGAz8DBgHvAhe4+/MBzlPujxQRKdDwAxGpmOFDd2V0rv1GMF2STo0fAz9x9xHAtcCtIU6iklZEQlJRKyIVde+l7Y/s6vxoL6k8MxsCjAHuijfdAwwzs6Zynkd/1yISmopaEam4HePn0vZpUBOUAo3AMndvBnB3B14H9i3nSRxXV62IBKUxtSJScb+/7Bjuf3Y5o3JdP5NWKq5zP+oW5aeZTQImta3371/a391JI4fS0tqrbCIi28S8Bq8J5XI5X7JkSdIxRKSKmdlSd88lnWNr4uEHi4A93L3Zoru5/g4c6e757o5TOyoioZXajuran4hIDXP3FcBTwHnxprOAfE8FrYhIGmn4gYiITARuN7MrgTXA+QnnEREpmYpaEZEa5+4vAUclnUNEZHtkuqg1s/FEl8z6Au+4+6UJRxIRERGRBFR8TK2ZTTWzvJm5mR3Sad9wM3vEzBaa2TwzG9nTZ7n7n939Qnf/LLCfme0aNLyIiIiIpFISN4r9BjgGeK2LfV3OamNme5rZzE6vG9sOMrPTgefdfW0F8ouIiIhIylR8+IG7z4It5wAvmtXm5HjTPcA0M2uK78I9rqvPM7OLgJy7fy1QZBERERFJuTQ90qvkWW3M7CzgW8CeZnazmQ3u5n2TzGxJ22vdunUB4ouIiIhIUtJ2o9hWZ7Xp8Gb3e4h6dHv+UPfrgevb1nO5XO3NOCEiIiJSxdJU1L4B5MysoWhWm0ai3tqyWrp06ftmtrLEw/oBWenizUrWrOQEZQ0hKzmhd1m7vHJULdSOpkZWcoKyhpKVrMHb0dQUte6+wszaZrW5nYCz2rh7n1KPMbMlWZjyErKTNSs5QVlDyEpOyFbWSlE7mg5ZyQnKGkpWslYiZxKP9LrJzJYAOeBBM1tctHsiMNHMFgKTgQsrnU9EREREsieJpx9cAlzSzT7NaiMiIiIiJUvT0w/S7vqtvyU1spI1KzlBWUPISk7IVtY0y9L3mJWsWckJyhpKVrIGz2nRk7NERERERLJLPbUiIiIiknkqakVEREQk81TUboWZDTezR8xsoZnNM7ORgc831czyZuZmdsi25Aixbxuz9jWz/42Pn29mfzSzpnjfkHh9kZk9Z2bHFB1X9n3bkPUBM3smzjnbzA5N6/da9DnfKv4dpDFr/Ft9Mf5e55vZp9KY1cz6mNm0+LezwMzuSmPOalXp78TUjqodbf8ctaO11I66u149vICHgQvi5bOBRwOfbxzR487ywCHbkiPEvm3M2heYQPvY7EuBB+Ll24Cr4+XDgdeAhlD7tiHrgKLlTwBPpvV7jY8bA9wf/xkPSWvWzr/TtP5egRuAqUW/1b3SmLNaX5X+TlA7qnbU1Y6WOysZaEeDNSrV8AKGAO8WNQYGLAeaKnDuwo+8pxwh9m1H5rHA4nh5HTC4aN884LhQ+0rMeT7weFq/V6AP8CgwrO13kOKseTo1xmnLCuwSH9svzTmr9ZXkd4La0ZL2lZhT7aja0dTlTM2MYinVCCxz92YAd3czex3Yl+hHmIYc6wPs6+2f7TLg/8xsD6DO3Yun0MwD+4bYt63hzOwOYHy8eirp/V6/Ddzl7q+aWdu2tGYF+LmZ1QFzgStSmHV/YBVwlZmdCGwAriZqNNOUs1qpHS2N2lG1o2pHe5lTY2q3zjutW5fvCq+nHCH2lcTMrgSGA18PmGm78rr75929EbgKuC6NOc3sKKJLgv/Zxe5UZY2Nc/fRRJf5VgE/C5int1l3AD4APO/uY4ku7/6SaPKZNOWsZmn5TlL99612VO0oake3L6cHvvyT5RdRF/hq0nHZrMscIfb1IutXiS5DFY+3Wk/3l7/Kvq8XmTcAQ9P2vRJND70s/vvPA83AUqJLfanK2kX2vYC1IfJsT1ZgENAC1Hf67ZyTppzV+kryO0HtaEn7epFZ7aja0VS1o4k3eGl/ATPpOFj5bxU6b56ONzh0myPEvhJyTgKeAHbvtP12Ot6M8HrRD7Ts+7aScTdg76L1M4El8X8oqfxeu/odpC0r0Rir4v8BTwJmpTTrA8CEeHk/YCXR/zxSlbNaX0l9J6gdLWnfVjKqHVU7mvp2NHijkvUXcADRYPOFRP+KPjjw+W4iaiiaif5lsnhrOULs28asOaJLBC8D8+PX3Hjf0Pg/gEXAAuDYouPKvm8rORuJ/kX5LPA08CBwaFq/107Z87Q3xqnKSnQp6ingmfi7vZf4X9IpzTozzjkfODONOav1VenvBLWjakc7Zs+jdrRcWWeS4nZU0+SKiIiISObpRjERERERyTwVtSIiIiKSeSpqRURERCTzVNSKiIiISOapqBURERGRzFNRK1XDzPJmdoiZXWBmIwJ8/gAz+9dO224xs4+W+1wiIklRWypZpaJWqtEFQMkNsZnVxXNvd2cA0KEhdveL3H12qecSEcmAC1BbKhmiolaqzXHAWGCqmc03swkAZvZVM5tnZk+a2R/MrDHefrWZ3WlmvyV6mPReZnadmT0WH/8XMxsef/bNwIB4++Px8TPN7PR4eaiZ/c7MnjWz58zs4rZQcc/Ht8zsETN71cyuqtg3IiJSuuNQWyoZ05B0AJEym0k0+8gP3P33AGb2GaLehqPcvcXMPgdMAz4eHzMeGOPuK+L3f9/dL4+XzwVuAE4Hvgg87u6HdnPuqcCL7n6mmQ0BnjCz+e4+L94/wN2PNrPBwGIz+293X1reP76ISFnMRG2pZIyKWqkFnyDqcXjCzADqgZai/b9va4RjJ5vZPwO7El3N2G0bz3MiMBrA3VfEPRYnEE0tCfDzeN9KM3sFGAaoIRaRrFBbKqmmolZqgQHXuPtt3exfV3ij2b5EvQRHuPsrZjYKeLiEc3Wed7p4fWPRcgv6709EskVtqaSaxtRKNVoD9C9avw/4spkNBDCzHczssG6O7Q+8Dyy3qCvi0k6fu7OZddeAPghcHJ9jMHAmpTXiIiJporZUMkVFrVSjnwDfbLu5wd3vBO4CZprZ00Q3MYzv6kB3fxa4G1hANKbs9aJ9bxNd9nq27eaGTi4DRpnZM8CfgX8vGgMmIpI1akslU8y9cw+/iIiIiEi2qKdWRERERDJPRa2IiIiIZJ6KWhERERHJPBW1IiIiIpJ5KmpFREREJPNU1IqIiIhI5qmoFREREZHMU1ErIiIiIpmnolZEREREMu//AcPHGJtMxj1hAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 800x320 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(dpi=80, figsize=(10, 4))\n",
    "plt.subplot(121)\n",
    "plt.plot(norm_values)\n",
    "plt.xlabel('Iteration')\n",
    "plt.ylabel('$L_1$ error')\n",
    "plt.title(\"Q-learning: Convergence of $Q$\")\n",
    "\n",
    "plt.yscale('log')\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.plot(cum_rewards)\n",
    "plt.xlabel('Iteration')\n",
    "plt.title(\"Cumulative reward\")\n",
    "\n",
    "print(\"Final norm:\", norm_values[-1])\n",
    "\n",
    "# how confident are you in the performance of the algorithm? maybe a single run is not enough"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "MVARL19_part1.ipynb",
   "private_outputs": true,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python [conda env:torch]",
   "language": "python",
   "name": "conda-env-torch-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}