diff --git a/.style.yapf b/.style.yapf index 612f695..8485660 100644 --- a/.style.yapf +++ b/.style.yapf @@ -17,4 +17,4 @@ split_before_bitwise_operator=True split_before_closing_bracket=False split_before_dot=True split_complex_comprehension=True -blank_lines_around_top_level_definition=1 \ No newline at end of file +blank_lines_around_top_level_definition=1 \ No newline at end of file diff --git a/nbs/01_tensor.ipynb b/nbs/01_tensor.ipynb index 91d8a86..8a606aa 100644 --- a/nbs/01_tensor.ipynb +++ b/nbs/01_tensor.ipynb @@ -74,9 +74,9 @@ " def __init__(self, data, name=None, op=None, eps=1e-8, requires_grad=False):\n", " global _num_tensors\n", " _num_tensors += 1\n", - " self.data = np.asarray(data)\n", + " self.data = np.asarray(data, dtype=np.float64) # , dtype=np.float32\n", "\n", - " self.grad = (np.zeros_like(self.data, dtype=np.float32) if requires_grad else None)\n", + " self.grad = (np.zeros_like(self.data, dtype=np.float64) if requires_grad else None)\n", " self.eps = eps\n", " self.op = op or ops.Load(name=name)\n", " self.name = name or self.op.name\n", @@ -86,8 +86,8 @@ " value_str = f\"v={lovely(self.data)}\"\n", " grad_str = f\"∇={lovely(self.grad)}\" if self.grad is not None else \"\"\n", " parents = (f\" parents=[\" + \",\".join([p.name for p in self.op.parents]) + \"]\" if self.op.parents else \"\")\n", - " # name=\"{self.name}\n", - " return f'Tensor{list(self.data.shape)}(\" op={type(self.op).__name__}{parents}):\\n {value_str}\\n {grad_str}'\n", + " \n", + " return f'Tensor{list(self.data.shape)}(name=\"{self.name}\" op={type(self.op).__name__}{parents}):\\n {value_str}\\n {grad_str}'\n", "\n", " def accum_grad(self, grad):\n", " if not self.requires_grad:\n", @@ -128,8 +128,9 @@ " def mmul(self, other, name=None):\n", " return ops.Matmul(self, other, name=name).out\n", "\n", - " def sum(self, name=None, axis=None, keepdims=False):\n", - " return ops.Sum(self, name=name, axis=axis, keepdims=keepdims).out\n", + " # XXX move name to the end of arg list\n", + " def sum(self, name=None, axis=None, keepdims=False, ):\n", + " return ops.Sum(self, axis=axis, keepdims=keepdims, name=name,).out\n", "\n", " def transpose(\n", " self,\n", diff --git a/nbs/01_tensor_helpers.ipynb b/nbs/01_tensor_helpers.ipynb index 3f5a6df..174c784 100644 --- a/nbs/01_tensor_helpers.ipynb +++ b/nbs/01_tensor_helpers.ipynb @@ -54,7 +54,7 @@ "def std(input: Tensor, name=None, axis=None, keepdims=False, correction=1) -> Tensor:\n", " if isinstance(axis, int): axis = (axis, )\n", " v1 = input - input.mean(axis=axis, keepdims=True)\n", - " var = (v1)**2\n", + " var = v1 ** 2\n", "\n", " if axis is None: numel = np.prod(input.data.shape)\n", " else: numel = np.prod([input.data.shape[i] for i in axis])\n", diff --git a/nbs/02_func.ipynb b/nbs/02_func.ipynb index fee7369..d87b232 100644 --- a/nbs/02_func.ipynb +++ b/nbs/02_func.ipynb @@ -160,6 +160,13 @@ "def layer_norm(x: Tensor, w: Tensor, b: Tensor, eps=1e-5) -> Tensor:\n", " mu = x.mean(axis=-1, keepdims=True)\n", " sigma = x.std(axis=-1, keepdims=True, correction=0)\n", + " if sigma.data.any() == 0:\n", + " print(\"x\", x)\n", + " print(\"w\", w)\n", + " print(\"b\", b)\n", + " print(\"mu\", mu)\n", + " print(\"sigma\", sigma)\n", + " raise ValueError(\"sigma is zero\")\n", "\n", " return ((x-mu) / (sigma+eps)) * w + b # tensor[10, 768] n=7680 (30Kb) x∈[-0.788, 0.579] μ=-0.005 σ=0.106" ] @@ -209,9 +216,9 @@ " sm = softmax(logits)\n", " loss = -target * sm.log()\n", " if reduction == \"mean\":\n", - " return loss.mean()\n", + " return loss.mean(axis=-1, keepdims=True)\n", " if reduction == \"sum\":\n", - " return loss.sum()\n", + " return loss.sum(axis=-1, keepdims=True)\n", " assert 0, \"Invalid reduction\"" ] } diff --git a/nbs/02_ops.common.ipynb b/nbs/02_ops.common.ipynb new file mode 100644 index 0000000..8140355 --- /dev/null +++ b/nbs/02_ops.common.ipynb @@ -0,0 +1,655 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | hide\n", + "# | default_exp ops.common\n", + "import nbdev\n", + "from nbdev.showdoc import *\n", + "\n", + "nbdev.nbdev_export()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Operations: Common\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | exporti\n", + "import numpy as np\n", + "_grad = True" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | exporti\n", + "def calculate_target_shape(s1, s2):\n", + " \"\"\"Calculate the target shape for broadcasting two tensors\"\"\"\n", + "\n", + " # expand shaped to be the same length. Note (1,) * is empty\n", + " s2 = (1, ) * (len(s1) - len(s2)) + s2\n", + " s1 = (1, ) * (len(s2) - len(s1)) + s1\n", + "\n", + " out_shape = ()\n", + " for dims in list(zip(reversed(s1), reversed(s2))):\n", + " if dims[0] != 1 and dims[1] != 1 and dims[0] != dims[1]:\n", + " raise ValueError(f\"Cannot broadcast {s1} and {s2}\")\n", + " out_shape = (max(dims), ) + out_shape\n", + "\n", + " return out_shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | exporti\n", + "def maybe_broadcast_elementwise(a, b):\n", + " \"\"\"Broadcast two tensors if they have different shapes\"\"\"\n", + " if a.data.shape != b.data.shape:\n", + " target_shape = calculate_target_shape(a.data.shape, b.data.shape)\n", + " # print(\n", + " # f\"Elementwise broadcasted {a.data.shape} and {b.data.shape} to {target_shape}\"\n", + " # )\n", + " a = a.broadcast(target_shape) if a.data.shape != target_shape else a\n", + " b = b.broadcast(target_shape) if b.data.shape != target_shape else b\n", + "\n", + " return a, b\n", + "\n", + "def maybe_broadcast_matmul(a, b):\n", + " \"\"\"Broadcast two tensors if they have different shapes, except for the last two dimensions\"\"\"\n", + "\n", + " a_short_shape = a.data.shape[:-2]\n", + " b_short_shape = b.data.shape[:-2]\n", + "\n", + " if a_short_shape != b_short_shape:\n", + " target_shape = calculate_target_shape(a_short_shape, b_short_shape)\n", + " # print(\n", + " # f\"Matmul broadcasted {a.data.shape} and {b.data.shape} to {target_shape + a.data.shape[-2:]} and {target_shape + b.data.shape[-2:]}\"\n", + " # )\n", + " a = (a.broadcast(target_shape + a.data.shape[-2:]) if a_short_shape != target_shape else a)\n", + " b = (b.broadcast(target_shape + b.data.shape[-2:]) if b_short_shape != target_shape else b)\n", + "\n", + " return a, b" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from fastcore.test import test_eq, test_fail\n", + "\n", + "test_eq(calculate_target_shape((1, 2, 3), (2, 3)), (1, 2, 3))\n", + "test_eq(calculate_target_shape((1, 2, 3), (2, 1)), (1, 2, 3))\n", + "test_eq(calculate_target_shape((1, 2, 3), (1, 3)), (1, 2, 3))\n", + "test_eq(calculate_target_shape((1, 2, 3), (1, 1)), (1, 2, 3))\n", + "\n", + "test_eq(calculate_target_shape((1, 5), (3, 1)), (3, 5))\n", + "\n", + "test_fail(calculate_target_shape, args=((1, 2, 3), (2, 2)), contains=\"Cannot broadcast\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "_num_ops = 0\n", + "\n", + "class BaseOp:\n", + " \"\"\"Base class for all operations\"\"\"\n", + "\n", + " name_template = \"??\"\n", + "\n", + " # out: Tensor\n", + "\n", + " def __init__(self, *args, name: str = None):\n", + " from tidygrad.tensor import Tensor\n", + " global _num_ops\n", + " _num_ops += 1\n", + " assert isinstance(name, (str, type(None))), f\"name= should be str, got {type(name)}. You probably meant something else.\"\n", + "\n", + " self.args = [arg if isinstance(arg, Tensor) else Tensor(data=np.asarray(arg, dtype=np.float32)) for arg in args]\n", + " self.name = \"\"#(self.name_template.format(*[arg.name for arg in self.args]) if name is None else name)\n", + " self.requires_grad = any(arg.requires_grad for arg in self.args) and _grad\n", + " self.parents = []\n", + "\n", + " def set_out(self, data):\n", + " from tidygrad.tensor import Tensor\n", + " op = self if self.requires_grad else None\n", + " self.out = Tensor(data=data, requires_grad=self.requires_grad, name=self.name, op=op)\n", + "\n", + " def check_backward(self):\n", + " # Add more checks here?\n", + " assert (self.out.requires_grad), f\"You are trying to backpropagate through a non-differentiable operation:\\n{self}\"\n", + "\n", + " def __repr__(self):\n", + " return (f\"{self.__class__.__name__}({', '.join([str(arg) for arg in self.args])})\")\n", + "\n", + "class BinaryElementwiseOp(BaseOp):\n", + " \"\"\"Base class for binary elementwise operations\"\"\"\n", + " def __init__(self, a, b, name=None):\n", + " super().__init__(a, b, name=name)\n", + " self.args = maybe_broadcast_elementwise(*self.args)\n", + " if self.requires_grad:\n", + " self.parents = self.args\n", + "\n", + "class UnaryElementwiseOp(BaseOp):\n", + " \"\"\"Base class for unary elementwise operations\"\"\"\n", + " def __init__(self, a, name=None):\n", + " super().__init__(a, name=name)\n", + " if self.requires_grad:\n", + " self.parents = self.args" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Load(BaseOp):\n", + " \"\"\"Load a tensor\"\"\"\n", + "\n", + " name_template = \"?\"\n", + "\n", + " def __init__(self, name=None):\n", + " super().__init__(name=name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Add(BinaryElementwiseOp):\n", + " \"\"\"Add two tensors\"\"\"\n", + "\n", + " name_template = \"({}+{})\"\n", + "\n", + " def __init__(self, a, b, name=None):\n", + " super().__init__(a, b, name=name)\n", + " self.set_out(self.args[0].data + self.args[1].data)\n", + "\n", + " # def __call__(self, a, b):\n", + " # return Add(a, b, name=self.name)\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " self.parents[0].accum_grad(self.out.grad)\n", + " self.parents[1].accum_grad(self.out.grad)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Sub(BinaryElementwiseOp):\n", + " \"\"\"Subtract two tensors\"\"\"\n", + "\n", + " name_template = \"({}-{})\"\n", + "\n", + " def __init__(self, a, b, name=None):\n", + " super().__init__(a, b, name=name)\n", + " self.set_out(self.args[0].data - self.args[1].data)\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " self.parents[0].accum_grad(self.out.grad)\n", + " self.parents[1].accum_grad(-self.out.grad)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Mul(BinaryElementwiseOp):\n", + " \"\"\"Multiply two tensors\"\"\"\n", + "\n", + " name_template = \"({}*{})\"\n", + "\n", + " def __init__(self, a, b, name=None):\n", + " super().__init__(a, b, name=name)\n", + " self.set_out(self.args[0].data * self.args[1].data)\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + "\n", + " self.parents[0].accum_grad(self.out.grad * self.parents[1].data)\n", + " self.parents[1].accum_grad(self.out.grad * self.parents[0].data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Div(BinaryElementwiseOp):\n", + " \"\"\"Divide two tensors\"\"\"\n", + "\n", + " name_template = \"({}/{})\"\n", + "\n", + " def __init__(self, a, b, name=None):\n", + " super().__init__(a, b, name=name)\n", + " self.set_out(self.args[0].data / self.args[1].data)\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " self.parents[0].accum_grad(self.out.grad / self.parents[1].data)\n", + " self.parents[1].accum_grad(-self.out.grad * self.parents[0].data / (self.parents[1].data**2))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Neg(UnaryElementwiseOp):\n", + " \"\"\"Negate a tensor\"\"\"\n", + "\n", + " name_template = \"(-{})\"\n", + "\n", + " def __init__(self, a, name=None):\n", + " super().__init__(a, name=name)\n", + " self.set_out(-self.args[0].data)\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " self.parents[0].accum_grad(-self.out.grad)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Pow(UnaryElementwiseOp):\n", + " \"\"\"Raise a tensor to a power\"\"\"\n", + " def __init__(self, a, power, name=None):\n", + " self.name_template = f\"pow({{}},{power})\"\n", + " super().__init__(a, name=name)\n", + " self.power = power\n", + " self.set_out(self.args[0].data**power)\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " with np.errstate(divide='ignore'):\n", + " self.parents[0].accum_grad((self.out.grad * self.power * self.parents[0].data**(self.power - 1)))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Log(UnaryElementwiseOp):\n", + " \"\"\"Take the natural logarithm of a tensor\"\"\"\n", + "\n", + " name_template = \"log({})\"\n", + "\n", + " def __init__(self, a, name=None):\n", + " super().__init__(a, name=name)\n", + " self.set_out(np.log(self.args[0].data))\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " self.parents[0].accum_grad(self.out.grad / self.parents[0].data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Exp(UnaryElementwiseOp):\n", + " \"\"\"Exponentiate a tensor\"\"\"\n", + "\n", + " name_template = \"exp({})\"\n", + "\n", + " def __init__(self, a, name=None):\n", + " super().__init__(a, name=name)\n", + " self.set_out(np.exp(self.args[0].data))\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " self.parents[0].accum_grad(self.out.grad * self.out.data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class ExpLog(UnaryElementwiseOp):\n", + " \"\"\"Exponentiate a tensor\"\"\"\n", + "\n", + " name_template = \"exp({})\"\n", + "\n", + " def __init__(self, a, name=None):\n", + " super().__init__(a, name=name)\n", + "\n", + " def logexp(x):\n", + " return np.where(x < 0, np.log(1 + np.exp(x)), x + np.log(1 + np.exp(-x)))\n", + "\n", + " self.set_out(logexp(self.args[0].data))\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " self.parents[0].accum_grad(self.out.grad * (1 - 1 / (1 + np.exp(self.parents[0].data))))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Matmul(BaseOp):\n", + " \"\"\"Matrix multiplication of two tensors\"\"\"\n", + "\n", + " name_template = \"({}@{})\"\n", + "\n", + " def __init__(self, a, b, name=None):\n", + " super().__init__(a, b, name=name)\n", + " self.args = maybe_broadcast_matmul(*self.args)\n", + " if self.requires_grad:\n", + " self.parents = self.args\n", + "\n", + " self.set_out(np.matmul(self.args[0].data, self.args[1].data))\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " self.parents[0].accum_grad(np.matmul(self.out.grad, self.parents[1].data.swapaxes(-1, -2)))\n", + " self.parents[1].accum_grad(np.matmul(self.parents[0].data.swapaxes(-1, -2), self.out.grad))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Sum(BaseOp):\n", + " \"\"\"Sum-reduce a tensor along the given axis (int or tuple of ints)\"\"\"\n", + "\n", + " name_template = \"sum({})\"\n", + "\n", + " def __init__(self, a, axis=None, keepdims=False, name=None,):\n", + " super().__init__(a, name=name)\n", + " self.parents = self.args if self.requires_grad else []\n", + " self.set_out(np.sum(self.args[0].data, axis=axis, keepdims=keepdims))\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " self.parents[0].accum_grad(self.out.grad) # This will broadcast correctly" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Broadcast(BaseOp):\n", + " \"\"\"Broadcast a tensor to the given shape\"\"\"\n", + "\n", + " name_template = \"broadcast({})\"\n", + "\n", + " def __init__(self, a, target_shape, name=None):\n", + " super().__init__(a, name=name)\n", + " self.target_shape = target_shape\n", + " self.parents = self.args if self.requires_grad else []\n", + " self_shape = self.args[0].data.shape\n", + " assert self_shape != target_shape, \"Why are you broadcasting to the same shape?\"\n", + "\n", + " if len(self_shape) < len(target_shape):\n", + " expanded_shape = (len(target_shape) - len(self_shape)) * (1, ) + self_shape\n", + " else:\n", + " expanded_shape = self_shape\n", + "\n", + " final_shape = ()\n", + " broadcasted_dims = ()\n", + "\n", + " for s_expanded, s_target in reversed(list(zip(expanded_shape, target_shape))):\n", + " if s_expanded != s_target:\n", + " if s_expanded != 1:\n", + " raise ValueError(f\"Cannot broadcast {self_shape} to {target_shape}\")\n", + " else:\n", + " broadcasted_dims = (True, ) + broadcasted_dims\n", + " final_shape = (s_target, ) + final_shape\n", + " else:\n", + " broadcasted_dims = (False, ) + broadcasted_dims\n", + " final_shape = (s_expanded, ) + final_shape\n", + "\n", + " broadcasted_data = np.broadcast_to(self.args[0].data, final_shape)\n", + "\n", + " assert final_shape == broadcasted_data.shape\n", + "\n", + " data = broadcasted_data\n", + " self.broadcasted_dims = broadcasted_dims\n", + "\n", + " self.set_out(data)\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " axis = tuple([i for i, dim in enumerate(self.broadcasted_dims) if dim])\n", + " summed = self.out.grad.sum(axis=axis, keepdims=True)\n", + "\n", + " if summed.shape != self.parents[0].data.shape:\n", + " summed = summed.reshape(self.parents[0].data.shape)\n", + "\n", + " self.parents[0].accum_grad(summed)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Slice(UnaryElementwiseOp):\n", + " name_template = \"slice({})\"\n", + "\n", + " def __init__(self, a, key, name=None):\n", + " super().__init__(a, name=name)\n", + " self.key = key\n", + " self.set_out(self.args[0].data[key])\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " p = self.parents[0]\n", + "\n", + " if not p.requires_grad:\n", + " return\n", + "\n", + " if p.grad is None:\n", + " p.grad = np.zeros_like(p.data)\n", + "\n", + " p.grad[self.key] += self.out.grad" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# class LessThan(BinaryElementwiseOp):\n", + "# name_template = \"({}<{})\"\n", + "\n", + "# def __init__(self, a, b, name=None):\n", + "# super().__init__(a, b, name=name)\n", + "# self.out = Tensor(\n", + "# data=self.args[0].data < self.args[1].data, name=self.name, op=self\n", + "# )\n", + "\n", + "# # def backward(self):\n", + "# # self.parents[0].accum_grad(self.out.grad * (self.parents[0].data < self.parents[1].data)\n", + "# # self.parents[1].accum_grad(self.out.grad * (self.parents[0].data >= self.parents[1].data)\n", + "\n", + "# class Where(BaseOp):\n", + "# name_template = \"where({})\"\n", + "\n", + "# def __init__(self, a, b, c, name=None):\n", + "# super().__init__(a, b, c, name=name)\n", + "# self.parents = self.args\n", + "# self.out = Tensor(\n", + "# data=np.where(self.args[0].data, self.args[1].data, self.args[2].data),\n", + "# name=self.name,\n", + "# op=self,\n", + "# )\n", + "\n", + "# def backward(self):\n", + "# # self.parents[0].accum_grad(self.out.grad * self.parents[1].data\n", + "# # self.parents[0].accum_grad(self.out.grad * self.parents[2].data\n", + "\n", + "# self.parents[1].accum_grad(self.out.grad * self.parents[0].data\n", + "# self.parents[2].accum_grad(self.out.grad * (1 - self.parents[0].data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Transpose(UnaryElementwiseOp):\n", + " \"\"\"Transpose a tensor\"\"\"\n", + "\n", + " name_template = \"transpose({})\"\n", + "\n", + " def __init__(self, a, dim0, dim1, name=None):\n", + " super().__init__(a, name=name)\n", + " self.dim0 = dim0\n", + " self.dim1 = dim1\n", + " self.set_out(np.swapaxes(self.args[0].data, dim0, dim1))\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " self.parents[0].accum_grad(np.swapaxes(self.out.grad, self.dim0, self.dim1))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Dropout(UnaryElementwiseOp):\n", + " \"\"\"Apply Dropout to a tensor\"\"\"\n", + "\n", + " name_template = \"dropout({})\"\n", + "\n", + " def __init__(self, a, p_drop=0.1, training=True, name=None):\n", + " if p_drop == 0:\n", + " return a\n", + "\n", + " super().__init__(a, name=name)\n", + " assert 0 < p_drop < 1, f\"p_drop must in (0, 1), got {p_drop}\"\n", + " self.p_drop = p_drop\n", + " self.training = training\n", + " if training:\n", + " # Note: We scale up the outputs during training rather than scaling down during inference.\n", + " scale_factor = 1 / (1-p_drop)\n", + " self.mask = np.random.binomial(scale_factor, 1 - p_drop, size=self.args[0].data.shape)\n", + " self.set_out(self.args[0].data * self.mask)\n", + " else:\n", + " self.set_out(self.args[0].data)\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " self.parents[0].accum_grad(self.out.grad * (self.mask if self.training else 1))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "class Embedding(UnaryElementwiseOp):\n", + " \"\"\"Embedding layer\"\"\"\n", + "\n", + " name_template = \"embedding({})\"\n", + "\n", + " def __init__(self, a, indices, name=None):\n", + " super().__init__(a, name=name)\n", + " self.indices = indices\n", + " self.set_out(self.args[0].data[self.indices])\n", + "\n", + " def backward(self):\n", + " self.check_backward()\n", + " if self.parents[0].grad is None:\n", + " self.parents[0].grad = np.zeros_like(self.parents[0].data, dtype=np.float32)\n", + " self.parents[0].grad[self.indices] += self.out.grad" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/nbs/04_model.ipynb b/nbs/04_model.ipynb new file mode 100644 index 0000000..0a93ffc --- /dev/null +++ b/nbs/04_model.ipynb @@ -0,0 +1,87 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | default_exp model\n", + "# | hide\n", + "import nbdev\n", + "from nbdev.showdoc import *\n", + "\n", + "nbdev.nbdev_export()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | exporti\n", + "\n", + "import os\n", + "\n", + "from lovely_numpy import Lo\n", + "\n", + "import numpy as np\n", + "from tidygrad.tensor import Tensor\n", + "import safetensors\n", + "import safetensors.numpy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "\n", + "class Model:\n", + " def __init__(self, params: dict[str, tuple] | str | os.PathLike):\n", + "\n", + " self.params = {}\n", + "\n", + " if isinstance(params, dict):\n", + " for name, shape in params.items():\n", + " self.params[name] = Tensor(np.zeros(shape))\n", + "\n", + " elif isinstance(params, (str, os.PathLike)):\n", + " model = safetensors.safe_open(params, framework=\"numpy\")\n", + " for name in model.keys():\n", + " self.params[name] = Tensor(model.get_tensor(name), name=name)\n", + "\n", + " else:\n", + " raise TypeError(\"params must be a dict or a path\")\n", + "\n", + " def __repr__(self):\n", + " return f\"Model with params:\\n\" + \"\\n\".join([f\"\\t{name}: {param.shape}\" for name, param in self.params.items()])\n", + "\n", + " def save(self, filename: str):\n", + " d = {key: self.params[key].data for key in self.params.keys()}\n", + " safetensors.numpy.save_file(d, filename)\n", + "\n", + "\n", + " def requires_grad(self, value):\n", + " for name, param in self.params.items():\n", + " param.requires_grad = value\n", + "\n", + " def parameter_list(self):\n", + " return list(self.params.values())\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/nbs/06_training.ipynb b/nbs/06_training.ipynb index 2703ff2..1916f63 100644 --- a/nbs/06_training.ipynb +++ b/nbs/06_training.ipynb @@ -239,15 +239,20 @@ "outputs": [], "source": [ "#| export\n", + "\n", "def one_hot_encode_batch(y, n_classes):\n", - " batch_size = len(y)\n", + " diag = np.eye(n_classes)\n", + " return Tensor(diag[y])\n", + "\n", + "\n", + " batch_size = y.shape[0]\n", " assert batch_size > 0\n", " assert n_classes > 0\n", - " assert y.shape == (batch_size, )\n", + " # assert y.shape[0] == batch_size\n", " assert np.min(y) >= 0\n", "\n", " # Initialize a zero matrix of shape (batch_size, num_classes)\n", - " one_hot_matrix = np.zeros((batch_size, n_classes))\n", + " one_hot_matrix = np.zeros((*y.shape, n_classes))\n", "\n", " # Fill in the appropriate elements\n", " one_hot_matrix[np.arange(batch_size), y] = 1\n", @@ -526,7 +531,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "871fb9ae514d498eaf006b70d83510fb", + "model_id": "fca1cf84a7d74983bf209d9eab87ed20", "version_major": 2, "version_minor": 0 }, @@ -540,7 +545,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "39bf1b0d1dca4206aad42dfed6454b0b", + "model_id": "3c29691b6e214e14baf786bcde45df92", "version_major": 2, "version_minor": 0 }, @@ -553,7 +558,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxUAAAGGCAYAAAANcKzOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOydd5wU9f3/n7N9r/cGd3AgwilNQBEQuxJU1EQTNInGWKKRiEri12DsifJLbMSCRqMSo1GjMYlRLBhjBRURRAEpUo5yvbetM78/Zmd2Znf2bq/QP8/Hgwe7s5+Z+czu3s779Xk3SVEUBYFAIBAIBAKBQCDoI7Z9PQGBQCAQCAQCgUBwYCNEhUAgEAgEAoFAIOgXQlQIBAKBQCAQCASCfiFEhUAgEAgEAoFAIOgXQlQIBAKBQCAQCASCfiFEhUAgEAgEAoFAIOgXQlQIBAKBQCAQCASCfiFEhUAgEAgEAoFAIOgXQlQIBAKBQCAQCASCfiFEhUAwQCxevBhJkti2bdu+nopAIBAIBALBXkWICoFAIBAIBAKBQNAvhKgQCAQCgUAgOADo7Ozc11MQCBIiRIVAsAd56qmnGDduHB6Ph5ycHL773e+yfv1605gtW7ZwwQUXUFJSgtvtprCwkFNOOYXVq1frY959911OPPFEcnNz8Xq9lJWVcd5554kbjEAgEPSDzZs389Of/pQRI0aQkpLCoEGDmDVrFl999VXc2ObmZn75y18ybNgw3G43BQUFnHHGGXzzzTf6GL/fz5133klFRQUej4fc3FxOOukkli1bBsC2bduQJInFixfHHV+SJG6//Xb9+e23344kSXzxxRecf/75ZGdnM3z4cAA+//xzLrjgAoYOHYrX62Xo0KFceOGFbN++Pe64u3bt4mc/+xmlpaW4XC5KSko4//zzqampob29naysLK688sq4/bZt24bdbueee+7p7dsqOERx7OsJCAQHKwsWLOCmm27iwgsvZMGCBTQ0NHD77bczZcoUVqxYwYgRIwA444wzCIfD/OEPf6CsrIz6+nqWLVtGc3MzoP6wn3nmmUyfPp2nnnqKrKwsdu3axZtvvkkgECAlJWUfXqVAIBAcuOzevZvc3Fz+3//7f+Tn59PY2Mhf/vIXJk+ezKpVqxg5ciQAbW1tHHfccWzbto0bb7yRyZMn097ezgcffEBVVRWjRo0iFAoxc+ZMPvzwQ6677jpOPvlkQqEQn3zyCZWVlUydOrVPc/ze977HBRdcwFVXXUVHRweg3hdGjhzJBRdcQE5ODlVVVTz66KMcffTRrFu3jry8PEAVFEcffTTBYJCbbrqJsWPH0tDQwFtvvUVTUxOFhYVceumlPP744/zhD38gMzNTP++iRYtwuVxceuml/XyXBYcMikAgGBCefvppBVC2bt2qNDU1KV6vVznjjDNMYyorKxW326388Ic/VBRFUerr6xVAWbhwYcLjvvzyywqgrF69eo/OXyAQCA51QqGQEggElBEjRijXX3+9vv3OO+9UAGXp0qUJ933mmWcUQHniiScSjtm6dasCKE8//XTca4By22236c9vu+02BVBuvfXWpObd3t6upKamKn/84x/17ZdeeqnidDqVdevWJdz322+/VWw2m/LAAw/o27q6upTc3Fzlpz/9aY/nFgg0RPiTQLAHWL58OV1dXVxyySWm7aWlpZx88sn897//BSAnJ4fhw4dzzz33cP/997Nq1SpkWTbtM378eFwuFz/72c/4y1/+wpYtW/bWZQgEAsFBTSgU4u677+aII47A5XLhcDhwuVxs2rTJFKr6xhtvcPjhh3PqqacmPNYbb7yBx+MZ8JX98847L25be3s7N954I4cddhgOhwOHw0FaWhodHR1x8z7ppJOoqKhIePxhw4Zx1llnsWjRIhRFAeBvf/sbDQ0N/OIXvxjQaxEc3AhRIRDsARoaGgAoLi6Oe62kpER/XZIk/vvf/zJjxgz+8Ic/MGHCBPLz85k7dy5tbW0ADB8+nHfeeYeCggLmzJnD8OHDGT58OH/84x/33gUJBALBQci8efO45ZZbOPfcc/nPf/7Dp59+yooVKxg3bhxdXV36uLq6OgYPHtztserq6igpKcFmG1jTyuo+8sMf/pCHH36Yyy+/nLfeeovPPvuMFStWkJ+f3+t5A1x77bVs2rSJpUuXAvDII48wZcoUJkyYMHAXIjjoETkVAsEeIDc3F4Cqqqq413bv3q3HuwIMGTKEJ598EoCNGzfy97//ndtvv51AIMBjjz0GwPTp05k+fTrhcJjPP/+chx56iOuuu47CwkIuuOCCvXBFAoFAcPDx7LPPcvHFF3P33XebttfX15OVlaU/z8/PZ+fOnd0eKz8/n48++ghZlhMKC4/HA6gJ3Ua0hSYrJEkyPW9paeG1117jtttu49e//rW+3e/309jYGDennuYNcPLJJzN69Ggefvhh0tLS+OKLL3j22Wd73E8gMCI8FQLBHmDKlCl4vd64H+WdO3fy7rvvcsopp1jud/jhh3PzzTczZswYvvjii7jX7XY7kydP5pFHHgGwHCMQCASC5JAkCbfbbdr2+uuvs2vXLtO2mTNnsnHjRt59992Ex5o5cyY+n8+yspNGYWEhHo+HNWvWmLb/+9//7tWcFUWJm/ef//xnwuFw3Jz+97//sWHDhh6PO3fuXF5//XXmz59PYWEh3//+95Oek0AAwlMhEOwRsrKyuOWWW7jpppu4+OKLufDCC2loaOCOO+7A4/Fw2223AbBmzRp+8Ytf8P3vf58RI0bgcrl49913WbNmjb4C9dhjj/Huu+9y5plnUlZWhs/n46mnngLoNr5XIBAIBN1z1llnsXjxYkaNGsXYsWNZuXIl99xzT1zI0HXXXceLL77IOeecw69//WuOOeYYurq6eP/99znrrLM46aSTuPDCC3n66ae56qqr2LBhAyeddBKyLPPpp59SUVHBBRdcgCRJ/PjHP+app55i+PDhjBs3js8++4y//e1vSc85IyOD448/nnvuuYe8vDyGDh3K+++/z5NPPmnyrgDceeedvPHGGxx//PHcdNNNjBkzhubmZt58803mzZvHqFGj9LE//vGPmT9/Ph988AE333wzLperX++t4BBkX2eKCwQHC8bqTxp//vOflbFjxyoul0vJzMxUzjnnHGXt2rX66zU1Ncoll1yijBo1SklNTVXS0tKUsWPHKg888IASCoUURVGU5cuXK9/97neVIUOGKG63W8nNzVVOOOEE5dVXX93blygQCAQHFU1NTcpll12mFBQUKCkpKcpxxx2nfPjhh8oJJ5ygnHDCCXFjr732WqWsrExxOp1KQUGBcuaZZyrffPONPqarq0u59dZblREjRigul0vJzc1VTj75ZGXZsmX6mJaWFuXyyy9XCgsLldTUVGXWrFnKtm3bElZ/qquri5v3zp07lfPOO0/Jzs5W0tPTle985zvK119/rQwZMkT5yU9+Yhq7Y8cO5dJLL1WKiooUp9OplJSUKD/4wQ+UmpqauONecsklisPhUHbu3Nm3N1RwSCMpSiTVXyAQCAQCgUBwSBIIBBg6dCjHHXccf//73/f1dAQHICL8SSAQCAQCgeAQpa6ujg0bNvD0009TU1NjSv4WCHqDEBUCgUAgEAgEhyivv/46P/3pTykuLmbRokWijKygz4jwJ4FAIBAIBAKBQNAvRElZgUAgEAgEAoFA0C+EqBAIBAKBQCAQCAT9QogKgUAgEAgEAoFA0C8OqURtWZbZvXs36enpcW3vBQKBQBBFURTa2tooKSnBZjv41p/E/UAgEAiSI9n7wSElKnbv3k1paem+noZAIBAcMOzYsSOuu/DBgLgfCAQCQe/o6X5wSImK9PR0QH1TMjIyer3/lvp2vv/ocvwhmVtmVVCQ5uGY8hxS3YfU2ygQCA4BWltbKS0t1X83Dzb6ez8QCASCgwFfMMzybxsAOG5EHk57vCci2fvBIWUNay7ujIyMPt1EMgN2srIyqGsLELZ5SE1LJy09nXSPc6CnKhAIBPsFB2toUH/vBwKBQHAw4AqGSU0LAJCWno7bYU84tqf7wcEXKLsHsUmQ7lYFRKsvBIAsunwIBAJBtyxatIjy8nI8Hg8TJ07kww8/7Hb8I488QkVFBV6vl5EjR/LMM8/EjfnHP/7BEUccgdvt5ogjjuCf//znnpq+QCAQHLTIhnZ1/e1cJ0RFL5CQyPBqoiIIqMkrAoFAILDmxRdf5LrrruM3v/kNq1atYvr06cycOZPKykrL8Y8++ijz58/n9ttvZ+3atdxxxx3MmTOH//znP/qY5cuXM3v2bC666CK+/PJLLrroIn7wgx/w6aef7q3LEggEgoMC4+J4uJ8r5X0SFb1ddXr//feZOHEiHo+HYcOG8dhjj8WNaW5uZs6cORQXF+PxeKioqGDJkiX667fffjuSJJn+FRUV9WX6fUaSIMOjRoy1CU+FQCAQ9Mj999/PZZddxuWXX05FRQULFy6ktLSURx991HL8X//6V6688kpmz57NsGHDuOCCC7jsssv4/e9/r49ZuHAhp512GvPnz2fUqFHMnz+fU045hYULF+6lqxIIBIKDA6OnQu7nQnmvcyq0VadFixYxbdo0/vSnPzFz5kzWrVtHWVlZ3PitW7dyxhlncMUVV/Dss8/y8ccfc/XVV5Ofn895550HQCAQ4LTTTqOgoICXX36ZwYMHs2PHjriEkCOPPJJ33nlHf263J4772hPYbJKeP9HapXoq+vsBCAQHMuFwmGAwuK+nIegDTqdzj/+GBgIBVq5cya9//WvT9tNPP51ly5ZZ7uP3+/F4PKZtXq+Xzz77jGAwiNPpZPny5Vx//fWmMTNmzOhWVPj9fvx+v/68tbW1l1cjEAgEBx+KHH0sy4nHJUOvRYVx1QnUFaO33nqLRx99lAULFsSNf+yxxygrK9N/7CsqKvj888+59957dVHx1FNP0djYyLJly3A6VaN9yJAh8ZN1OPa6d8KITYL0OE+FEBWCQw9FUaiurqa5uXlfT0XQD7KysigqKtpjydj19fWEw2EKCwtN2wsLC6murrbcZ8aMGfz5z3/m3HPPZcKECaxcuZKnnnqKYDBIfX09xcXFVFdX9+qYAAsWLOCOO+7o/0UJBALBQYTCPvJU9GXVafny5Zx++ummbTNmzODJJ5/UV51effVVpkyZwpw5c/j3v/9Nfn4+P/zhD7nxxhtNK2mbNm2ipKQEt9vN5MmTufvuuxk2bFjC+Q70ypSERHpMTkV/VZ1AcCCiCYqCggJSUlIO2gpBByuKotDZ2UltbS0AxcXFe/R8sd8PRVESfmduueUWqqurOfbYY1EUhcLCQi655BL+8Ic/mO4HvTkmwPz585k3b57+XCuRKBAIBIcyppyKvSkq+rLqlGhFKRQK6atOW7Zs4d133+VHP/oRS5YsYdOmTcyZM4dQKMStt94KwOTJk3nmmWc4/PDDqamp4Xe/+x1Tp05l7dq15ObmWp57oFembBJkuIWnQnBoEw6HdUGR6G9PsP/j9XoBqK2tpaCgYI+EQuXl5WG32+PuD7W1tXH3BeO8nnrqKf70pz9RU1NDcXExjz/+OOnp6eTl5QFQVFTUq2MCuN1u3G53P69IIBAI9i9afUE8DjsuR99qLw1kTkWfZtDbFSKr8cbtsixTUFDA448/zsSJE7ngggv4zW9+Y0rkmzlzJueddx5jxozh1FNP5fXXXwfgL3/5S8Lzzp8/n5aWFv3fjh07enehFtehV38SORWCQxQthyIlJWUfz0TQX7TPcE/lxbhcLiZOnMjSpUtN25cuXcrUqVO73dfpdDJ48GDsdjsvvPACZ511FjabesuaMmVK3DHffvvtHo8pEAgEBxPNnQE+29LI8i0NfT6GSVTszZyKvqw6JVpRcjgc+ipncXFxXNJgRUUF1dXVBAIBXC5X3HFTU1MZM2YMmzZtSjjfgV6ZkiRIi3gqOgJhZEXpd01fgeBARYQ8Hfjsjc9w3rx5XHTRRUyaNIkpU6bw+OOPU1lZyVVXXQWoiz+7du3Se1Fs3LiRzz77jMmTJ9PU1MT999/P119/bVpAuvbaazn++OP5/e9/zznnnMO///1v3nnnHT766KM9fj0CgUCwv1Dbpob4B0N9VwPKAIY/9cpT0ZdVp0QrSpMmTdKTsqdNm8bmzZuRDRJp48aNFBcXWwoKUPMl1q9fv8djgY3YJIlUd1T4dEWEhUAgEAismT17NgsXLuTOO+9k/PjxfPDBByxZskQvxlFVVWXqWREOh7nvvvsYN24cp512Gj6fj2XLljF06FB9zNSpU3nhhRd4+umnGTt2LIsXL+bFF19k8uTJe/vyBAKBYJ/R374SEOup2MslZXu76nTVVVfx8MMPM2/ePK644gqWL1/Ok08+yfPPP68f8+c//zkPPfQQ1157Lddccw2bNm3i7rvvZu7cufqYX/3qV8yaNYuysjJqa2v53e9+R2trKz/5yU/69Qb0Brsk4bDZcNltBMIyHYGQ6FMhEBzinHjiiYwfP75fPRIG4hj7M1dffTVXX3215WuLFy82Pa+oqGDVqlU9HvP888/n/PPPH4jpCQQCwQHJwIiK6OP+rpP3WlTMnj2bhoYG7rzzTqqqqhg9enS3q07l5eUsWbKE66+/nkceeYSSkhIefPBBvZwsQGlpKW+//TbXX389Y8eOZdCgQVx77bXceOON+pidO3dy4YUXUl9fT35+PsceeyyffPKJZenZPUUknJdUt51Ap0yn8FQIBAcMPYX6/OQnP4kzcJPhlVde0b2uAoFAIBDsLQZCVCgGO3avVn/S6M2qE8AJJ5zAF1980e0xp0yZwieffJLw9RdeeKFXc9wTOCKqIsXloKkzSIc/ZPowBALB/ktVVZX++MUXX+TWW29lw4YN+jatGpKGVvK6J3JycgZukgKBQCA4IJBlBZtt3+YW9lcEQExORT9FSt/qTx2iaJ6KlEhehZqsvQ8nJBAIkqaoqEj/l5mZiSRJ+nOfz0dWVhZ///vfOfHEE/F4PDz77LM0NDRw4YUXMnjwYFJSUhgzZowpdBPU0KXrrrtOfz506FDuvvtuLr30UtLT0ykrK+Pxxx/v1Vybmpq4+OKLyc7OJiUlhZkzZ5qKUmzfvp1Zs2aRnZ1NamoqRx55JEuWLNH3/dGPfkR+fj5er5cRI0bw9NNP9/2NEwgEAoGJHY2d/G9DLQ3t/p4H70H6mwMB5pyK/i6U98lTcahij4RPpLrUt63THxLhTwIB6g9RVzC8T87tddoHrIrRjTfeyH333cfTTz+N2+3G5/MxceJEbrzxRjIyMnj99de56KKLGDZsWLdJwffddx+//e1vuemmm3j55Zf5+c9/zvHHH8+oUaOSmscll1zCpk2bePXVV8nIyODGG2/kjDPOYN26dTidTubMmUMgEOCDDz4gNTWVdevWkZaWBqjN49atW8cbb7xBXl4emzdvpqura0DeH4FAIBDAhuo2AL7a1cKJIwv22TxCA5xT0d/DCVHRC+w2TVQYPBWio7ZAQFcwzBG3vrVPzr3uzhmkuAbmp+y6667je9/7nmnbr371K/3xNddcw5tvvslLL73Urag444wz9BDRG2+8kQceeID33nsvKVGhiYmPP/5Yr6r33HPPUVpayr/+9S++//3vU1lZqfftARg2bJi+f2VlJUcddRSTJk0CMFVNEggEgv2FqpYuNta0M25wJlkp1pU+9xcUReHz7U3YbRITyrL17aHwvl1YNnoq+hqOZVwcF+FPexFJkrDZ0A2YzoDwVAgEBxOaIa4RDoe56667GDt2LLm5uaSlpfH222+bilFYMXbsWP2xFmZVW1ub1BzWr1+Pw+EwiZbc3FxGjhzJ+vXrAZg7dy6/+93vmDZtGrfddhtr1qzRx/785z/nhRdeYPz48fzf//0fy5YtS+q8AoFAsDdpaA8QDMk0de6Z5psDSSAs09IZpLE9MCAhRwNFeAC6YSsD2FFbeCp6id1m03tVdPhF9SeBANQQpHV3zthn5x4oUlNTTc/vu+8+HnjgARYuXMiYMWNITU3luuuuIxAIdHuc2ARvSZJMfXi6I1FMq6IoepjX5ZdfzowZM3j99dd5++23WbBgAffddx/XXHMNM2fOZPv27bz++uu88847nHLKKcyZM4d77703qfMLBALB3uRAKHijmEKE+j/f+nY/G6vbOLIkk8yUvlcPNIY/9VXryAN4bcJT0UvskkSq2+ip2McTEgj2AyRJIsXl2Cf/9mRX6A8//JBzzjmHH//4x4wbN45hw4aZEqb3BEcccQShUIhPP/1U39bQ0MDGjRupqKjQt5WWlnLVVVfxyiuv8Mtf/pInnnhCfy0/P59LLrmEZ599loULF/Y6UVwgEAj2NJoBeyDYUaYGcQr097azurKZzkCY1Tub+3wMRVEIhwfCUxF9LMKf9jJq+JPmqRDhTwLBwcxhhx3G0qVLWbZsGevXr+fKK6+kurp6j55zxIgRnHPOOVxxxRV89NFHfPnll/z4xz9m0KBBnHPOOYCa+/HWW2+xdetWvvjiC959911dcNx66638+9//ZvPmzaxdu5bXXnvNJEYEAoFgoAnLCl9UNlHZ0Jn0Ppr5tL96Klp9QVZsa6S5MxC3mu+wx5vP63a3sr6qtVfn6E8oVWySdl/fxljB1B+EqOgl9siKLKiJ2vvrH4NAIOg/t9xyCxMmTGDGjBmceOKJFBUVce655+7x8z799NNMnDiRs846iylTpqAoCkuWLNHDqsLhMHPmzKGiooLvfOc7jBw5kkWLFgHgcrmYP38+Y8eO5fjjj8dut+8XfX4EAsHBS3Wrj8b2ABtr2pLeRzNm91cr6ovtTbR0Bvl8W1NM2VVwxiREB8Myu5u72NXU1bvV/n54PIJhc0it0sd3UpSU3Yc47JKeU9Ep+lQIBAckl1xyCZdccon+fOjQoZY/pjk5OfzrX//q9ljvvfee6fm2bdvixqxevbpXx8jOzuaZZ55JOP6hhx5K+NrNN9/MzTff3O35BAKBYCDpy4q7tsf+GvFhrOykGOz3qKciHBknx6z2K9iTVAv9iaIKxlSe6mvokgh/2ofYjDkV/tB+VQVAIBAIBAKBYG9j70MpU82YPRBK88eKBuPlBsNKnw3z/uQEhuI8FX3DVFJWJGrvXey2aPiTLyTHuZ8EAoFAIBAIDiVsBuM42cVWRQ9/2v8XZ2PzDowzDoRlk6jojV0+kJ4KpY/mqNzHuVshREUvsUkSKc7o29bmC+3D2QgEAoFAIBDsW2wGazLZLs/aqP00+knHbpdiDG+zZyJoEf6ULP2pIhWKcfH0NYysr3O3QoiKXuKwS9htNrK8asJkZWN8pYNt9R3sbEq+AoJAIBAIBALBwUCy4T+aR2N/FxUOmxTTIM6c0NwvUdEPX0XsaapafL2qvmV1HJFTsZexR2TlyKJ0ANbubjG93hUIs7m2nW+qkq+AIBAIBAKBQHDAYrBFY1fQe9plf03U1rBLUlxJWeOMgyHF/HovwpD646mIfd92N3exsaYNXzDcq+MMZEdtISp6iS2SnVNRlAHAuhjxEDR8m0S5WYFAIBAIBAc7ch9Wu6PN7/Y/W8mYF2K3SXGeCOOU1ZyKvnoqkqeuzU9VS5fhPNbjkgk/C4RkttV34AuG+yyIrBAlZXuJ5qkYVax6KrbUtdPuD5EWqQgVm6yzB5v9CgQCgUAgEOxzjGv3SYfQKKb/9iuMC8SxokKJCX8KhOQ4T0bSJGkjhsIyX+5oBiAn1YXbYU+4cJ3M+792dwsN7QGqW31xr8myoi+g9xbhqeglWtm0vDQ3eWkuZAVWbm+KDlAsHwoEAoFAIBAclCQTl1/T6qPNF4zuo++b2FoKywo7mzrxh3oX0uMLhtnV3NXnsv+mHhWYry82/CnWU9GbsqzJ5lQ0dgb0x1rVp0SXloyoaGhXj9fuC8WJoP6UlRWiopcYazEPz08DYNX2Jhra/YRlxaTW90eXnkAgEAgEAsFAYrR2rMJvmjoCfLWzhU+3NOrbouFPiY+7qbaNb6ra+GpnS+JBFmyubWf97lbq2/292k/D2C5AlpX4krLG8KcYT0WvHBVJOgQaOwyiIiRHztN3T4WR2MP0x3YVoqKXGEVFeV4qAB9/W8+qymY21rT1uVaxQCA4MDjxxBO57rrrEr5+++23M378+L02H4FAINjXmFbqLYxaq/L72i7d2Uo7G9UcgubOYOJBFmieDX/IOkkgEJKpbfMl9GQYe0DICjF5B+YF5EBoz+dUaJ4FiIZmJTpLb0VB7Pj+5FUIUdFLjA1eyvNVUfFNdRsKCruausxxdyIASiDYb5g1axannnqq5WvLly9HkiS++OKLvTwrgUAgOPAx2qVWngorQ3dPJmprjoZEx/58eyNrdrSwI0H5f2MFKwXFIqciOjYQjkl27pWnomdZ4QuG6QpEw79CYfP7FpRl/t8b63nkvc0oKEn3CdEQnop9iDF3pTQ7BZukKvCGiGtqIDsTCgSCgeOyyy7j3XffZfv27XGvPfXUU4wfP54JEybsg5kJBALBgY05pyJ+qdvKUO2ppGxf8yHUOSim/2Pp9KtGek2rdXhUMJQ4MTtsyKnoCIT40/tbeH9TbZ/mnUz4kzEUy/hce5s317Szua6DVZXNVDZ29vp9i/NUCFGx9zCqSpfdRml2CgDb6jpwOmyijKxAsJ9y1llnUVBQwOLFi03bOzs7efHFF7nssstoaGjgwgsvZPDgwaSkpDBmzBief/75fp1XlmXuvPNOBg8ejNvtZvz48bz55pv664FAgF/84hcUFxfj8XgYOnQoCxYs0F+//fbbKSsrw+12U1JSwty5c/s1H4FAIBhojJEZ1p4Ki330+CfrY7YHoiFTktS7Mv3JekESFTkyVn+SFSVmwVjR5/LGV9V8urWRG1/+Ku7ciTBeRzLhT8akcYiGZmnv+YaaaGuDz7Y29iqnwhcK84c3N/Damt3R+SW9dzyipGwvMX4BHHaJkiwP2xs7qWv347TFN0gRCA4JFAWC+6iLvDMlqeUeh8PBxRdfzOLFi7n11lv1BYKXXnqJQCDAj370Izo7O5k4cSI33ngjGRkZvP7661x00UUMGzaMyZMn92l6f/zjH7nvvvv405/+xFFHHcVTTz3F2Wefzdq1axkxYgQPPvggr776Kn//+98pKytjx44d7NixA4CXX36ZBx54gBdeeIEjjzyS6upqvvzyyz7NQyAQCEBLLFbwOO0DdkxT+FPYSlQYV/4VJEnS90lkAxvzMBRFrbLkdtgN2xQ6AmG9pL+RkO6p6H7eicKPTNWfFPP8ZSVqeDcZqjIZX+8O43uVlKcixvOjhWZpx9lQ3aq/tmJbE6GeLtrAkq+qWFfVyrqqVmaNK4kL7eotQlT0Eqcj6tyxSRJpbicA7f4QTofNVIpLaArBIUOwE+4u2Tfnvmk3uFKTGnrppZdyzz338N5773HSSScBaujT9773PbKzs8nOzuZXv/qVPv6aa67hzTff5KWXXuqzqLj33nu58cYbueCCCwD4/e9/z//+9z8WLlzII488QmVlJSNGjOC4445DkiSGDBmi71tZWUlRURGnnnoqTqeTsrIyjjnmmD7NQyAQCAA+2FgHwIkj83HYByZgxWjuWK2UxxrldsmQqJ1gbbzDb07uDoTMomJTbTuVDZ0ML0jTC+fo5+gh/EkjoaciHOOpMNjpssFTkeKKmtGBsIzLbutxQdn8es+qIs5TEYp6YQIhmS31HfprjR0BdjR1cVhheo/HBVi7KypI7JJESFH65aoQ4U+9JM3tYFh+KhUlGdgkiQyv+oVq7QohYY6lE5pCINi/GDVqFFOnTuWpp54C4Ntvv+XDDz/k0ksvBSAcDnPXXXcxduxYcnNzSUtL4+2336aysrJP52ttbWX37t1MmzbNtH3atGmsX78egEsuuYTVq1czcuRI5s6dy9tvv62P+/73v09XVxfDhg3jiiuu4J///CehUHwVFYFAIEgGo43iS1AZqU/H7aFPQ6xRbpxHIrs/EDO/2OeVDap3/NvadtN2RVF0MdFz+JO1Ud/uN3tJYhO1o/tHHzd0+PXza1j11zBebzKeijhRYfBUbG/sICxDltdJYYYbgJqW+IZ2VgTCMtsboxEG/rBWVcrwHQmGafMFTe9HdwhPRR8YFulPsb2+g/SIp6LNH1T7VMTE3QkEhwTOFNVjsK/O3Qsuu+wyfvGLX/DII4/w9NNPM2TIEE455RQA7rvvPh544AEWLlzImDFjSE1N5brrriMQiHdx94ZYF7vm/geYMGECW7du5Y033uCdd97hBz/4Aaeeeiovv/wypaWlbNiwgaVLl/LOO+9w9dVXc8899/D+++/jdDr7NSeBQHDoERuGNFD01PzOJDpkBZtdsnzNSGxuRiDJsJ6ewtBNOQ0WRr0sK3Qa8jkUlLjr00RSh2FcQ3uA4gyvHnJV1dLF2l2tlOen6n3NYueUzEeghTu5nTb8QdlU/UnrX1GY4cZuk6hp9VPT1r2o0K5/Q3WbaXtXJJRMe/9auoKs2NqIoig0tyTXJ0SIin5gt0mkedS3sN0XIqzElB3bVxMTCPY2kpR0CNK+5gc/+AHXXnstf/vb3/jLX/7CFVdcoRv4H374Ieeccw4//vGPATXJetOmTVRUVPTpXBkZGZSUlPDRRx9x/PHH69uXLVtmCmPKyMhg9uzZzJ49m/PPP5/vfOc7NDY2kpOTg9fr5eyzz+bss89mzpw5jBo1iq+++kpUqhIIDiBkWcGWKNZmb85jjxkmhkRti5wKo0CIraaEYv3+aFWkbDbV0+EPJicqjKLGSuAY52KTpLhzdwRCJs9KOKyYSswa7bx2Q96H1mhPe319lRpatLWuwyQqzEKi5w9Em6/XaccflKPVnxTV8AfI8LpwO2xAGzWt/m6/b9rx6trMla86/CHS3A7d01PT4uPfq3fx8bcN5LmS62guREU/yE1zR8OffOqX0JRTMXCeRYFAMECkpaUxe/ZsbrrpJlpaWrjkkkv01w477DD+8Y9/sGzZMrKzs7n//vuprq7us6gAuOGGG7jtttsYPnw448eP5+mnn2b16tU899xzADzwwAMUFxczfvx4bDYbL730EkVFRWRlZbF48WLC4TCTJ08mJSWFv/71r3i9XlPehUAg2L+pbfWxZmcLFSUZDMry7tO5xOY2DNxxo48tDfnYHAVFMo1/b2Mtg7JSGFkUzQXQqhx5nQ46/KGkPRVhU2hV/FyMr7d2BfnfhlqG5kW9CVqoT6rbwTdVrdy1ZD3nHlXCd44sjjumMZlca1CnV55KMN3eeio0EaEm1gf154qi6KIi0+skxaVmNOxs6uS9jbVMKMsmK8WV8PpbfeaGgu2BEIVAZWMnqyqb+XJnM/9ZUwWAvzO5zuQip6IflGR5ouFPvqApeQdE8zuBYH/lsssuo6mpiVNPPZWysjJ9+y233MKECROYMWMGJ554IkVFRZx77rn9OtfcuXP55S9/yS9/+UvGjBnDm2++yauvvsqIESMAVeT8/ve/Z9KkSRx99NFs27aNJUuWYLPZyMrK4oknnmDatGmMHTuW//73v/znP/8hNze3X3MSCAR7jzU71dCR9btbexi55zEasf3pA9HdcUMW1nQoxtCPtY9kGXY0misIasZviktNzo7NqUhE2BRqFf+6cS6dgTCKonoTNDTvQ4bXwcrKJkKywn/X1+pzXrq2hoX/3UiHP6Qb9QD1EVHRk1DobWXQYEhmza5m/f0JhVVbU0EVRQCZXgc5qWpORUNHAFlWGzNboYuKLrOo0BLjNXGk/V+U4eF/vzopqbkKT0U/SHE5yExRRUUwrMbgieZ3AsH+z5QpUyzjiXNycvjXv/7V7b7vvfdet6/ffvvt3H777fpzm83Grbfeyq233mo5/oorruCKK66wfO3cc8/tt6jZH1i0aBH33HMPVVVVHHnkkSxcuJDp06cnHP/cc8/xhz/8gU2bNpGZmcl3vvMd7r33XpOYWrhwIY8++iiVlZXk5eVx/vnns2DBAjwez964JIEgabTwnT2JLxjm062NDM72mkJtYjEZ3AOZU4HRkO/eOyAnWbZUM/5T3Xbq2sCfrKjoKfypB49HW8S4zvA42VKnJoE3dQapbfNTmO7hof9tBuA/a6pMoqIhEv6UqOLUyu2NyAocbqjM1NPbEJYVFryxnk+3NpHpdXLv+WORJImQrCa7Gz0VmldCy7NIFGwX0j0V5uTr9pjnmifj5FEFSZcfFp6KfnLyyHyckYSj5s6gyKkQCAQCAy+++CLXXXcdv/nNb1i1ahXTp09n5syZCStqffTRR1x88cVcdtllrF27lpdeeokVK1Zw+eWX62Oee+45fv3rX3Pbbbexfv16nnzySV588UXmz5+/ty5LIEgah23Pm1pb6joIhmTTirsV5vCnPeOpiMuZIDanQklKVGg5FZpBm2z/hXDMuWKxas5nRPOIeJ02thrKtW6obsNnqOa0ubbNZOdVtXQhKwpNnQE+3lyvCy2bTZ17U0eQls6gOQm8h/fhvQ21fLq1CVDzJxoigiEYliM5FeqxMr1OslPVRe7GjgAKCvYEORWxnor0SG7wC5/t4Hevr+Ph/22mKxjSXy+IVJVKhj590xctWkR5eTkej4eJEyfy4Ycfdjv+/fffZ+LEiXg8HoYNG8Zjjz0WN6a5uZk5c+boXWUrKipYsmRJv867N/C4HOSlquqw1ReKaZgiZIVAIDi0uf/++7nsssu4/PLLqaioYOHChZSWlvLoo49ajv/kk08YOnQoc+fOpby8nOOOO44rr7ySzz//XB+zfPlypk2bxg9/+EOGDh3K6aefzoUXXmgaIxDsLzj2QoJ2MqVJwZzr2ZP3ZHdzF8s211Pb5mPZ5np2N3clPq7S/fOw4WRhOT78yfiaOjeFpo4A76yv0eeZbLSWyVPRQ06FFdouOxq76DIkh39T1cbOpmiI1rZISVuXw4bbYaMrKLOzqZMbXv6SH/35U76O9ICw28w9zLQ+E+q5up+LluytoX0GwUgIlO6pSHGSm6Ia//6QzLUvrKay0bohrRaepnkiSjJV725nMMy2hk5W72hmzc4W3ZORm7YHRUVvV522bt3KGWecwfTp01m1ahU33XQTc+fO5R//+Ic+JhAIcNppp7Ft2zZefvllNmzYwBNPPMGgQYP6fN69ifaG371kPW9+XaVv33NVFgQCgWD/JxAIsHLlSk4//XTT9tNPP51ly5ZZ7jN16lR27tzJkiVLUBSFmpoaXn75Zc4880x9zHHHHcfKlSv57LPPANiyZQtLliwxjREI9hcGqsFcdyTqtxBLT/0kjNS2+ekMhFmzo4XOQJh13eSExHo9jM9kObZ5XGL7SEtCDsoyL67YyQsrdvB///jS8hyJLjm2fG0siTwVWo6JJnjWVqm5MJoo3NrQzo7GeGGV6XUwolANOfumuo26NtWb8NXOZgCcNsl0/YFw1NvRk5m4ocbcg0MTNWFZISjLelJ5pseJy9CcuTMQ5p31NZbHjHoq1H2LLYoH1LT6aYuIDm3hPBl6/U3v7arTY489RllZGQsXLqSiooLLL7+cSy+9lHvvvVcf89RTT9HY2Mi//vUvpk2bxpAhQzjuuOMYN25cn8+7N8kxvOEP/+9b/bFI1BYIBIcy9fX1hMNhCgsLTdsLCwuprq623Gfq1Kk899xzzJ49G5fLpVfCeuihh/QxF1xwAb/97W857rjjcDqdDB8+nJNOOolf//rXCefi9/tpbW01/RMI9gbGMJQ9FcGQrDPEFP7Uw8pnf8KjjNcZjHGJKDFFbYxoFZ7CssJn2xoB+GRLI4GwHCcQjKLi822NeiJzj9WfLEregkFsRP7bXKuGPh0eEQxtXaG4ZHKANLeTUZE8iQ831Ue3e9RwJLtNMs3DZ/B+9PQWb4wkWx9ZkgHA9oiokRWF5k7V6LdLamNmgJGGfI21u1tZsa2R6phmeKGwgi8U1t9rq4pkdW0+/EFV/OwxT0VfVp2WL18eN37GjBl8/vnnBIPqG/Lqq68yZcoU5syZQ2FhIaNHj+buu+8mHFFzfTkv7L2bSMIyZ0JTCAQCQbfN/2JZt24dc+fO5dZbb2XlypW8+eabbN26lauuukof895773HXXXexaNEivvjiC1555RVee+01fvvb3yacw4IFC8jMzNT/lZaWDszFCQQ9YAx/CiYwaPuL8e+pu9yDnhrDGemNAIoLfzI8jhUD3XoqIvkMIVmhODNadOHmf37FN9VmG854zc2dQb2Zm0lUyPHXESty9O16R2kVre9EWY7ag8kXkvUcC+NissMuMbJINfqrDAZ8ZyCsH8/oFTImnHe3+BwIyWyuVa/p2GE5AGxvUM+/dF0N176wGoDsVJf+Xvzs+HKuPH5YZGwnu5u7WF/Vqnf2/nxbI09/vJXmTtWb4rRL5KZFryU1Ummrts1PU0S05KXtIU9FX1adqqurLceHQiHq61VFt2XLFl5++WXC4TBLlizh5ptv5r777uOuu+7q83lh791EclOjKs5tcD8JTSE4mJH3dDkTwR5nT3+GeXl52O32uN/p2trauN9zjQULFjBt2jRuuOEGxo4dy4wZM1i0aBFPPfUUVVVqeOktt9zCRRddxOWXX86YMWP47ne/y913382CBQsSXtP8+fNpaWnR/+3YsWNgL1YgSIJgksnGRjr8IVZVNumGoBVGjd5dIrLSQ2iQkd6EcMcax+YSs7GiQkloIGmiKxxWTNfb2BnkuU/M4e6JnDNWIqa71/V5GjpVQ7SaU3GmR/cE7WhSPQU/ixjuoIY3leV4yfSaC6pq4UOKYvYK+Q3J3oneh5auIG98XUVYAY/DxrjB2YBatrbDH+Lx96NRMVopWIBMr4ujh+ZQmK7apde+sJpPtzSwpa6DrfUdXPP8Kp76eBvvrq8FIN3jJNUdnffEIep5djd30RHovaeiTyVle7PqlGi8cbssyxQUFPD4449jt9uZOHEiu3fv5p577jGVYezteefPn8+8efP0562trXtEWNwwYyS7m7tYtaMZf0jGFwrjcdhFSVnBQYnL5cJms7F7927y8/NxuVzd/h0K9j8URSEQCFBXV4fNZsPlSn4lqje4XC4mTpzI0qVL+e53v6tvX7p0Keecc47lPp2dnTgc5luT3W7X562NscVU1LHb7d2GVbjdbtzu5G+OAsFAYfxG9kVUbKhpo7E9QEN7gFOPsBbjRqO1O1Fh9lR0f96eRIeR2D87oxcktsO22vwuQfhTZBW/pSuoJ0n/8JhS/vbZDnY1dxEMyTgji7dqHolFeFPMscOyuRKScT5b6zvYUN3KhCHZcR4Mre9EVopqeBsb3Q3K8nDLWRW8tbaGEw/Px26zcWpFIf/4Ypc+Rst30DpUa5g9Fdas2dnM2kgOS0mWl8wUJ0NzU9jW0MmGmjZ2NkVzO0ZFvCRGRhanUxPpmP3kx1sZnJOCTYp6UpZ/2wBAhteph04BHFWWzQeb6vX33mmXyPA4aAuSFL0SFX1ZdSoqKrIc73A49JrjxcXFOJ1O/cYBUFFRQXV1NYFAoE/nhb13Exmal8oNM0Zy6eIV+EIyTZ0BijO8A1quTSDYX7DZbJSXl1NVVcXu3bv39XQE/SAlJYWysrI4A30gmTdvHhdddBGTJk1iypQpPP7441RWVurhTPPnz2fXrl0888wzAMyaNYsrrriCRx99lBkzZlBVVcV1113HMcccQ0lJiT7m/vvv56ijjmLy5Mls3ryZW265hbPPPtt0HxEI9geMtkCyXaGNJGPch01GfOJzGMf1ZKP0xobpbmxsM7xuHBX6+6MZvykuO7fNOpK/f76TkKywo6mTYZE+HInWsrT3a0dTJ2+vrcYfCnPciHwyvU7TfN78uoqXIyJge2MnQ3JT6cwO62JL81RkeJ2ke6KiwmW34XHaOawgncuOG8b/NtQSDiucMDLfJCo0T4WsRN/35Vvq2VrXwQXHlGGTpISLz/6gzK6IcBic7cVpk5gwJJttDZ0s+aqKoKzgsEkcPyKPn0wdGhdWd9aYEhRFzfEIhhX++N+Nel4GQFdE2GR6HHpJWYARhWmkuuxRL0Wqu1eLhr0SFX1ZdZoyZQr/+c9/TNvefvttJk2ahNOpfsDTpk3jb3/7G7Is6ze3jRs3UlxcrK+g9fa8extJkshMceJr9dPcGaQ4wyvCnwQHLS6Xi7KyMkKhkJ77JDiwsNvtOByOPe5lmj17Ng0NDdx5551UVVUxevRolixZwpAhQwCoqqoyVfG75JJLaGtr4+GHH+aXv/wlWVlZnHzyyfz+97/Xx9x8881IksTNN9/Mrl27yM/PZ9asWXrIrECwP2E0HPuSU5HqctASiW+PXXXXMNrt3Z3D6NFItrRqMsQONXfujpmDwaO4q7mLpz/eynePGsSRJZm6J6eqRTWo89JcuJ12CjM87GruYlNtmy4qEk0/LKuJyHf8Zx2gejTcDrvu5QnJCtsbO3hlVVQANHUE6AqE2VjThsMuEZJlGjujjeWMq/mZKQ4kJD38ymGTCIcVUpwObjqjgpXbG3lrbY3eTE4hWv3qyY+2ATCiMJ2jh+YkzKmw2yR2RcrHDsry4nLYmDQkh1e+2KWXsj12WC4/nDyE0YMyWVXZbNo/J9XFT6YMZda4Eu59awO1bX7e21AfexoyvU5SnA5uPesInHYJj8NOfoaHjkjuSG4v8imgD+FPvV11uuqqq3j44YeZN28eV1xxBcuXL+fJJ5/k+eef14/585//nIceeohrr72Wa665hk2bNnH33Xczd+7cpM+7r7HbJLK8Lmpa/XrDENGnQnAwI0kSTqdTXxwQCBJx9dVXc/XVV1u+tnjx4rht11xzDddcc03C4zkcDm677TZuu+22gZqi4BAnEJLZ1tBBSZbXZEAOBEZbIJBkV2gjWoNdULseZ6bE/+b2VEY1OhfrfaxIxlPhC4bZUN3K4o+3Mq40iyGRpGZTh+2Y4xgTtV9cUcm2hk4eeGcTf754kkFUqJ6KvDQ1WbskSxUVm2vbmXGkdi3W8wvLCv/5MupF10KFgmEZp91GWFZ48+tqZAV9VV5bmVePi+6VsEmQ7naYvhNZXtXQ1tZj7IaFmWF5qWR4HLy1toZWX0gVFBFPhfE9qWk1V2SKxWGX2NWsioeSbC9ZKS4mDc02jZkwJCsyj8QLQzkpLl79xTR+88+v+O83dYAq1LTQrhNH5gNw3Ig8QmGF3c1dFGe42RYRFb39W+j1X05vV53Ky8tZsmQJ119/PY888gglJSU8+OCDnHfeefqY0tJS3n77ba6//nrGjh3LoEGDuPbaa7nxxhuTPu++xm5T4+4APblIaAqBQCAQCPZ/vqlupbbVz47GTk6pSBxW3Rf6m1Nh1Aht/qClqDAKie7OYS4pm/x5E7FyexP3L93Ayu3NfLy5gd+fNxaI9VTE5zhoBraxuVxtm0/vCl0bMboLI92cB2V5WUETmwx9GxLZWLKisNEwrjFikzV1BihI9xAMy9S2qqFNp1QU8uqXu01drpd9W8+/VqtejEyvE0mSSPMYPRXqHDVfhS3Gc6SNDckK/pAqZGRZoSsYFS56ZagE19DuC+n9LgZneclNdenhWwCjSzIYMygTAK+z+5DPgnQPf/7J0Tz2/rfsaOyioiSdT7c0cvyIfL2KlU2SkCR1MjNGF7F8i1rON8viu9YdfZLjvV11OuGEE/jiiy+6PeaUKVP45JNP+nzefY0kSQZRoSXn7MsZCQQCgUAgSIaWrmilnoHGaFT3xVNhXOE2JguD6imoavGZkn+7T9S29iAoisKOxi4yU5y68ZqMp6LDH2Ll9mYAGjqi1ZqMe8Z321b0bZ2GykUrtjZSnqeGNtW2qaIiP1LFaHB2CgBbIivo3c0vJCvUR5KUQX3P2vwhGtpVUREIybrQGJztjVxH1OB/4sOt+uPsFNXoTndHjWvt/dG0RGzHdLfDhsMmEZIV2n0hUl0OZEXRw6EAvVyr1RUEw7Ie+pTucZDuUT8Tm01izknD+WhTPTedUaF/FzxOG5OGZmO3SThsNtr9IdbubtET0jXRM3N0MZtrVbF17vhBuBw2/ftot0mEZXXc4KwUbvzOSN5dX8tPp5VbvseJGFgf3yGMXZL0L1qz9uNk+Lr4Q2HW7m5lcJaXggyP5TEEAoFAIBDsfaSEBUr7j9Fw7M7gT4TRo2BcUQfVU9AVMOe1hbtxQSTqU9HSFWRjTRuZKU6OHqr2ROgphLszENIrFIGaVK2hJBAv2hwURU2YrmuPGv9f727he5G510Q8CUURT8XgHNX41/pEGIVJLB1+VURoc+oMhNWwnkwPsqzQGQjrBnlpRKyEZIVAWI5L/s6OrOQbPRVZmscgMjbWUyEhkeF10tgRoM0XIj9drf7UZhBQdRHRZPUeB8PRJO1BWV5S3Q79HOeMH8RRpdmkGRLH1UXtaO6D12W3FFzGjts2myp+dFEhSaZrH1GQzo+OHUKGp3eeij3fO/4QwW6LfqjNXfHhT5tq2mlsD7BmZ8u+mJ5AIBAIBAcVLZ1BPUymv+zJegWmkKMElrAsK+xq7ooTCGBeoIwtz2o1PjZROywr7GzqxBcMm4xYqzK02vEVRek2PKrdH+KL7c2mhnTG4xtnEHvNYVkmEA5T1+Y3iZzaNr8aGqUofBtZUS+PJGXnR3oltPlC7GrqTBz6JCt6B+kUl53DIvvvbu5CjggHLUTdZbeZGrt1BUJ6WJRGRqT3RLoxpyIm/Mlu8eXRela0+0Nqn4oYT0VdxJNidRn+oMyXO5sBGD0oU8+dgKh3RAt3S1S4z+qzM/ZRczvspoR/uz1eVvflT0KIigHCJkm6em21cGv5gqJCjkAgEAgEA8WKbY2s2dmi9wPoD3u0BprBGEi0+r+ruYv1u1tZviW+Qo9xl2TKy8YKj20NHXxT1caKbY0mI97UeVoXA5qo6P4c31S14guGae2KvveyYujNYDBqY6/59lfXcu4jy/RQptyIN6ClK4QvKPNtXTstvhAOm8ToQWoZ1HSPQ09Y/2hTfcLSvI2dftZXqV2o89PcFGepkSG7m7sIK2qOQ0MkSTknVc2X0DwsnYEwNa1dpuPVR/IaTDkVsYnaFtW4tLCp1khZ2WDYLCo6AmG1aZ3F+/z6miq+qGzGJsFPpgzF7Yh6gLSkbE0EJkrS1qo2GcvFukyiwmbaN9ZTAVofkN4hwp8GCLtN0j+8Vl989ac+eDwFAoFAIBD0QFcg3P+KTXvUUxF9nMhY13I6rFaYTSVpkzAmYpu4aSvz/qAcIySMJzH/HxuyFIsWNtMRI+hafSHSPU6Td0U7j90m0RUMs2JbEwrw6Ra1Advw/DQ6/c10hWTq2nxsa1DFRnleKsUZ3si+NrJSnNS1BWjoCFgu1CqKwncXLWNHoyoM8tNdDMpS99/d3IWswPrdrXy+XU1C1jpFayFSHYEQVS1mT4VWHcnoqdBC3bWvjJXxnRMRFU2RPJOwrNDuN3eQq2v3k27owt3mC1LT4uf3b34DwPcnlTJmcKZpH+1c2ueY6Gt7REkGu5t9FGdGw+2NosLjtJsS+lWPR0wYVx/+JoSoGCBsktruHNRqBiFZTrgiIBAIBAKBoO8YF+0GInRpz+ZU9LzAaDT4ZFkxxemby8X2nOgda2+kuh00dagGrbERnfG4MZoiiXKz6v/+GI9Ba1eQQVleS++K3SZR0+rTz7Eu4lEoyvRQ1eJmR1MXde1+1uxQw8SPLs/GYbfp+2aluKhrC9DcFbRMeK9vD+iCAiA/3WMQFT58wTBX/fVzveJUdiSMSfNU+IJhqiP9MWYcWcj3Jw6mIN1DbZsfjyFfRItK0Vb6Hfb4787E8hze+aaWf63ezQcb6xhVnI5NMgcH7W7uojwvVX+PPt3SyMP/20ybP8TQ3BQuOja+uqk9RlQk8ia4HXb92Bouu9lTYfyeOGw2Yh0uffmbEOFPA4TNJpHqsusfiuoCNFdWEAgEAoFA0H+MBtGebd/Yf8yeCmtbwGgc+kLmVXhTfoIcX6JVQWFDTZseEhRbUtYRCbxXUPjT+1v434ZawPweatNSFNXzE1tlKhbNk9ESySHVKiDpkRrGOUfGOmwSu1vic2AGZ3v1Kk/LNjfonoQpw/L0MTYp2h+iuTOAPySzvbGDRe9tZkdTJ4uXbeNVQ28KUMOqiiIr9W3+EJ9822AqYaslYadGvBCdAZnqSE5FeW4qJ48qxB4RDHmpLgZnexmen0qKWxUY2kdmNMYrSjIYlp/KjyaX6dsaO4Ms+7aRjzabQ9vWV7Xq73tjR4C6Nj+rdzQDcOm0cn1eRrRz9iQqrJAkSRevHqfdtK8a/iQ8FfsN2geS6lYz8tu6QmalLkSFQCAQCAQDwkDfU/dkonYyodBGz0BnIEyKy2H5Gqjx9C6bpIuL/66v5YUVOzilooALjy6Ly6nQTPyt9R28/lUVAMcMzcFhdxpGRHMqPt4cn9eRaL5ap++SLA+VjV20al2kI6+v3N7E3Oe/4OihOVxwdBlVzV1xxyrNTmFzmpqY/fn2Jv1404YbRUU0b7WlK0iHP8SfP9xKVYuPDdVtpuZ1GoUZHtwOu97sbUnk2jUyPWZPxetrdutN94qyPEhS1Gi322w8eMFRdPhDevne6Fcm+qgow6PnWJw1tpjX1pjPCXD00GxWbGtibVWr/r43dgT4cLPanO6I4nRKIl20Y9E8WCFdVMQN6RatjKzbaTN95222eHHel78J4akYILQvnlZ+q80filHqhsciFEogEAgEgj5jjAIaiDvqnvR2KEl4KoxCoLnTHN4Tu4sWwqR5Jl5YsQNQxYWCYlnCFcxdnNfsakZW1EpLbb6gPiZZraZVUtJW/rUwo9Yus6finfU17Gr28a/Vu3n20+1UtZhFhcehVmDSPBUAZTlebjqjQvcIgBb+pPUCC/IfgwCIFRRDclKYPamUi6cOpSjTw5HFarJ3TZs5X6IwUt4/NSLgdjX7kBWYXJ7DoCwvUkzyssdp18OxIBr+ZDTsjY/nn1HB5PIc5p8xikFZ0dyG8aVZuOw2WrtC7Gzsos0XpLKxg2URMXf84WoeR4Ynft0/Wv0p8v3o5Rd3WH4qJVle8lLdZk+FLT5Ruy/hT8JTMUBoLjItWazNF0zYUVJbZRAIBAKBQNB7wqbV//7LikRVdAYCY05FopkaQ5G21XdQ3eJj2mG5SJIUJ0T08q+yQnVMtaLqFh8lEQNfQ3t/qpqjouLzbU0cW57Lxpp2djR24ol0ZY7tXbGhpo0JZVl6CBWo9oxiqPRkk6IGepsvyI7GTrqCYWaNK6HNF01O/nBTvamXBUBJxHjPNZR2PWtsMZ7YkqcGUVHd4uPZTyot3kWVc44q4crjh5PqdpA5KJNRxRm8v0k12L1OG9eddjiyrHBYgVpu1muY0+BsL5dPL9fPbTS8XQ5z3oH2kvG7Y3w8KMvLz08cTiiscFhBOrsi7392iouRRWl8tauVf67axfMrKhk9KJPmSCWtsYOzcDlslsUHtPlomqK3FZoK0j0UpHtM84eIqIgREX0xU4WoGCA0Rall8rf6Qvofpywrph8MkbQtEAgEAkHfscoH6A8DpSn8oTBOm82caG3wqiTuAm3Og/AFw4RkBaddihMi4bDWU0JmxbYm02tf726lONNrSvbWDm3MZ/hyZws3vLyG288+kjS3Q6+mZJzePW99Q3Wrn+8eNYgzxxTHXYMmGNI8DjL00KQQd7y2DoCjyrJMJWdbuoJ6lSsNzaAfmpuK22GjIN3N+NJswNz/wSZJepnW7Y2dABSku3Da7Xr3aY3Y0KHDC9P1x+dPKmV4XhoTh2SzMhJqZQw1G56fikTUwDYa1m6HzSR0pJj/rVCNfoXh+am8v1ENb5o4JAuv087Xu9axZpealL6xRg3/SnM7cNltZKe4LIXuQJR9tcKqpGxfhLYIfxog0j1OUt0OvexYuy+aUxFbT1nkVwgEAoFA0HeM3v8B8VQkOHZv6PCH+HBjPSsro4Z+rJch0VStFhu1hOv48Cd1QyAss71BNbBzIknHa3erRqpVw73YfIbmriDf1rWbthn305KWP9vaaJ5rZIzWdyHd7dRDv7/eFW3wu7vZpyduG8lJjeZyFETCnjI8Th6YPZ4bZ47SDWVbAk+FxnkTBjMuUnLVGCqUm+oyCZLpI/LI8DiYXJ7D8SPUPA2j6Eg1eCrK81TvRfQLET2O22EWi9Hwp8TGt/aa5hUBGF6QzsVTh/LTaUPjxmvXmOG1XvOPbbTXH01h/F5Zhz/1HuGpGECOKssiT+/6GNRdnrGVGMJxSVQCgUAgEAiSxRz+NPDHtvXBpKqO5CxoycsQP7fEngqtmVnU2NM6Y8d3pNY8FYremfmEEXn8c/VuttR16HkVmoGnKGrvitrI2HvOH8ufP9zKhpo2GiN9FDCMjZ+bbDmmI6CKijS3Q8+JMO7e1BnQcywGZ3vZ2aSKmpmji8lLc/OfL3cza1yJPj7T69Q7hMd2irZLkl79CWDq8FwmDMlmVHEGNW1+Th6Vjyyrc5GQTMZ/QYaHhy48ikBY1j0QxjKwXpOoUMuwRntQROfgcthMRr3uqejmq6Ltn5/uZtrwXLJSnOSmulAUmDo8j4lDspnzt1X6+MFZXvLS3XqOSvzx+h+ipGH8XkkW1Z9sfTi4EBUDiMdpZ2xpFi+t3Emr0VMRU0859g9UIBAIBAJB8pg8FQOsKvrq+bAyweI8FQn21YTCxCHZfFPdRrsvRCjGU+GwS1S3+Gho91OU6SEYlqltU4XMUWXZ/GdNFZ2BMLVt/riQq9pWPwrgdahN5AZne9lQ00Z9uzl52YrYalLaXNt9qgBI9zooyVJ7QhhDkT7Z0kBTRGAdPSRbFxUnjMxnSE4qYwZlmo5rqkZksSLvctiYfXQpHf4QZ48rQUKiKMPDP34+lU+3NOhlcK06XNttNiTDdRhzRIzXpzWLs/JCqDkVRk9FZK7dGd9a3gUSP51WzpThuZFjq+d0O+ykexz63MvzUxlfmpXwcPHvy8DlAhmP1NfDClExwGjt5tsNLr9g7B+kCH8SCAQCwSGCoigDnghtvI8OxC01thdEr/dPcI2xeidh9afIwGXfNhCSZVKcDt120PZp7Qpy0z+/YnB2Ch/830lUt/gIhhVsEhRkuCnLSWFLfQdb6zrM7w/oTd0Ks7xIRBOjYz0VVhgb7ilKtLqUlqidHvEOTB+Rp1eiAjWHorFDFS1jBmdy5KBMwoqCy26zFG6xORSm1yKG+2kVhabtmkFvNOytjPw4z4dhzPiyLA7/No2K4gz9vNqrxnm67DbTcTSvR1GGh52NnXoImum8Ca7D+F3JTXXpoqIow9pDoZ8z5jr681cV+xFYCabeInIqBhjtS9VmSNSOC38SidoCgUAgOAT4prqVjzbXx90H+0t4gHMq+pOjUdvm472NdTRYrPorMb6JxDkVMjsaO7n6uS+Y+/xqQrIczamIjPl6VyuyApWNnexq7mJHYzSfwmGzUZ6vhu5sre+Ie3+0MKmCSIi2Zqs0tFuLCuO8jQujihIpJxuSWb5FraikVSmaMixX7yUB0NYVosOvejNSXA5mjStheF4aimL9PsTmUBix8j5ANPzH+HJs3oH6uuHYMR2wU10O3r7+BH40eYi+TRseMryPDru1p8Juk5g8LJcRhoRwq/NaPQdMYqQo0x33em+Plyyx303jofq6CCBExQCTF1H/bf5o+JM/pjtmfGMagUAgEAgOPnY2duEPyuy2aHrWH5KpqNSr4xkO0dtogjU7WgiHFZoNuRSad6GlK6h2TtaayylWydsKsgxb6juix9zZohu02vXVGUTLp1sa2NGkigotn2FYJB/gm+o2Ov3RqkuKouj7amMLI2VFNU9CLH5D2Lap0hbqe/X4h1uoa1MFSW5EqKS6HfzxwqM4a6xaKaqpK6Bfg9dl17tuy4qivx956W4cdokjB2VYlmvVSGQ8a8av0QiO9UqAWZRo8xhXmoXDLjG2NDPunNr5YsVwInGTiNjhVteYmxoVElpp3uSP1w9REfM1lxI87g1CVAwwhRkebJLaEXNjTRuA6YcGBuYHUCAQCASCQ5WBTtQ2GvrKADhVwrLClrp2rvnbKu5bupF31tcazmUeqxneWn4EwIeb6uKqP21viIqOT7Y0sDvS90CroHR4YToOu8Su5i5+/txK3fsiK1Af8UjkRwTA0IgAae4KWeZ5dvqji6H+kKzbLYqilsjfGRE0Z44tZvKwHH1shsehF6ypNTTbS3Ha9TwGo6ciJ8XFiSMLKM70moRBrLchkS0fTahOHDoVu007dn66mxNHFkT7NlgcN3YRuKfzxM0vZoipJK0mKtKMnoqeRMXAVX+Ks0WFp2L/I93j5PQjiwB4ccUOZFnRS6rlRL44IRH+JBAIBIJDiL505+2OgQ5/Mh5hIPIew4rClroOPo2UY33p82iuQaJqTlWGPhJf72qlI+Jt0MZvNXgy3vi6mnfW1wBRUZGd4uKmmRXYJFi9o0Wv9iTLCvWRx7npqh1SmuPFGQkDauqIL/vaGTR4OkCfi4IqLLRysr86fSROg2vAJkl6OFRtpCSt12lDkiRTxSVNMJkasJm8DYm/L1qjPu18YBYEVt4EU9frBMc2GdKRh8PyU7HZYEhuStyxk7G7Yz0opkZ5kZPkGsKfinvKqYgt+9qPP6vhkTK32rUZ/0b7WlVKiIo9wKXTypGAz7c3Mf+VNYRlBZfDptdRFjkVAoFAIDiUGOiG1UbDfCCc/1Z9HbbWd7Bye2Of7tmybI5Zl5Vo5ccvd7bwTXWr/pq20GgMEVOIGuUK0OoL6pWUQM3b1J6X5aTq28eVZum2hpZHEZIV6jvM4U+ZXpcey19vEQKllXbVaI2ICEVRG/P5IuFR2TG9I+w2SW8CrI3xRprLOexSXK5CoopPsSvyRqHncdrixvXGU+FIKCoMjyMGdrrHyQmHF+j5Et2FaFnR7bxiPBVepz1hf4pEx+hP+FOGx8lJoxJcWx8XAUT1pz3AkNwUzp84mJdW7uTFz3dSkuXlpFEF2CNqXuRUCAQCgUDQd4yG/kAs1BkPoYUNfVurNobb1dRFWWQ1N+n5KYreo0Fja30HIwrSaeoI0NQBIwvTkSSJcFhNfNZClFLddjr8Yb3vhaIobIuEPhWmu7lhxkj84TBb6jrIS3VzeFG6LqzsNokMr5PmrpBeLrah3U9YBrukhhuB2lwtL81NTauf6hYfFUUZprl2xomKIIPwIisKTZ3qPCXQm95pSBJ6E2CNlEgfCJskYbdJhMIKwZDmqbBe+Y+1+7UeFV6XHafdFrePyeNhWVI2cRK4fqyY67Aab7PwNHRHdx4U7Vlpdgpnjilm0tDsHsOO4sPC+qfWzZ4X4anYL5GAGUcWMf2wXEBtv7782wY9UUrkVAgEAoHgYMeYpzDQnooBD3/qJkcjEA7TW4IhmZpWswfgkf9t5vNt0e7U2ilDskxVq+qlyE5x6ga+lmMhywqvfVkFQEVJBiVZXk44vIApw/KYPDzXXDVJUkUFRD0VNZH/8zM8uhHqstsYWaSuUK/Z2Rw3/85AyPRcK3kK0BgRFekeB7aYTswSEukea1FhlyQ9ryIQCX8yhSVJiQ1/l8PGCSPzmTIs1yQqrDwVPVZ/SlhJyigYrLH1MvzJeEzjvNX9o0nm1582gp9OK+/xeAMZ/hR37IRPkkeIij1B5MMoz1fj1V7/qooFb3zDBX9azpc7m0VOhUAgEAgOeoy3ugHvUzHA91GjLonNqQiEen+uQFjWPQ1eh40sr5N2f5j/rKnSx2hiKKwoVEWSrg8rSKM4S03WrWsLIMsKr365my31HXgcNmaNLcYflnUjP8Pj4LCIrVGS5cVmi3oP6tr9KIrCF9ub1NcNScCSJHHyyAIA1le14TNUqVy9o5l3v6kzXU9bxOuiKNDcETl3JFTHXIpUzXkw2s8pkfAnmy2aV6FFbBhX+xN5BDScdhs2m4TLEe+pMFZ8sqr+ZNyW0FORRN6FVUft7jAe0xlTytbsGUnu70OSJPO1DmTzuwEIfxKiYg+gfcjluamm7bIC//lyt8ipEAgEAsEBzcaaNlP3ZCv2pFfeaPg3dQb4elcLvmDvPQoappyKmHu0VXWknvAHZWoiomLK8Fx+f/5YAHY1d9HSFYicUx0blhW9POzIonS9q3NDu4/bXv2aVyNeip+fOJxMr4twWKEp0rQuze2kNCeFY4fnUlGcjk2SyDR4Kv6ybBtvfF0NwOTyaJUmgJmjixiU5SEkK6zd1QKoeSAP/2+zKSkc0D0pCgrNkflnRkKSjAaoJEW8FYYQKK/TrhvCsfkMiUKeujOWXT15KqzCn5LwVJiuI8G5Tc3vkkrUjj522GI9FYbj9kIcGAWIMb+kv4hE7f0U7bOwisHc2dRFINT3Hz6BQCAQCPYljR0BKhs6Wb+7tdtxe3IBzWj4+4My1S0+1lV1P59EKIpi8lTEiqFgH/Ig/aGwXn2pIMNDptdJeZ5qE3wded+084TCit7IrqI4g5Isddz7G+v56yeVAFw8ZQhXnzgcd8SI1HIe0iKhRmluB1IkZ0ELf6pv9/Pfb9RStjOOLOTy6cNMc7TbbUwdngdEe2R0xQizwkhi9wcb69lS34GioOdUaB6RRAnOGiluu24Ixxr05mZy5kpJiXA64o3fngSJ8dhJJWonMKrNoVU9W97dhT+Zzt3jkayPaayE1V8kk2ASidr7DdqHobn8NFKcNjqDMjuauphi2N7qC7J+dyuHFaTpTWQEAoFAINgfCSXZHdtom8c2fOsvVoIltmJRssROTY5pUJfs9Rrxh6IN/wozPEjAmEGZbK3vZO2uVqYNz9PPq3oq1LEVxRk0RhK2tVDpUyoKOH5EPnabjRSXHX8wko9gg1SX2ai0GXIqqpq72BhJNj96aA6pbgeTh+WYjFutL0JrlxrSZOxPAaqXZVdzFyu2NfHW2ip+OLmMlkjVKWP3bA3NFp1cnsOOpl3qNiTdEI41rM1eh+h2q7wIDXOitjau+9Apc6K2tXGvh1KF/ZSsfBD++SrIYZDs6os2O1nYODakgGTH6XCA0wE2e2SMzfBYHV8WVMgPKiiSHa/bBR6nPmZke5AwEopkJ8PrBq/L4ji2SC3a6PahTT5Csrpfek1agjnYLI7T3XY7jpBMdk0HSHZSPU6wZ0fHdMWXHbZCiIo9gPbVVVAoz0tla30H500YzLd17aze0czWOrNbcc0O1W27qrKZU48o3PsTFggEAoEgWZJcxBzosq9GrHpJ9DW63KpvhFGz9CUPsq7NrydqD81LxWaTGF2SyatfVrGlvt103ro2v54jMaoonS0GG8Fhg7PHlgCqnep1OmhCNfBSXY64FWW7TdJLyq7Z1UIwrGC3waBsr1qZKaZak1YNqs2vHrPDb07QTnE5mDo8jxXbmqhpVXM0miMGZmaK5qmQ0Dp9aLb76UcW8da6Gtp8IcpyvPpnE+upMHodElUiisWq+lNP1Z2SESwSEpl1n3PE578htW2r5Rg7kJZwZvGkRv5ZUdCL4xgZ0sf9eiIFmJjoRUd+UscQomIPIEVVBb84+TA+3dLAjd8Zxb1vb1BFReQHRSO2DbxAIBAIBAc6yYiK5s4ALV1BynJSehVyYZnm0EdVETs1WVFMc+/uHp0oxOvrSI5CYYabdLcDCTUMCqC5M4hC9BybI96EwVleUlwOSrKiDdAmDlE9DJKkGtpeg2ciViCAajBrHgQtbGtQlhenzWa5gp8d6VWhNbNrj6n65HXZyIv0UWho8yMbSuVmRUSF2YaP5jgs+N4Y1u1uZezgTP1aY0OPrCo5QeK8B7DOqTCO7rH6k93i2P42ij6+mZy1fwEg6M3Hecb/g9zhkaYjMihhOn1+1u9uASXMoEwXReluUMKqRyMyJvpYpqa5g/o2H5ISpjjDSbbXob4mh9la10YwFERSZArSXWS6JdO5zMeMbt/Z2A6yjKSEGZTpTjA+HGldnvz2sCzT5Q+AIuOQZDx2InOVVa8F3yb8TDSEqNgDaD+MCmp5uNOPKCLV7eDIErVM3LaGzpgd9vIEBQKBQCAYAGRZSVgpx2j4K3Gmu8rn29TKRE67zWRM90RYUfhwUx3VrT7OmzAYmyT1uWJNrKciVlSEwgqKoliKnkSiYn0kv2NEpGuxJEl65+SQrHak1nb9tq4NiHY4NjaUO3poTmR/9XmKSVTEm3DG6k8aWnM8q3dHm5PW3K4jJvzJF5TJTVXDsrtCMne9vp7/bVArQ2WluIjF+BZ5nXYmlGUDmPpoGMc6EpRo7S5R2FhFKZpT0X0+RmzZXROb3oHXriOnRe16vqv8fJqm3crowyx8AoEQTaEGAPIK0yA3kR9Cpb2unaqI5yl3cCZkRCtwVX1br4ebpZZkkJnk9/+bdTX640EDGN0SCIT5ZHM9ADlpLv2zA6C1Fa7N7PEYQlTsAbTva1hW9B9Vu01izKAsAHY0dhIOy9i7SdoRCAQCgWB/xGi8y4qCLYExb+5S3f0xW7qCvRIVsqzwl+XbASjLTWXy0BzL5NrOgNp5uiTTk9ATEpdTIcdvC4Rl3I74pNhEomJTrSoUDstXe0FIEjgdNtI9jkg37ICet1EdCZManO2JjJX4waTBfFPVxndGF1Hd4tPnbvRUpLnjTTi7TSLFbZ7nkEjRGCtPhdZVu81nHf50WEEaLsO8//75Tv217IioSNTfQZKi76P2XTB6Jpx2W0zDte7zIjQcdhs2m/o56Z4KY3hTstWfOhvhrZvgy+cBCKSX8dWEO2kqnEqBxzq/tbfN77rr5C31kAeyt+lrNSojQlTsAbSPwliGzi5JjCxKwyZBRyDMjuYuhuYmXj0QCAQCgWB/pzutYA5/6l5V9DYM2Nic7eudLUwemmNpCG2obqOhPYDHYUtYCKUnTwVAIJRAVETGVrf4KM7yICFR3+7XqykdVqje522SmqycneKkzReiuSuoC616vUpUVFRdMrVcT/SGqJ3gddp14y/NylMR8djkp7uoawuQneJk8jDV22HlUdLek2BYwR8K0xkRFeNLs5g1rpghuSnIMuSluU0N8ACyUxymuUFsh+xoroWVpyI2advUp6KHmqZOuw2/LBv6VHTjiSA2pwJY+09YcgN01KlXcOzVVI6+lqZIAbFEgqG7sCwrjKOdDvP1JuuZSURv59Ib+npksVS+B9D+qDRNYbOpX3iPMxorqcVbCgQCwcHOokWLKC8vx+PxMHHiRD788MNuxz/33HOMGzeOlJQUiouL+elPf0pDQ4NpTHNzM3PmzKG4uBiPx0NFRQVLlizZk5chsKC7XhTGRfyePBXBsIyiKNS0+vD3UHY9LCt6tSKADdWtKCiWngotyTrQjWiJvYSvd7XoeQ4agVD8/p2BEDWtPt7bUMetr67lT+9v4b6lG/j1K18RlqGiOJ2iiFCQUA3HrEhvh6aOoP7e1XdEREV6VPRoDd78kfNqdoXTbmP0oEzGDMq0LFGqCaufTi1n9tGl3PXdMaQ4HQlLtKa67foKepsvRHskHKcww82QnFQGZaVwWEEa+enxoU45kbAoowVq/AisRJ5xtd7lML9u8jb0sFKuXbv2vhjtayuPlCY6XF21pP/7EnjpElVQ5I+Cy5bCd+4GV6rhGNbnNV5TMn1YTCVlY/tUJHySHN2VqO0LA+Gp6NOMenuDeP/995k4cSIej4dhw4bx2GOPmV5fvHgxkiTF/fP5fPqY22+/Pe71oqKivkx/jxP7URg/HM0Nua6H+t4CgUBwMPDiiy9y3XXX8Zvf/IZVq1Yxffp0Zs6cSWVlpeX4jz76iIsvvpjLLruMtWvX8tJLL7FixQouv/xyfUwgEOC0005j27ZtvPzyy2zYsIEnnniCQYMG7a3LOqRRTB6IxOPMRlf3BlggpLCtoZOvdrawZmf3i24hWabFFy1x2dgZpLKx09Iu0/pZdNczwzjP+nY/v/nnV1z515WmPBArUbJscwNb6zpYtUPNC/l8exPrq9r0178/cbD+2CZJSBJkpar5Dk2dAf28DZESsoXp0Xh7LRlZE1hGo7kww6MnfceirV4fXpjOaRWF+nEShX4Zq0W1+kJ0RDxAWmhVhtfB0LxUCg1elCE5KZw4Mp+hEXsmtvmd/rib+UG8UWwOf7Kcro47Iro0kZKoM7d+PKBky8tMeXMmzk1LwOaAE26EKz+A0qPj5p4I47GTqQlm/A45YjtqJxnulQgtdG2gMPcM6dsxeh3+pN0gFi1axLRp0/jTn/7EzJkzWbduHWVlZXHjt27dyhlnnMEVV1zBs88+y8cff8zVV19Nfn4+5513nj4uIyODDRs2mPb1eMx/NEceeSTvvPOO/txuH7imHwNJ7Idh/BKW56Xy8eYGPYlLHR91EQoEAsHBxP33389ll12mi4KFCxfy1ltv8eijj7JgwYK48Z988glDhw5l7ty5AJSXl3PllVfyhz/8QR/z1FNP0djYyLJly3A6VSNtyJA9VWhREIvxbtW9pyL5krLf1rVz39INzIqUT61p9VGYwHDu8Idpi6mb//WuVsYNzrKYQ+T/bqKrjHpjS30HYQWqWny0dgX1rtGd3fTAMK5A2yQ4pjyHI0sy9ARpQK/epOUhNHcGUBS1B0ZzpO9DQUbUSNRKreqeiiSXshOXS7VGQiLN46CxM0i7P6gnaqe6HZTnp1IU+QwyDaFWv5pxOF6nA4cuWMzH07D0VMTkVBgxJ1t3f73D8tNI9/j0RHOjEyDuPWjcSsa/55Kz/QMA5OKjsJ37CBQeaRoWmw/SE8l4Koxitvtu4slb8ccMy6GmxUd5XvdJ4r1lIIKpeu2pMN4gKioqWLhwIaWlpTz66KOW4x977DHKyspYuHAhFRUVXH755Vx66aXce++9pnGa58H4LxaHw2F6PT8/ubq5e5vYL4dZVKjVHYyrGftjTkVtq4+1u1tMXUsFAoGgNwQCAVauXMnpp59u2n766aezbNkyy32mTp3Kzp07WbJkiRoOU1PDyy+/zJlnnqmPefXVV5kyZQpz5syhsLCQ0aNHc/fddxMOJzb8/H4/ra2tpn+CvpFsArbxtZ7uJC+v3Mnn25q47dW1AGyr70g4tt0X0qsVaazd3WLpjdBWikPdqAqj52VnY7Q6405DTkN3jfXaInkIRw/N4Q/nj+Xy44YxZVieaYwkqfd6TVQ0dQZRFKhr96OgipH8tKiI0gzucNjc/6EnjMa1y2EUOwnEhhQtTfvgfzfz5c5mAIozPAzPT9NX07MNlZ6yU9yRfTUvgfl40XPGn8/RrafC+Lj7C870OjmsID0qbIxiRjusHIbli+DRqTi2f0DY7mHjuBsJX/p2nKAwXk/s8RKRTO8V43cy1ltkDhXr+VgaGR4nIwrTTQJtIOiv5wR6KSr6coNYvnx53PgZM2bw+eefEwxGVxra29sZMmQIgwcP5qyzzmLVqlVxx9q0aRMlJSWUl5dzwQUXsGXLlt5Mf59hVM3DIsqyutWHL9i37p97gzU7W6hq9rGzqavnwQKBQGBBfX094XCYwkJz2cPCwkKqq6st95k6dSrPPfccs2fPxuVyUVRURFZWFg899JA+ZsuWLbz88suEw2GWLFnCzTffzH333cddd92VcC4LFiwgMzNT/1daWjowF3kIkmynbOOiVE+rusbE6811bbT7QwkXtVp9QVoj4U8jC9XqSt/WtdNhYfjHVh+yQlHUBnSLl23lw031+vZdhvtfVzf3a61y0ikV+XrORCxaV2mtXGxzp5pTUdWihnlnpbhM/RPi4uWTtPGMi5iphipQ3YmKDIuE7zSveduUw3I54fA8/t/3xuiNB7VzSYnCZixOaZyfK1ZU9JBs3R2mRGybBLXr4cnT4a35EOwkVDqNT07/D5UjL8Nuj+/vEXuMRDkoRpLpEp9s48S+lkMeSHrrqbGiV6KiLzeI6upqy/GhUIj6evWPd9SoUSxevJhXX32V559/Ho/Hw7Rp09i0aZO+z+TJk3nmmWd46623eOKJJ6iurmbq1KlxyXtG9tXKVOyHYYyjy/Q69C9uU2fAcvz+RKCblT+BQCBIhtgVukQ1/wHWrVvH3LlzufXWW1m5ciVvvvkmW7du5aqrrtLHyLJMQUEBjz/+OBMnTuSCCy7gN7/5TUKPOcD8+fNpaWnR/+3YsWNgLu4QxOSp6CasyGhzdTcOzEbV+xvqUJT4Rmwa7f6Q3oDtsII08tJchGVYa1EARZtDd8Wlalt9/O71dXy0uUH3OgDsbu7S78/dhT9pVZGsmtHp84gkkmfpngo1p6JGExVep8mQjjO4kzQUjONSXA7DduvxEhIpFqVp093ma3HZbVx07FBOHlWgiz1tvqbwp5jVbq2vhrbZ1OwuxgI1xfT3chFeLy0bDuD86B54bDrs+hzcGTDrj3T98J90pQ+JnDdRiFgvPRVJzKu7ZGrT+7YflE3aZyVle3ODSDTeuP3YY4/l2GOP1V+fNm0aEyZM4KGHHuLBBx8EYObMmfrrY8aMYcqUKQwfPpy//OUvzJs3z/K8CxYs4I477ujFlQ0MsV9GU51iu41Ut1rvubEjQHFm8nW5BQKB4EAiLy8Pu90et+hUW1sbt9iksWDBAqZNm8YNN9wAwNixY0lNTWX69On87ne/o7i4mOLiYpxOpymvrqKigurqagKBAC5X/Gqx2+3G7bYuKSroHSZPRTemVdiYU5FgnNbLoNWQeL09EoLU7gvFNXKTZYXOQEgfn+FxMKIgjfr2Rn0/03il50TtJz/aaunl2NnURarbQbsvRDAkEwrLcSEngbCs5z3ENqOz2yU9fElRVFtA60LdGQjjC4apblVFRXaq01wpKDapN+HszZhFRfTvI5GNZpPAb+GFyYjxVGi7t/qChGUFu13Sk6UTOSdsksSY0kw21bbr8f+msrFSrK0UfdxbT4UkQUbjGipW/AZnSyQ/9/CZcNb9kFFCOjAo24vHmTgX1yyOej5nMjkVQ3JT6AyE9NyUmDPqj/aPPhX9T9TulTbqyw2iqKjIcrzD4SA3N9d6UjYbRx99tMlTEUtqaipjxozpdsy+WpmK81QY5LjdJuk/PFpy1v7g9krM/jw3gUCwP+NyuZg4cSJLly41bV+6dClTp0613KezsxNbzBKmJh60Balp06axefNmZMPy98aNGykuLrYUFIKBxeSBSLL6U0/2l7EHQlWzj0BYpt0f76noDIaRZWiLlJTN8DpJi6yqd1h4NrQ5dGcAfrU73sMBsLvZByh6fwGrECht3nab2kPCSK6hOo+sqJ6KFFe0hGt9W0AXFVkpLtPKvdbgTaOnxGUNo9FubJSX0FMhSZwyqpAUl93UTC9WzGm2QH2kUlVOikufk3E1PtYwT3U7GF+aRaY33osTWw0p2eZ3cQQ68fzvdo7+7w9Ib9mAkpIL5z0JFz4PGSX6sIrijKSTm5M5e2zitRVOu42xg7Msq3X1VClrX6B95/o6n16Jir7cIKZMmRI3/u2332bSpEl61Y5YFEVh9erVFBcXJ5yL3+9n/fr13Y5xu91kZGSY/u0N4tS3sSqBTdL/cBs7AntlPgKBQLCvmDdvHn/+85956qmnWL9+Pddffz2VlZV6ONP8+fO5+OKL9fGzZs3ilVde4dFHH2XLli18/PHHzJ07l2OOOYaSEtVA+PnPf05DQwPXXnstGzdu5PXXX+fuu+9mzpw5++QaDzWMXofucxW6VxKKoqAoEJbNAkIBqpq79FwFI6FIHJMWppTuceodpGO7QWvHgsSeikBIZnON2pPiru+OZuzgTH58bBkZHgeBsMwf39nEyu2N/PqVNSzbHA231q6tPTLHdLczboHQWPJTUdQFRAmJjIiBXdfuZ2udeu6cFGec7WAy1i1nH4/dJjEo20txlscUwpQ45AdKc1J48IKjuPL4Yfp2j9O6UVt7REQZr21kUTpZKU7GDM6Mqf5kPcdh+ankpbvJSzV7Ds3VnxJfo4mtH8KjU/GseARJkakqOxtpzgoYc36vl9uN71F3u1aUZJCd6mRIbv+qL8V6dfYHtM+vu+ij7uh1+NO8efO46KKLmDRpElOmTOHxxx+Pu0Hs2rWLZ555BoCrrrqKhx9+mHnz5nHFFVewfPlynnzySZ5//nn9mHfccQfHHnssI0aMoLW1lQcffJDVq1fzyCOP6GN+9atfMWvWLMrKyqitreV3v/sdra2t/OQnP+nThe9JYj8Ko6fCYRAVVjkVW+s7SPc4yEvQ+VMgEAgOJGbPnk1DQwN33nknVVVVjB49miVLluglYKuqqkw9Ky655BLa2tp4+OGH+eUvf0lWVhYnn3wyv//97/UxpaWlvP3221x//fWMHTuWQYMGce2113LjjTfu9es7FDE3tUu2+Z1FZabIJk1QSMDR5Tl8trWRHU1dDC9IS3jMlkhORabXQWokd6DDH4oLx9aM/3CCeW6qbSMoK3idNiqK0rnr3DHUtPoYkpPK79/8huVbGlm+pRGAF1ZUMmN0kWnurXo+Rbw5lZvqBtr069dMgSyvk8aOAKt3NPHeBjW39PDCjLj+Ck67DX/Q3PwuGSqK1QVUY8O+RHsbDelRxemcNbaY7BRXt5WKAJON4nHamTRU7dpd0xrtL5ZozsPy4z9X6F31J3wtsPRWWLkYgHB6CWvG3UZDyUkUp1pHwfREsgFng7K8DMrqf+j6QIQbDTiReey1PhW9vUGUl5ezZMkSrr/+eh555BFKSkp48MEHTT0qmpub+dnPfkZ1dTWZmZkcddRRfPDBBxxzzDH6mJ07d3LhhRdSX19Pfn4+xx57LJ988sl+WZs8vk+F8bGNNI/ZU2Ec/m2ki+epR1iHk+1t9psvukAgOGC5+uqrufrqqy1fW7x4cdy2a665hmuuuabbY06ZMoVPPvlkIKYn6CXJNr8zegesxmmbtO7Y2alOjizJ4LOtjexq6iIUVuJEgqwodARCeuJ0hsep5w50BsLICtilaK5nNFHbeqJfR5K7h+SmIkkS7sgK/fCCVK6YXs5j72/R56n1cNDmAdHKT0ZRkeFV52QMP5KV6Cqw5qn4YGM9gbBMfrqLoXkpcffbRGFFyWIUKd2FqWl5LRIS545XG0jGel2Mn4HdJpmuzTQuweNkcNhtFGS4UZQeukVveANeux7aqtTnky7FdurtyDVhBrn6lCqszreXORX9xfRe7Se2ljaNvZqo3dsbxAknnMAXX3yR8HgPPPAADzzwQLfnfOGFF3o1x31JrDq3G3MqpKinQsupsCIsK5ZdIQUCgUAg2JeY+k8kmVNh7alQt2lJ1wXpHiqK1FX2nc2d+rmMofehsMKj730LqCvGKW47qe6opyIsK9gkWLGtSTeWoTtRoVaFHJKbgk2S8DhUY1mWYeKQHH4908Uj/9tMqy+k5z8Y34OoqIiGGk0akh0XbqQoir4SnxlJgt4UWUQ8ZmguElJccrKxAlRfjDxT9+duPiibJMV5cmLDj5I2gPtZQWisRQNDnY56eONG+Ppl9XnOMDj7IRh6HBIwsZ9rzH3O6egjxnd8vwl/ijRj3is5FYLkMX4/jD8Udnt8ToXVb50/tH+Uct0/vuYCgUAg2H+I3rQShRVBbJWoxEfRREVumouRRWrfCa1HUqwY+HpXM99Ut+Gy23jsxxORkPC6VFNG9VQohGSF1q4gLYaFu0Q9LzbWqOFJpdmqqHDH5BJMH5HPk5ccDailZzXjXNYFkeplMSYiW+UvyEp0wTEzppfF2NJMdb9YUeHofU5FIrptl2Bx8HhPRfRxdwueprKwA2VAKAqseQkePloVFJINpl0LP18GQ48boJP0z8vSF4xCb38RFdpHu9fCnwTJYVwhMTa0cRiqP2k5FVYrOL6gTMp+UMSkr8k6AoFAINh/qG7xsbm2nbGlmRaVfXqH2VPRTUnZHsKfdMM8kh+Rl+bm8MJ0JEmtqtTSFYy7P+6KdLkeUZjGmMGZ1Kzz6TkVmqiQLM6VSPzUtfsBtfqSJKF7KjRsNiiPJOT6QjLNnUGyU11x+SD56W59vBVKpPoTmAWIBJRmpUT2jc2pGLiV8+4+J6sjx57OKDK6K/c64MnHLTvhtXmw6S31eeFo1TsxaEL/jx3LXg5/Mn4i+4ulpSdq93FGwlOxhzD+MdljXGpa+buopyL+j7277p17mmS6RAoEAoHgwOHrXS34gmG+3tnS72OZO2onHmcuKZs4UVsry5qX5sbrsjM0YsTvbO6Muz/WtPr1sRrGnIpgSLacU6Lwp/o29XiZXgeSZPYOgHrPzvA69b4NlY1aWJZCZzDEut2tpvnELsRpyeYji9J1u8AoKgoy3LgcNsvV//7mVBjp7q5uZfzHbknU4C5uv4HyVMgyrHgSHjlWFRR2F5x8M/zsvT0jKOh987uBJNmSwXsa7TPbK30qBMkzODtaGcAR46nQw5/aA2yt77D8AfTtQ1GRZFd5gUAgEBxgdBeulCzGkrLf1rWzcnujZXiRyaPRzfFaDKICYFQkBGpXU1ecGKjVRUXUlW/sCN3qC1ku1FltC4RkQ/UmtaRrfO8EVWjkR+a2tb5dP97LK3fS1BmkLCeFaYflRsab9y/PS+XkUQWqJySyzSgqSnNUL4WVERcrcPpDd1W6rM7dnVHZXfiTOYSoj5Zpw7fwl7Pg9XkQaIPBx8BVH8HxN4C9f1627rCZhNMeO43O/rh+q122EBX7GYcVpFNRkkFxlodsQxyT3R4Nf2roCPBtbbvlCsq+FBXGFaX9QzsLBAKBYCAYCEPG0HOQUFihqSNo6ogdPZchUdtSdKjbGjtUoVCYoRruWl7FjsYu07kAatrUZGkt3Cg/3Y3TZtNDhdSQKes5x86hIXJeuwSpLrsqKmIMZm3lvSBdbV62vUH1VCjAqspmAO4850hSIiFYVva2tgpt5akojZQm7clT0d9Qou4WC62P3V1OReJjGccl3WtCIxyCjxbCo1Nh+8fgTIHv/B4ufRPyR/byYL1n74d777+qYq9WfxIkh1UtY4dNIjXSqCcQlgmEZMvVCF9Qjtu2tzD++IiUCoFAIDh46G7FOlkUC2PIanHMlFNhdZzIRs37MDTS7XhUpALULovwp9pIuFJ+mmrkjx2cSTCskOpy0NwVpLkzgKKkWM47rCjYDMZyfZsagpyV4kSSJCRJNSztdolwWD2vZlwVZ6rn2xEJf2rtCuphWxOGZLOzscs03grtJS2UCiA70kTOKk/BmFPR7/CnXuZUxGocU2hQtzkVfcwDqf4K/v0LqFqtPh92Esz6I2TvvbYBpnyQvRCOtD96KrTPTHgqDhBskoTHadeV/h2vrbXsGurfl54Kw8///vilFwgEAkHfGIifdMucBcuQo+73UVArHTZFqjRpCdEVxaqnYnezj66guUt2nSYq0lVjXJIkXA6bHlbcmsBTAfHCp96QpA1RQ9rordAelkQWCLVE8cqIxyLD4yDD49RX5bszxiT9+DY9fOvIkgz9OmIZSE9Fd/dyq3PHNb9LUNEyfkfLh4kJ+eHd38HjJ6qCwpMJ5yyCi/65VwUFxOSN7IXz7Y/mlR7+1Md3QHgq9jIuuw2n3UZpdgrbGjqpafXz3GeVBIIy48uymD4iDwkJXygc1/RnT9MVCONy2HosAygQCASCA5QB+FG3rORk4VxXekzUVnQvRVaKU1+1L8tJIdProKUrxNe7WimJVEcKhGS9wIkW/qShNZVt6QomXJWP9XpsrlXLyWqhS9r91mGz4Ue9IM2Y1/Ikd0dExdaGDiDqwdDGJWv833LmEfhDsl5e1mph3DWAidq9zqmIeW4qPtNtSVnjcXuYdOWn8Oo1UL9BfV4xC864D9L3TfPf/bLD9V5G0r/HfdtfeCr2MjabxKShOfzu3NGcODIPgM+3NbFmVwvPLN/OI//7lkBYRpYhtBczprsCYT7eXM+anc0xlT2ErBAIBIKDBavQpd5iZaD26KmwPE40R0Kr+ASqYXNkidq7YcW2Rn17bWSswyaZchUB3VORKKcC4j0Vm2o7IvuqIckep/q/VdjRkFxV2FS3+lEURfdUFGWqYiMZUWF8i1LdDnJSDfmWFlaczSbpJen7auQVRUTPEMP7G0tSJWX7kMSccM7+drWJ3VMzVEGRWgA/eAZmP7vPBAXEeioOzfAnT6RPi9tp3TG9J4So2AekuR0cNyKfeafFJx6t3tHMUx9tBQYm9jVZtBK2XcGwuQzgXpuBQCAQCPY0A3FbsRQI3SRixz42HkgrEVueZzZ6xwxWRcUX25sAtXjJp1tVgZGV4owzwrUCKB9uqqe6pcty3kZviqIoeq+o8rw0jirLoiRigDsswo60PA9/SKau3a+Xli3J0jwVmP7vLYli+KPeir4d+IjiDCYNzWZ4fmJRYXXuuOZ3hsfJN7+zGLf5v7BoCnz6GKDA+B/BnE/hiHMSHnNvYapctVeqP+1/FlZFcQZHl+eYign0BiEq9iEpLjtnjS0G4OIpQ7j+1BGAKizCsmzpTt5T6F9uxXzD2A+/8wKBQCDoIwNS/cnKU2GVqG0Kf7KYCwq1rfGeCoCxg1RR8dWuFvyhMB9tquebKrUnRHaqK85gTY14Kj7+toHfvrbOct7G+QTCst50b2huCrlpbkP4U3yycXaqi+wU1dBav7tVrwKlFWPRPRXdGNzdvfeJPBxaXkV/xIra2C+5MrD6tm48Fd02vzNWfzIO62qCf10Nz34PWiohswx+/AqcuwhScrq/iL2ESRDthfPtj+aV027rs6AAkVOxT7FJEueMK+Gkkflkel0oioLDJhGSFRo6AgNSTzxZtDPJSuwNY3/82gsEAoFgX5FsorbSg9dbUaIhTUPzzBWbhuRG8yqWrq3B47RT1aKOLUz3xBm9OYZwqE217ZbzDoVlttZ3kJum9ozQyuDmZ5jzM4y9KoxlUYszvTR1Bvl6dwuVjWroVEmsqBjgJW4tFGtP5ldaJ2rHbel2fHSUFP943auw5FfQXqMeZ/KVcPIt4E7rx6wHHtNl7RVPxZ4/x95GeCr2IXabhCRJeqKWJEl6/OPuZl/CDqB7Ak1IKChJd0sVCAQCwaGHVdhGbPiToii6t90fCrOxujXunhYIh9keKcWq9abQsNtsjBucBcAbX1cB0STpkixPnPE+Y3QRIwrTIse1dvNXNnbybW07n21ppMMfoiFS/Sk3NUZUGJSE0UguzVEFxBMfbKXdHybd49Cb12ml4Z32bjwV3SzSJQp39rri8zwGGuvmd931qUjOU2HvrIUXL4K/X6QKirzD1Z4TM3+/3wkK6Ec53D4yEPlN+xtCVOxDrL60xZGkr6qWLssY1T2GFv2k9Ly6JBAIBIKDg+bOANURD0CyWN0XYj0VRgHx9LJt3LXkG374xCe0dEZLqG+qaScQkvG67BxeECsqJMaXZQHwyZZGFBTdU1GS5Y0LB3I5bFx5/DAA2hN01W73hyLzV5j7/Cpq2wLYbXBYgdnANRrwxvNoydrNkbCpn0wZijsiJgrS3RxRksHw/MTGcneLdIni68vzUtVGupley9cHAitbJHaLKaeiJ4NbUSje+grFz54A618FmwOm/wqu/BDKju33fPcUe7uk7MGICH/ah1ip/eJMdcWkusW3T8KfFBCeCoFAIDhE+HybmgjtddrJTEkultpqwSvWC6FVL+zwh/RzfLq1kcXLtnGtIX8Q4PCCtLhcBJukJke7HTYaOgJsre+kpjUqKmITiTM8Tr0ClKyoAiLDY74eh81GKKz2xfhsWxM2CebPrNBDmPRxhkRt433aWEHpO0cWMr40y5RLEXucWFLciSvqJFpDdDvscU10B5rkciqSK7cqtVRy1AdzyK35WN1QPA7OfhiKx/Z/onsYc4WrvRH/tOdPsbcRomIfYi0qNE+Fb69Wf9LztBUlpvrTQfitFwgEAoGJVl+QNI+DujY/uWkuU+O1WIKyzKL3NpOf7sbtsLGtvpPbzj7CNEYTGSsrm0zbtzd0UN/uJ9Xl0EVFRXFG3DlskoTLbuPwwjS+2tXK6somZAW8DhvZKc44w3ZYfiqBkEya20G7P0SrL2gSFdWtXfx1eSWzxpUQjsRlFaS7OeHw/LhzGxO1jcbl6EiZ29xUFxdPGUpnINyrKkEF6R5GFsm4HDa+2tliem1fLuBZ96noZfUnWYYVT+B65w5ygx2E7W46ptxAxknXg/3AMDXN+SB7noPRujowPumDFMvwp0h5uqqWLsLhvemp0HIqRPUngUAgONQIhGV2N3exobqNIbkpjChMTzh2Y3U7X1Q2m7at293GxCHRKj4hWaG+3c8762oAtUlcVYuP7Y0drK5sJtPrYE3EsB5VHH8u7f6o5Tus2aWOLY54KWI9G067jWH5qWR6nbT7Q1S3+MhJdZHiVM2cR/73LVUtPja+vYEfHF2qH0vLhTBi1VEbYFxpFnecfQRpbqd+n+xt7H1pTgq+SAl3I3tzETEWy/Cn7qo/xYqKug1qE7sdnyIBTXmTWH/0XQwbNZ6MA0RQQN96cfSHg9G+OnA+7YMQK7VfkO7BYZPoCspsa+igeA+7PTUUQ/yTyVNxEH7pBQKBQGAmEIomN/tD3dcz39HUGbetocOvP5Zlhf97eQ1L11Wr3gWnjTPGFPHkR9vY3ayGMH1T00Ztmx+HTYrLp4Do/TErEpK1s0lN0h6UrVVbip+XTZLI8DrY1QyPvb+FNLeDE0fmU9fm1/MxFKBKS/jO9OJ2xIckWfWp0DimPJddTV10BcKWryeD1T77UlRYEScqTNWfIg/CQfh4Ibz/BwgHwJWOfOrtrEw9E6QDL2XXXFJWJGr3BSEq9iFWyU4uu41Rxel8vauVd9bX4rTbGDM40/KHbyCJagpzo4qD8UsvEAgEAnNuRDAs6wnKPdm3OxrjRUVjR0B/vLmunbfWVgNQUZzOxccOJRQJOWqONJxbt0vtOXFYQRoeV/z9TfNExNbML87Ums1Zr64bQ57a/SFeW1MVN25bg1YO1mNZVcmqT4VGSsxc+9I/wmqfVNe+M8esE7W7qf4kSbB7Ffz7Gqj5St142GkwayGkD4JvauP2ORAwVZTdC5ooze3AHwz0PPAA4sCTkgcRtgTv/oTSLABWVTbR3Blka33HHp+LdnNRFHPC2H62eCIQCASCAcK4Oh4My2iVWHtaNd8R8RpcdtxQTqsoBMyiojbSJTsnxclvzxlNfrpb9zh0BWX8oTBrd6ui4siSDEujVlt0y00zl3vVkpatDFa7TYpLzraiMlLGtijTg9tp5alInJTsjREVfUnoNV5vWW4KJVleyxCwvYV1SVnrsbaQj9QPfwtPnKIKCm8OfO8J+NFLkDn4gBMSRvZ29aeK4gxKsrwcM2z/aP43EAhRsQ9JVOt5wpBsANZXtSIrCqG9mVuhCO+EQCAQHAoYF5D8IVkXE91VHlQUhZ1N0W7SWSnqCrtJVEQa2hVmevRQIo/TrpdfrW8P8E11G6AmP1vdCbXbY06qWSRoFZaswlNskkSmN7riP2tsMYt+NCHhtRRneHFZJKQ7E634oVbJMtIXI9q4T5bXyRElGXs8GqE7kikpa7NJZNWtYPLbZ+P59CFQwjD6PJjzGYz9gX5RpipRe3LSewBzhas9P3uP084RJRlJCeEDBRH+tA9JFIs5sigdr9NGV1BmR1Nnj2XqBgLjPcRYGlB4KgQCgeDgQlEUJEkyeSSMoiJRzwSAujY/Hf4wElCU6dVzFZoMoqKuTfVUZHqdOGySam8qElkpTmpa/XxT1UogLJPqsjM422tpmGvhTzmGxnRep033eFjnVECGIVxqeEEaLruNX88cxRtfVQMKX0aSw4fmpnB4UZplorbNJjEsP5WwrOCJERGxxn9fciqkvb0k3gM9Nr/ztZL57s0MWvMXAOS0ImxnPQCjzujhwAM4yb2EzaYWsjoAp75fIDwV+xCjpyLN4yA71cmRgzJwOewMzVPrYVc2xMeu7gmM3glZtt4uEAgEggMTo42oaQajqAiHFT1Zu7u+qxtr2gG1HKvLbtON+OauBKLCLunn1vIjtkRCeosyPUiSlNDrAGqeYbpHXf809qewzqmQTKVwyyN9JUYWpXPNyYcxelCm/tp9PxjPYRYJ4hrD8tMsK2A5LPpp9AdHN16RvYVlnwrtwca3YdGxpEUExc5hP8D/s096FhQHKNr360AO49qXCE/FPsSYqO20S3o5vm31HZTlpLC+qo3Kxs69XtpMVH8SCASCgwtJiv6ehxUFG1KceOiKlDq1am6noYU2ZaW41P81UdERNIyJ8VQgAQpZXnWfDTVq6FNeulufWyzG+2Nuqos2X8jUBC7RfVETAlleJ6mRZnhOm41wOMykoTl8vq2Rc8YPYmIkzLi32GwSdrukl3zvi6cCYGheKp2BENlJNhzck1iF+ti6GuDNW+GrvwMQyhzCl+PvpKlwCsd54/uKHCyU5qTQFQjHhbkJkkOIin2Isc620Xi32yTKclIA2N7QuZdKm0UxxtMKTSEQCAQHPpphD9GFo9iEbK1MaqKciro2P6si/Sm0sCCtC3dXSKbDFyI10kAPVFFht9l0AZAdyY9o7lQFSH5aYlFhXMAvyvSwraGT0sh9Ud3H+r6Yn+7m7u+ONlVTctglCEK628ENM0YxtjTTct9k0URKorknw2EFaf2aw0BiugZFoXDHEtJfuws669UySMdeTd3EeTTVhoDE+aCx9FVw7Uv2p8/lQESIiv0E40+4zSAqdjZ16aX49uj5FWP4k2K5XSAQCAQHJsZQVu1nXYm5tejbE/zsr93dojdu87pUq9/rtOO0SwTDCtVtXQz3pOvejAyPE7skRQSAElceVhcVFgtnRsP1B5NKOf2IQooyop6KRHatXZIoSPeYtsWHLFns7G+HkE99U+SwmoisP5ZN2zNamnD6A0hKGLeUAm2OmPFhNY5YkSOPwzGPlQTb5Zhz9md7d3OQTfMtCQTI9geRFBlHsI305vXqe1JwBJz9MAyeiNzcBbTq73F3lOak0O4Pkpvq6nac4OBDiIr9BKMhb5ckCjM8uOw2AmGZysZOxg7O2rPnV6wfC0khEAgEBz5WIa6JSscm2q4o6KLC41RFhYSagF3XFqCq2cfw/HTdU5GV4sRul3QBMCzfvAqc3034k9HwL81J4XsTBuvduWNfN+8Xv80RU+HJZBR3NsIbN8JXL5HsHW9sUqMOHLyRfxqyzUlw6vW4T7wBHKowMC4wxnYzj2Vk0b4rjyvYtwhRsZ9g9lSoP5ilOV6+retg7e5WBmenUJ6bqrua9+QMRPUngUAgOLiwqvCXWFRYH0NWFHxB1b1hrIKU5XVR1xbgx09+xitXT6XVp4bJZHiMORUwLC+VNLeDdr/6emGGKiqsBIJxmxbfbrdJ+twTigoLgzfOU6E93/g2vHoNtFcbXpXUkB+bHSS74bFNfxxUJMKKhCLZcTodOGz2bsf3b7tdVV3dbrepz02PbUlvr+8IUdUWBNRrass+gnHjjsLtiJqIwhYQJIMQFfsJxj9Y7cfysII0VVTsamFiWTb1bX5OPaKw2+N0BkI4bDbLMnnJnt98oxG/JAKBQHAgExvGqomGRLkTmuf8i8omAI4qzUKSJBQF/KFI+JMhkfX4w/PY0diJLyTzyLubATWPIcVtx26LVn+ySRKHFaSxekczEC3/mqiRnYbW38JUiTVRczaLF5yxnopAG7x9C6x6Vt2Qdzic+ygMmphUksSm3a3sblYb6I0tzYwLtzrQ6GrspKa6zZTMH/suCFEhSIZ9X8tMAJjjXTXX7IhIubuNkUoZgF7yzwpfMMyyzQ18sLGuD+ePIqo/CQQCwcFD7O+4JjIS/b6rTVdlGtsDNLYHdM8CqB2xAdxOO+keBykuOyeOLOBHx5YB8MEm9f6T5XEiIUU8FVHOmzgIgBSXPVq+02IORueC5hUxCo3EoiJ+m3G/nJqPSXnyuIigkGDKL+DKD2DwpKSzrl2O6LgDMRk5lgyPE5stWtEL4t+KnrqsCwQgRMV+g8lTYdNEhRp/Wt3qp9WnVsto6PAnPIbxh78/5zeFP/X5iAKBQKCyaNEiysvL8Xg8TJw4kQ8//LDb8c899xzjxo0jJSWF4uJifvrTn9LQ0GA59oUXXkCSJM4999w9MPODg9jf8Z7CnxTFHALV4Q/rj/1aorbDzjHlOUwZnovdJjE4Wy0uEoyUWtUqPameiqiF+r2jBrFw9njmzxylb+up+ZrbkL+h0ZvwJ6ddwh7sYOTK25nw/k+xte6C7HL46Rsw4y5w9q7BrLG3xMEgKjJTnJx4eAFDcw3VtWKknhAVgmQQomI/IbakLECq26HX5V5frXor6tsCcfsOyPmNze+Midrid0QgEPSDF198keuuu47f/OY3rFq1iunTpzNz5kwqKystx3/00UdcfPHFXHbZZaxdu5aXXnqJFStWcPnll8eN3b59O7/61a+YPn36nr6MA5pE4U/dNbkzGpHt/hDBsOqh6ApFE7WlSGUnh81GcaYHY9TtyCK1l4HDZjMtVHmcDs4eV0JxptGQtzbMM7xOJAlyIlWEjPZ74kTt+O2e3Z9y7NtnU/rt3wAIT7ocfv4xDJmS4Oq7x+kwioo+HWK/I1aMxb6N2ueliUWBwIo+iYrerjq9//77TJw4EY/Hw7Bhw3jsscdMry9evFj/cTL+8/l8/TrvgYRV+BPAEcXqD/Nzn2xnV3MXHYHkvBG9LQVrrFprDn8SqkIgEPSd+++/n8suu4zLL7+ciooKFi5cSGlpKY8++qjl+E8++YShQ4cyd+5cysvLOe6447jyyiv5/PPPTePC4TA/+tGPuOOOOxg2bNjeuJQDlthfce13XcudaOwM8L8NtSzfUk8wcjMIyWZRoYXeap4KtyGnwm5TxUOhoeTr6JKMyGsSgXDYMFbCZpNMfSgSGeaThmRzwuH5cTkRkEiGmI9lC/kYsfpuCv7xPbwdO+hKKWHlCYuRzrgXXKkJjtAzTmMY1l7oI7W3MAqyWFHhddk5YWQ+E8r61jRQcGjQa1HR21WnrVu3csYZZzB9+nRWrVrFTTfdxNy5c/nHP/5hGpeRkUFVVZXpn8cTTX7q7XkPFLREtaKM6LUaf2zPHl9MeV4qnYEwr6+p6janIqZ/Ta9QElV/6t1hBAKBQCcQCLBy5UpOP/100/bTTz+dZcuWWe4zdepUdu7cyZIlS1AUhZqaGl5++WXOPPNM07g777yT/Px8Lrvssj02/4OF2PuB9hOvbf/r8m0892klT360jWWb1TAz432g3RcVFXqfCmf0RpUSaTQXNqxOleepRrvdJmHVaslswCYOZTKWg5ViXrPcJ3KsjIYvmbz0HIZsXIyEQvXwH/DJjNdoLpraY0nUnjDOyXYQxXuYEuEtxJLTbkv4WQkE0AdR0dtVp8cee4yysjIWLlxIRUUFl19+OZdeein33nuvaZwkSRQVFZn+9ee8BwpHlWUxZnAmww31u42eCq/TwXePUhPbttS3EwjJSXkPEsfKJo6htdpXOCoEAkFfqa+vJxwOU1horlpXWFhIdXW15T5Tp07lueeeY/bs2bhcLoqKisjKyuKhhx7Sx3z88cc8+eSTPPHEE0nPxe/309raavp3qKDELA9pVZ+03/rq1mhUgJa3Z+yd5AuG9Zw9Y6K2xsjCdAbneJkyPA+Ashwv9oi1naj7sinpug/XlAi7HGD4mvs4+t3ZpLZtxe8poPHcZ9kydQFhZ1q/BQWoORoaB5ORbRQSB9FlCfYivRIVfVl1Wr58edz4GTNm8PnnnxMMBvVt7e3tDBkyhMGDB3PWWWexatWqfp0XDoybiNNuozDDY/qhi40JHRJJnqqPVOEIhK29FcYfN6tYWVlWWP5tg14mMBHGVaXYm5FAIBD0lljDS1GUhMbYunXrmDt3LrfeeisrV67kzTffZOvWrVx11VUAtLW18eMf/5gnnniCvLy8pOewYMECMjMz9X+lpaV9v6ADjDhPRUyidmtXNKy2MxJiG4q5iWjFQqKeiqiosNkkBmV5mXFkIT+dOpTrTz0ciCZYazgMxrhx8SzpZOeehlV9Sfk/z6L8mz8hKTJVZWezfMZrBIedpp+vp27QyWAMxzpYcioA0/t7MF2WYO/Rqz4VfVl1qq6uthwfCoWor6+nuLiYUaNGsXjxYsaMGUNrayt//OMfmTZtGl9++SUjRozo03lBvYnccccdvbnE/YLYlZRUl4PCdDc1bX621ncQCMmmxkMaRi+ElaeisTNAZyBMZyAc91rCPhVCUwgEgj6Sl5eH3W6P+52ura2N+z3XWLBgAdOmTeOGG24AYOzYsaSmpjJ9+nR+97vfUVNTw7Zt25g1a5a+jxxZCXE4HGzYsIHhw4fHHXf+/PnMmzdPf97a2npICQsj2k+8rIAvFMZvCKvt9MuR18w//v6QTFiW9epOHqf5HpTqcuCw2Zh2mCr0Jg3NjhvjMoUN9X5VPGH+QjgIH94PH/wBjxwi4M5h/cQ7qRt8emS/qGdkIKo1GZvpHUzefHMfECErBL2nT83verPqlGi8cfuxxx7Lscceq78+bdo0JkyYwEMPPcSDDz7Y5/MeTDeRoXmp1LT52dbQkTCvIrbXhC8YNv2oh7sp9WHKqTCGP/V9ygKB4BDH5XIxceJEli5dyne/+119+9KlSznnnHMs9+ns7MThMN+a7Hb1d0xRFEaNGsVXX31lev3mm2+mra2NP/7xjwl/491uN263uz+Xc8ASKxBkQ/hTS1fQ9FpHAk9FICTjM9x7PDFeCFukyZ12KmPPg3SPgzZfSC87C/FdrpPB8nZfux7+eRVUrQagfdgZrBx9K0FPjmHHqIgZiBwIY06FVRL5gYqU4LFAkCy9EhV9WXUqKiqyHO9wOMjNzbXcx2azcfTRR7Np06Y+nxcOrptIeV4qn25tZFNNm2lVyYjxxqEl2009LFdPojPeJGIFmck5IXIqBALBADFv3jwuuugiJk2axJQpU3j88ceprKzUw5nmz5/Prl27eOaZZwCYNWsWV1xxBY8++igzZsygqqqK6667jmOOOYaSkhIARo8ebTpHVlaW5XYB1LX5aewwlyI35lS0dJpFRVfEky3L8ULEH8mncNgkU68GDbtNIhSOv2kcVZZNqy9IbmpUaJhCfpMUGKZRchiWPQT/uwvCAfBkwZn3UVf0HYJ1HeYdFQY0/Ang2OG5hGUFl+MgEhXdVH8SCJKhV38NxlUnI0uXLmXq1KmW+0yZMiVu/Ntvv82kSZNwOq3rHSuKwurVqykuLu7zeQ90xg7OpDw/WvLuyEFqeb51VW18tbPFeicLAVDXFm2WFw4bw6PM44yCRORUCASCgWL27NksXLiQO++8k/Hjx/PBBx+wZMkShgwZAkBVVZWpit8ll1zC/fffz8MPP8zo0aP5/ve/z8iRI3nllVf21SUc0Hy5o5kdjZ2mbdGSssR5KhLlVKBE8yncCQxpzRORlWK+t7scNvLS3Caj1Wjc99Z+TWnbCk99B965TRUUh38H5nwKY85PGL2ghT8lShzvLWluB5neg6tng/bOSJIIfxL0jV6HP/V21emqq67i4YcfZt68eVxxxRUsX76cJ598kueff14/5h133MGxxx7LiBEjaG1t5cEHH2T16tU88sgjSZ/3YKMgw0MBsDWy4lKc4eXUUQW8800tf/5oCzNGF8XtYxXdZIxBDcfkXNgNryWSDsJTIRAI+svVV1/N1Vdfbfna4sWL47Zdc801XHPNNUkf3+oYgsREm99Fw5+0vL2OiKciNlxWy7+A+HwKjWF5qaS47OSmuSxfN2LvU06FQunGv3DYV/dB2AfuDPjOAhj/I/0gVsdSDOcbiOpPByvaeyf0hKCv9FpUzJ49m4aGBu68806qqqoYPXp0t6tO5eXlLFmyhOuvv55HHnmEkpISHnzwQc477zx9THNzMz/72c+orq4mMzOTo446ig8++IBjjjkm6fMeClw8dQjvfFPLqsomQmFZj+vUQpmsvArGH4eQoWpUbIxtIvEgNIVAIBAcmMSGMGlogkFRop6KokwvNW3+aPiTRR6GLxL+lEhU2GwSJVley9disfe2gVzTNka99TMyaj5Vnw87Cc55GDIHm4YlOpbuqRAWc0Ki3pyDJ6RLsHfpU6J2b1edTjjhBL744ouEx3vggQd44IEH+nXegx1JgsPy03HZbQTCMpWNnQzLT6OqpYv1Va1UFGdYeiqMBMMKw75aSEveBJQR34t5tXd9LQQCgUCwfxNO8PutFfsweiqKMz18uVMNewqE5bjcCK34B0B2ipPJw3LoD0l7KhQFVj4Nb91MRrCDkCOFTWNvpGLWtUkvqStKtOrTQIU/HYy4HXZGD8o8qPJEBHsX8c3Zz9HcyKU5KXhcdooy1cTzbyNhUTubupBlWLurVf/BN2L8zXVXvs+w9Ys46sPLcbxxAwSicbaKwR3+0sodPPXxVhSRUSEQCAQHLMYQpto2H698sRNfKKz3OgobREVRhkdf4+8KhOK92URzKnLS3KR7+pdPYEsmKbhlJzz7PXjtegh20FZ4DJ+e/iq7DruwVzE6bodNb1gnREX3FGV6yEntOXxNILBCiIr9nDGDMhlbmslh+Wm4HTaKM1XX8jdVaiM/Y93vbfUdcfsbk61a8iZQedhFADi+eBL+NB12rQSifooXPqvkrbU1LPu2gdo2v4h/EggEggOQpo6AHsoEcNM/v2bJ19UsWVNFICSjKAqLP97K2t3qvSQzxYnXpYY1dQbCcYnaiqLoJWXT3NbhT73B0V34k6LAqudg0VT49l1weGDGAjbOfJ6utLJuj2vUGuPLshhRmEZ2qoviTC9D81L1ZrICgWDg6VP4k2Dv4bDbKEj3AJDqdjA0Ulr2yx3NQExPCctE7Sg+yc3GCbdQX3IS47+4CVvDZvjzaXDCjciDL6HNF+TdDXX6+E5/WPgqBAKB4ACjwx9i5fYmfVW+qqVLf21lZRNfVDZxTHkur35ZpW/PTnWS4rLTGQjTEQjFJ2rL6nEBMvrppYBuwp/aauA/18LGN9Tng4+Gcx+FvBGkVrfS1NlFsuSluclLU737LoeNwwrS+j1vgUCQGOGpOMCYOCQbgG/rOwiF5W4b2kFsorY6trHoONov+xCO/C4oYXjvbo548wdITd+a9u0MhkT1J4FAINhP6QyELMNetW1hWQ1iXbquRn+tptVPdaufV7/crW/7+QnDGJTlJSXiqegKWN9bmjrVfheFGZ5+z93UUVt78PU/YNFkVVDYXXDKbfDTNyFvBADD89MYnONl0tDsfp9fIBAMPEJUHGCML80CoKq5i45AyLLRkBGjKAgaqj8p7mw4/2n43p/BnUl6/WrO/PgH/ND+X7SYp65AWIgKgUAg2A8JhWU+3dLIyu1Nca8ZBcHfPq3kg031CY/zi5MO44rjhyMhkepWgxeaOgP89rW1/Gv1LtPY5g6tUlT/RYUx/Mne1QB//wm8fCl0NUHRWPjZ+zB9HtijARVOu41RRRmmbt2xiCRjgWDfIf76DjCG5Kbgskv4QjL/XV9LSLburq2h3VqCYdkkEGRFUd0YY78PVy+juWgKTtnH3c4necp5D/k06/G4ogKUQCAQ7F8EwwphWYks/ph/o7V8CAWFDyOC4tzxJZbHOXJQBpp9742Uin19TRVf7WrltTVVprGap6JoIDwVETd6/q6lZDx9PKz7F9gccOJ8uOJdKDyiT8ctSHdTnOWhoiSj33MUCAS9Q4iKAwy3w85ZY9VO47966UveXV9LuBthod1srMoD6mQOZt0pz/De0OvwK05Otq/mLff/Mbz+3cgxBvgiBAKBQNAvjL/hsZFKmqeiKxhNuD79yCI9vAngsmlDmT9zFE6bTS/ooXkqGjoCluds7hxAT0WghSM//RXjPp6DrbMO8ivg8v/Cib8Ge99zNiRJ4siSTAYl2S9DIBAMHEJUHID8ZGo5HocNWYGnl23jy50tCcdq951A2Cw8YnWCItlYVnABswK/Y508hBypnYt33MIRn/0axd86wFcgEAgEAiuC4e69zxrGIh2x+Q+akGjzqYnVHodN73Gkccm0ciqK1dV8zVNhFB0amngJyjJtkUTtfnsqNi0l8+npFG9/FUWy4Tv2WrjyfSgZ37/jCgSCfYoQFQcgxVke5px0mP68qsWXcKx239GqdmhYdUv1B8NsVEo5N3Ani0JnIyNRsu0VbI9Ng20fD9wFCAQCgSCOyoZO3t9Qx+7mniscKQbtEft7rnmv2yI9KNK9qgciz9B/IBCW9f20Skw5FrkK/pAaBtsS8VI47RJZKX30JPha4dVr4LnzsbVX05FezoqTXyB44i3gcPftmAKBYL9BiIoDkFSXg4riDM4ep4ZBNbT7E47VSsJWt5qFh1VIk1aDPICTP4Qu4M7ce+hKHYzUsgMWnwlLb4VQ4nMJBAKBoO9srGkDYN3unr3D5vCnWFGh/t8a8VSku1UR8MvTRzIsL5VfnnY4/mBUVGj5DdNG5DF6kDkXIRCSqW/38+baagAK0j2m/kdJs+V9eHQqfPEMIBE6+io+Pe1ftOaO79vxBALBfocQFQcgXqcdSYLcSP3tunY/dW3+uBsLqOLBFwzT2K7GyGru7XhPBfhjShOuth3BJ6e/SnjcjwEFPv4jPHEK1KzbA1clEAgEAkiu63P34U+qqmj1qd6FDK8qKoYXpPHMZcdQUZyheioi4kOz6T0OO9eeMoIbZozUj+ULyTz98Tbei/Qwyk/rpUch0AFLboBnzoaWHZA1BC55nfCMu5Edat6DkBQCwcGBEBUHIDabhNdp15v6rK9qY/4/v+Lvn++MG6sQTbrLTHGSEknEi03sUxQFX1C9w2REXOVdgTBhZxqhWQ/C7GchJRdqvoLHT4BlD0MPlacEAoFA0HtsSYgKk6ci5qc4HJNTkeFRf9MlwGVXb/vG3A2bZOwZITGyMJ2syH3AFwyzIeJBAchLS1zONY7KT+Cx4+Czx9Xnky6Fny+DodNw2KLmh9WCmEAgOPAQouIAxe20kxvz4/7O+hpWbGtklyEeV1YUgpGwphSXXV8Rii1BqBCNndXiaruCocjY/8/eecdJUd///zmz9Xav9zu4ozcFRMEoKIoNY8GWosYuWIIlQhITe2I0pioaWzSgMbHFWL7+FAtKxIYFBBsH0usdx/W+beb3x+zMzmy7veMOuLvPk8c92J35zMxn9vbm83l93g0YNwt+uhxGnQwhP7x9i7by1LC9529OIBAIBjD2KFGxobqZjzbU4A9GhICSJKbCCNTWYyrCFbAlSTKsIIFgfFGh47JrVm1fQLEEcLfHKbYXQ6AD3roFFn0f6jZB5iC46GU4/T5wpYevae5/56cUCAQHPkJU9FFcdpmcNAfRC1p/f38T97+73nivqpEBxi7LxuARszCkYlgq9MJC7f6o5a+MIvjJ83D6AnB4YMsHmo/sl8+LvLMCgUDQQ0S7P22paaPdH2J7fZuxzSwkQjExFdr7pnCCjgyTpUI/t89iqYCxJRnIMpTlegBwObTpgS+oWMTMBUcMSd75nSvh78fA8gcBFSZdCHOXw4jjLc0kSSLH68DjtJHhssc/l0Ag6FOIv+Q+itthwxYWCdGrVHWtfoKKEjYvq4aosMmS4TsbfYyKalgq9MweFkuFjiTBlMtg2DHw8lWw43N4+UpYt1hbhfLk9vzNCgQCwQAi2lKhY643FB2ovbupg017Wpk4OMto19RujamQpIioCIXb2GwSkiQxOMdDaVYata1+tte1GZaK5o6AMYbcf94kjhyRF7/TQT8s+yN8eB+oIUgvglkPwJjvJ7zPyUNyUVVVBGoLBP0EYanoo7jDq0hBk934ymOGG6/1rB+KGhmI7DK8/90eKqqa4sRUaCtSADlhS0WbP4Qa/hdD3gi47E047latCuqaV+DhqbDhnZ66RYFAIBgwKKaHcqKYCnNAtsX9SYGvdzTS6gtSUdkUE1MRsVRIllgGsAoYWZYM67duqagLx+TJktWF1kLV1/D4cfDBXzRBMf6HMPeTpIJCRwgKgaD/IERFH8Xt0FaRijK0YO2SLDffG5pLVji4TvelVdXIQLTwo8386a11PLx0A0GT6VtVVSNLFECuV1vVUlQtnWBCzyabHY79JcxeAnmjoKUK/v0DLdOHvy3BQQKBQCCIJmBSCbYEE22LqEjg/hRS1Ej2J91S4Y61VOhEiwzdRdYdtlToosLrsiMRsXZrFwvCsj/DY8fB7m+0ZB4/+if8cKGwWgsEAxAhKvoouqi49viRHD+mgHknjQYig4duqVDD7k+balr51/JtALQHFbbVRSb9+nhkZH9yR2I12gOheHYKK4MOg6veh+9dqb3/7DHNp3bnFzFNa1t81CSpqyEQCAQDkYDJtSnRMzekJnB/irJyKKpKSFFo9WsLRXr2J4h1rbLbrO91UeGya9MDPXtgenTcQ/VaWHgi/O8uUAIw9nSY+ykcfFaSuxQIBP0ZISr6KPoDvyQrjRtPGWtkbNJ9Z5tMlopgSOGbnQ2W49fvbjFe68ORHlPhcthIC4uWdn8oJlOUmbpWP++tq2Z3hwyn/hkufBHSi6F2PSw8SVvFCmkCJ6SorNrWwOptDRZLiUAgEAx0zM/ERClWQ0r8NjF1hxRYG04Da5clvGFBIElhFyfTyB9tuZDC+wxREV4ESndpY4KkKFrNor8fA7tWgTsLznlcSzueXpDy/QoEgv6HEBV9FIct8qszm8T1FSm96JHu/rSuShtg9EWp9dWRvOO6aNBjKtwOGW94AGnxBZNaKlZtqycYUvl6R6O2YeSJWqaPg84CJaitYj3xfajdaJjkITZbiUAgEAxkLJaKBM9Ha6B2ZLs11kJFUVVeCNctOnZMgWF9kMIREeYUstGWC/2dK7ywVGu4PzlIa96C99lZsOR2CPlg5Ela7MTEH4OIjRAIBjxCVPQDQopqPM8jlgo9UFulLRBi455WAH44eTAAG6rjWCrC7k9uu42CcKzGo8s2sruxI+G14459nlz40ZPa6pUrS8sQ9ejRyCufFKlnBQKBIA4BS5wbpteJYyd0zAJDUaGisokd9e2kOWRmTSw19unjhDmOInFMhV4kT0VC4Wz/axz59hnYd34Gzgw4429wwQuQWYpAIBCAEBV9Gl1AFGe5I6IixlKh8sF3ewgqKvnpTo4fWwTApppWY7BSVU18+MODmssuM/vo4eR7nTS2B3nj66qud06StNWrn34EQ6dDoA3HG/M55MOrcLbvEdpCIBAITCSyQqiJLBKmHWYrsKqqrNreAMDhQ3MtsRC6YDC7PCWKqXCGLRWDpT087fg9lzQ+jC3UQXDIdJj7MRx2sbBOCAQCC0JU9GEOK8/m8GG5FGa4jYFAFxqfbq7jz2+t5eVVO/nn8q0AnDC2iOEFXmRJSzW4p1nzlVWJCArQzN7DC9KZMlTL3lHbtheB1dllcPGrMPNuVJuTgsr3OPLt05HXvd79c+4nFEW1BEQKBAJBTxFIIV7CLCrMYsNcnC6kqnwZFhWTyrMt1zAsFSYhERNTEX7rtkmcZ1vKm85fMc22Bp/kZu2ht9Nx/kuQXd6lexMIBAMDISr6MHabTJZR1CgsKsLZnwDW7W5h0UdbADhyeC63njYOt8Nm5Cyv1kWFCr5wOlkJcNi0XOVZ4SJ4ekrBbiPLMO1a2i59l+bssTh99Tj/exG8cg10NO3dufcRqqry0cYalm+qTRq4LhAIBN0hGIovGBLFTphfm0XFhuoW6tsCOG0yY4szLdfQ5YNZSDjiuD+52qq4ePMv+IPjH6RLHXymjOHhsf9kx6gLkSQxbRAIBPERT4d+gmy4Pzni7v/++GIy0hxIEoYQ2WMSFe1hUeF2yEhIyJJEdrjmRX1roEf6GMwfx2cn/JctY69ARYLV/4ZHj4KtH/fI+XuTjoCCL6DQ7g9ZBnOBQCDoCQJRtYN0zJYKc90h83azqHj9q0oAJpZl4bRZh/hITIXJUmF2f1JVbF89x5FvncaIxk/wqQ5+F7iA8/y34c8cshd3JxAIBgJCVPQTIu5P9ph9TpvMkFwPoFk0dBepr3c0aoXvUI3MT65wwSNZkshK09LU1rd2z/2pqrGDD9bvobEtEt+h2pxsmPhLOi78f5BVDg3b4IlT4Z3fQHAvLSK9iDlAUkgKgUDQ01hERQrtzGsbuvvqppoWVmytRwLOnVIW5+g4MRX665ZqeO4CbK/+FEegmd0ZB3Oq//csDJ2GgozHlJZWIBAI4iFERT9Bf9DHFCgChuZ7TGIBssLWjM21rdS3BVBVaPPrNSq0r4QsRywatXEsFZv2tPDh+pqkffpmZyO+gMLmWi3zlHmFPzj4SC2Ie9IFgAof3gf/OB6qKyznCCkqOxvajWrf+4tEvswCgUDQE1gzOCWuQRERFdr2F1Zu54F31+MPKXy6uQ6AI4bnMqY4g4mDsxhZmG4cGy/7k02W4JuX4KEjYN3rqLKDDePn8dqUJ9moDjLaZYfdYSWEqhAIBPERoqKfEMnqIfPLk8fwsxNGkevVLA0jCtKNlSnZZKloag/gDyqowO4mLW1sfrrLaKcPIvVtvpg4gk17Wi0T/Si3XMs+vYiSJaMJgDsTznoYfvwvSMuFqq/h78fC8oe06k3Axj0tVOxqYuXW+m5+Mj2DRVQIW4VAIOhhrC5PxH0NkdgLRdESbLz17W6+2NbA4q8rqWxoB2BsUSY2WaIw022kB4fYmAqHr47M/3cF/PcyaK+DoglIV/6PrQf/FKfTaRyX63VSnOnWziE0hUAgSIAQFf0EczGjMUUZTBiUxbQRecgSTB6Sgz3sWytJkQxRjR0BdjW08fvXK1izSwuYHpSdZpxPb+cLqoYlIxFS1EhT0xJxmXKEfXZVNcFq/0FnaAWURs3UCiq9dTP860xo3EF1k3ae9k6u39tYUzbux44IBIJ+idUamthSsaO+nZCiElJVYzEI4N01u1kfrj9UnO22LCTpSKaUshn1azjyrdNxrvs/kGxwzI1wxVIonqAFa4et2wCHlmULC4VAIOiUWF8ZQZ9EjvO8P3NSKadPLMEuy4bfrIRkuDU1tgf4w5tr+WxzxApQaogKSHPYcNgkAiGV2hY/3jiuVTrRl69ticRH6K7C1tzrUTPzjCL4yX9g5RPw1i2w+X14eBr5k+9gx6DT9vvymAjOFggEvUmi2hSqYm23u6mDDLcdVcVSmLTdFKxdkuXGplfRNj069Zd2WaItfQghWxpK/mjks/8Ogw6LtJMiFmaA8YOyLPsEAoEgHsJS0U+I96CXkAzf2ciqVSRWoqk9wEZTZW2A0mx3uJ2EJElkuPW4iq4Fa/uCsTnXo7OYxL2JKZfD1R/CoCnga2Tsx/MZ/8k87L6GhNcKKVpe9l1h039vYC1MJQSGQNAVHn74YYYNG4bb7Wby5Ml88MEHSds//fTTHHLIIXg8HkpKSrjsssuora019j/++ONMnz6dnJwccnJyOPHEE/nss896+zZ6FYsll8TulioqK7fW0e4PUmmyVOhkpdnxOu3GM98iKsKvbbJEyOFl1TELCV2xzCIotHYSHqeNoXkeynM9jCvJ2NvbEwgEA4BuiYquDhDLli1j8uTJuN1uhg8fzqOPPpqw7XPPPYckSZx11lmW7b/5zW+QwhNd/ae4uLg73e+XRLsfRaMXO5IlKVJ1uz3A4HBWKJ1Sk/uTJEUqdJstD/FQVJVASDGKw8ULNLTGJSQhbwRc/hYcdwuqZKN4+2KOfPt02Lg0bvOd9e3safYZLly9QVAEagsE3eL555/nhhtu4JZbbmHVqlVMnz6dU045hW3btsVt/+GHH3LxxRcze/Zsvv32W1544QU+//xz5syZY7R57733OP/88/nf//7H8uXLKS8vZ+bMmezcuXNf3VaPY84wZ/K2NCwYXpedUUXpvPlNFT9/4Sv+8vY6Khs0UTHRZEnQYx/iuT/pl9At1+0ZQ7E702L6IkvamHLraQfxh3MmWAK7hRuUQCBIRJdFRVcHiM2bN3Pqqacyffp0Vq1axc0338z111/Piy++GNN269at/OIXv2D69Olxz3XwwQdTWVlp/Hz99ddd7X6/Re5MVMQJ1G4PKEa6Vx132I9WkjRTeWYcS0W84m+KAsvW7eHzLVr2ESXOAKkmc3+KxmaHY2/ki5P+Q2vGMNzt1fCvs2HxjeBvszQ1VwPvLUJKaukeBQKBlXvvvZfZs2czZ84cxo0bx4IFCygrK+ORRx6J2/6TTz5h6NChXH/99QwbNoyjjz6aq666ihUrVhhtnn76aebOncukSZMYO3Ysjz/+OIqi8O677+6r2+pxoj0soxdoZAk217TyympNOK2tauGz8PN26og84zh/SG8fR1SE/9cFh80mxV2QMh9jT1DrQiAQCKLpsqjo6gDx6KOPUl5ezoIFCxg3bhxz5szh8ssv5y9/+YulXSgU4oILLuC3v/0tw4cPj3suu91OcXGx8VNQUNDV7vdb4sVUWPeHG0jgcdoMkbGjPuIylOeNZPuwybr7k2apqDFZKpKFFzR3BIHoirBx3J+Sd9egKW8in570CttHXqBt+Ozv8NixsGuV0WZfDHLBBEGUAoEgMX6/n5UrVzJz5kzL9pkzZ/Lxx/GLXk6bNo0dO3awePFiVFVl9+7d/Pe//+W0005LeJ22tjYCgQC5ubkJ2/h8Ppqamiw/BxLRbpVq1HZZlvjvyh2EFMhPd1raDspOY0SBF4CjR+YDVpdX45zhc+mWB3uCgcP8TI1uIjSFQCBIRJdERXcGiOXLl8e0P/nkk1mxYgWBQGSV/M4776SgoIDZs2cnvP769espLS1l2LBhnHfeeWzatClpfw/0QWRfYh5gJCLWCn2yPHV4LvNOGmW0lyXNyG3EVFgCr1X8QYW6Vj9ba1u549VvWVNp/WzNrk7xREXKcQkqKPY01h12B1zwIqQXQ8138I8TYdmfIRTcJ4OcOaZCSAqBIDVqamoIhUIUFRVZthcVFVFVVRX3mGnTpvH0009z7rnn4nQ6KS4uJjs7m7/97W8Jr/PrX/+aQYMGceKJJyZsc88995CVlWX8lJXFKw63/4herDCekeH/JODTTZpl4spjhjN1eMQ6UZDp4voTRnHNjBEcM0oTFUZyjjiWigy3nYIMF0NyvXH7Ei9jVKL3AoFAoNMlUdGdAaKqqipu+2AwSE2NVjzto48+YuHChTz++OMJr33EEUfw1FNP8dZbb/H4449TVVXFtGnTLMF70Rzog0hPYvZznTg4i6H51liJaFN4dlhU6Pxw8mCKM9NM7bXVKq9Lc4dqbNcEYCCk4A8p/HP5Fm588St+93oFOxva+dvS9ZbzxRMQFgtHyprC1HDUiTB3ORx0JihB+N9d8MQp2Bu3pHayvcBiqeh9byuBoF8RPRFVVTXh5HTNmjVcf/313H777axcuZI333yTzZs3c/XVV8dt/6c//Ylnn32Wl156CbfbnbAPN910E42NjcbP9u3bu39DPYyqqpY4Cm2b9r/+6NnZ0M6eFh92WWJEQTqXHTWUsyeVctlRQ3HIMl6nnUPLc4zPVY5jhdDPKcsSh5RlU57niWkDVlEhLBUCgSBVupVStisDRKL2+vbm5mYuvPBCHn/8cfLz8xOe45RTTjFeT5gwgalTpzJixAj++c9/Mn/+/LjH3HTTTZZ9TU1N/VpY6OSnu3A7bWypicQeyFFBe7npTjbVtBr7o9PFSpKEhITHqW1vbA8QUlSWrdsDYFRu1QmEokz3cYo3dcf9KXqgxZMLP/onfPU8LP4l7PiMwc+fRMvEm9g1/Medfhe7izWmQtgqBIJUyM/Px2azxSw6VVdXxyw26dxzzz0cddRR/PKXvwRg4sSJeL1epk+fzl133UVJSYnR9i9/+Qu///3veeedd5g4cWLSvrhcLlwuV9I2+4t4hluzhff/fbWL/1u9C4AxxRlIkvYsP21iacJz2vbiOWgWEp3F6wkEAoFOl0RFdwaI4uLiuO3tdjt5eXl8++23bNmyhVmzZhn7lfAEzm63s27dOkaMGBFzXq/Xy4QJE1i/fn3MPp0DeRDpTfQgazPROcvN8RNajIXVaCVLmsuU16lZKpraA/iCWgG6eJNqvR2EV90sgdopppRNFUmCQ86DIdPg5Z8ib/2Qg1beRsGupYQGP4o9q+ezgplFkwipEAhSw+l0MnnyZJYsWcLZZ59tbF+yZAlnnnlm3GPa2tqw261Dk82mPV/MLkJ//vOfueuuu3jrrbeYMmVKL/R+32HO/CRJWi0KX0DB7bDR1BHg9a8qjf3jB2UZiy2yHFl4Kcl2U9XYYTyfbLZ4YiC1h5eURFQIjSEQCBLRJfcn8wBhZsmSJUybNi3uMVOnTo1p//bbbzNlyhQcDgdjx47l66+/ZvXq1cbPGWecwXHHHcfq1asTWhZ8Ph8VFRWWVauBjDUXuRQzEOgrT/rmXE9EVOjB2GZdYZMlbLKEJ2zBaOqIxL+0+mKrW7cHQhY3J/PEWx8wzVaHHlntzy6HS/4fdUfdjiI7KKj8H7a/HwVrX9/7c0eRcjpcgUBgYf78+fzjH/9g0aJFVFRUMG/ePLZt22a4M910001cfPHFRvtZs2bx0ksv8cgjj7Bp0yY++ugjrr/+er73ve9RWqqtzP/pT3/i1ltvZdGiRQwdOpSqqiqqqqpoaWmJ24cDHfOCy2eb67j55W+4f+l3ACz7bo/hfjn/pFHMGBNJUCJJEiXZbrI8DkYXZVhcnuJZKlJdEDFbe6NPI2IqBAJBIrrs/jR//nwuuugipkyZwtSpU3nsscdiBoidO3fy1FNPAXD11Vfz4IMPMn/+fK644gqWL1/OwoULefbZZwFwu92MHz/eco3s7GwAy/Zf/OIXzJo1i/Lycqqrq7nrrrtoamrikksu6daN93ein/vR7k85JkuFnjZWGywirmlmS0Vje8BwY6pvi61ZoajQ0B4g1+MkEJXiVY3j/tRjBaplmcZDr2ad93DGf/oLMhrXwXM/gUMvgu/fA66eKdoksj8JBN3j3HPPpba2ljvvvJPKykrGjx/P4sWLGTJkCACVlZWWlOSXXnopzc3NPPjgg/z85z8nOzub448/nj/+8Y9Gm4cffhi/388Pf/hDy7XuuOMOfvOb3+yT++pJIrEO8Pf3tQQkiz7cwu2nH8y7a6oBuOCIcubOGMl7YRdU0J7nB5dmWd6Hws9wW7yYihT7Y42pSCwwBAKBwEyXRUVXB4hhw4axePFi5s2bx0MPPURpaSkPPPAAP/jBD7p03R07dnD++edTU1NDQUEBRx55JJ988olxXYGV6AJF+qqVPkDkmVIS6sHY5gFJd39Ks4gKbV+DqbbFYeXZrN7egKJCXauPXI/TsqoPETHRpToVXUBRVVqzx/DZiS8yfeffcX7yIKz6F2x+H855DMqP3OtrmGMqekwQCQQDhLlz5zJ37ty4+5588smYbddddx3XXXddwvNt2bKlh3p2YKA/I3c1tFu2+4IhVm7V4teOGV0QIxSirRHmd/FiIVJ97FpiKkwWbCEqBAJBMroVqN3VAeLYY4/liy++SPn88c7x3HPPpXy8II6lQo+pCL83uz/pBe+ig/NskoQ3HKjd5g/hD2oTa71g3qHl2bw09yi+v+B91lY1U9saYGSBdVUfIq5DoZ6KqYhCFyiqzYlvxh04x54CL18NDVvhiVPgqBtgxk1gdyY/UQJCitrzrlsCgUAQRn9GfrShxrL90011hFRIc8iU5aQZFmS9fbQxwvxkileDItVnV0JLhcj9JBAIktDl4neCA5NYv1fre321SXeDSndH9KQv7K5kHjxssoTNFrFUADSE3Z7q27X/9WDvggwtGL6uRau6HW2piOf+lEhUbKhuZm1VU7hN9HniH2S+nKIAQ4+Cn35EaOL5Wv7XD++FfxwP1RXxL9oJwZhcj906jUAgEMRFf4Z9taPRsv2Nb7QkJ4Nz0rCFH+Jma0W05cL8jDTHVxRmas/ospz4KWSjSVb8TiAQCBIhREU/Jcb9SY42k0fet/tDMW0kSTOty5KEx2mtVaGLi/x0baAyREWrtj16Eh5xfzIHO8fOzBVFZUtNGzvq2llb1WTxHdbOE/9ezWLFsIa4M9l89F/4atrfCLlzoOpr+Pux8MkjcfLUJidGJHXpaIFAIEiOqqrUtfnZ2dCBBAzL14rSvfGNlvWpNDvNmNybLRBuh816ngTnnzg4mxljCmJShyfCPD5YArOFwBAIBEkQoqKfEC0iErk/AUwfnc/RoyI1QdLCA5NZVMhhMzsQR1Ro/+dnWC0VetVtvfq0bh1RVT3NbKQ/yfKyg+ZbnCg2I+Y4k0YwH+MLhqgefDLrf/A2jDwJQj5489fwr7OgcWfcc8Uj2p1LxGkLBIKeJKSorNmlWWhHFKQzoiAdiDxrB+V4jMm9LYmoSIbdlvpwb46jsLo/CQQCQWKEqOinxKaUjbx32W24HTZuPnUsQ/M8/GjK4HAba3t98NLjKvRYCn2gK0jXqtcOztEqce9q1IIM9Ym9zTQyKWrnaVnNc/dPN9Xx57fW0tAeyTSVaDJvdauKvNZrS7Q6C+CCF+C0v4I9DTYvg0emwupnYM930LAd2uog0B73IkqK4kYgEAi6g6LClzsaADikLIviTGt9pUHZacZCkd1mfpZHDeE99GhKVFFbpJMVCATJ6FagtuDAJ/rRH88v9oRxRQzPT48cEzWQxLNUeJw2asKxE4VhC8XIfC1ta02Ln1Z/kNv+7xsqdjVx99njjZ58taOB5o6g0S/z5F9RVDbsaTHqZUAkreJzn2/n6mO04oeJJvPxamIARmpbf1DRTDeHz4FhM+ClK2DXF/DKT+OeD4cHHGng8IIjjXSbm8khByF7GorNTXpGFnjTTe0ibSPbPOD0xG5zpGnCRj5w9HxjewCXXe7SqqdAIOg5tte1sXpbAwDTRubR1B607B+UnWZM9M2LNbHuTz2jKhJV1BaSQiAQJEOIin5CZ4Ha8XKWR6cjTHfZqQu7MElm96dwytlmX5AtG+to9YdId9kZGvb7zUizk+d1UtvqZ9OeViMWYv2eFkYXZvD5ljoeXbaJI4fnMefoYYDVKlHb6mdbbRsuR+xEe3ttm/E6FUuF+byBcLYqv7luRv5ImP02fHgffPEv8DVBoA1CptobgTbth1pA+yPJiX/p7mNPswoNpydWfDjibUsDZ7SAiSdq0kDuXCS0+IJ8vllLWXniQUU9fZcCgSAFnvl0KyowqSyboXle6lsDXHHMcP69fAuDsj2ku+xxYyqiLRU9ZURNVPxOGCoEAkEyhKjop0SbqeOZraOFhttuY+qIPGO7PnjpMRf1rX5e/WoXAKdOKDYsGDZZoizXQ22rn4831hrn217bzqjCDF5epcUvbKtrNV3NFFwdVgLRRfNAEzI6iSwVoSirh44uJgJBBVVVI5+BzQHH3qj9GCcJQrBdc4EKtGn/+zVx0dDYyLbdtdhC7cjBdko8KtmOYKRdoC3ctt26LRC1LdgRuV6wXftpr4t7Tz2C3R1HfHgsYkVSnYz22agrmgbjzhezBoFgP6A/N2dNLDGeU0cMzeXQsmxDTOgWA7PlIMZS0SvuT+KZIBAIUkOIin5CQYaLnfXtOKJ9bJMQLSpkGUt2EH0w8YRjKtZUNlHX6sdpl5kxpjBS+0KC8lwPq7c38PmWyCR5W10rn2+uZXeT5i7V1B4pmmce/HSxoAdcdwRDxr42f8gQBCm5Pyn6uVQjYBzAF1SSu/fY7GDLiFuBu6Oxg2pnJNWjtyiD7LzUUjNaUEJhkREtPhKIEH+cbfHaBdrB3xoWLqbiWcEO7ae9PmGXvOGf8g3/grrFcOpfIUNYLASCfUlLePGkKNNtmcQ74wRXm12cYmIqeohERfREnQqBQJAMISr6CfnpLg4fmmu4KqVCTHXWqPd2I1BbO+dX2xsAKM1y47TJRntV1URFNJtrWtlQHbFOtPhCBBUFuyxbPH+jszw1msQHwJ4WH4UZ7oQpZdU4KWUDUWljAyEFp01GkroebBiKEjPd9luWbeBK1356C0XRhESgHQKt8QWMSaw0NjXSXL2V0k0vIFf8P9jyIXz/jzDxx8JqIRDsI/S03l6XPeG0Xa87YV4skfdBEQlroHavX04gEPRhhKjoR2R5HF1qH23Wjo6xkGUJSYK0sPWiLaBN1EuytGxPZlExqjAdl13GF4xM5nc2aO4+XqdNszigrchlpzmj4iCiREWbVVTsqG+jMMOdsPidJauULipC1rYdAYWvdtTisMl8b1hu3PMkIjr70wGd/EmWtfgMpwfI67R5U10b66qa2TH8XI78+nao+gpevhK+fQlOvw8yS3u/zwLBAMYXDBEIP2M8LltCdyN9ch9TjLMXsMZRiEBtgUCQGgdOChrBPifW/Sl2yJBlifx0p2VbabaWSlYf/BRVxeuyc1KCQN9TxheTE66+rVshrO5P1vYNUZaK7XXtlmM2VLewfGOtEYNhtXpo/weC1oG3sd1Puz9EU3uAjkCIrhAtIg5kTdFdWnIOgiuWwvG3gs0J370JDx0BXzx1gKsogaBv0+qLPI/SnfaE1gD9eZvj0Z6lNltsw/xwRr7sLi4wRWN2c7IMC0JVCASCJAhRMQBIlL002jJhjyMq7LLE6KIMy77SbKulQmfmwUUUZDgZnO1mwqBMAGaMLmDmwcXkhgc5PVVivDgInWj3p211WgYo3aKxu6mDVl+Q5g7tXGZLwva6Nr7YVm/N+IR14G7xWdM1dkaM+1N/nWTbHHDML+Gq92HQZC0z1qvXwb/OhoZt+7t3AkG/pDX8PHLaZBx2OaGo0LcPyfMypjiDI4fFWiIPLs1kTHEGEwdn91j/REyFQCBIFeH+NABINBBEi414lgqbJOG22xhRkM663c2AljMdIpNrfY7tcdi588zxuO02GtsD+EMKQ/M8NLUHyfU6YU8rTR1hSwWxLks6uvvTkFwPW+va2Fqriwptvy5CjKDsqDl+XYvfyFi1o6GNP7+5jqCictJBRZw1aRAtHUHy063FpZIR7Z6VKLajLxJXHxWOg8vfhk8ehv/dDZv+Bw9PhZN+C5MvP6BqbAgEfR19ccTlkJGlxNmW9Oe4nm0vHg6bnHBfVzB3wSIqhKYQCARJELODAUCigSAmUDtOQ11o6FWzAfLCrkz6fNQ86XbIMoqqku6yM2FQlnGN7LDJvjmO+5NuCVBRuf/d73jz2yoAxg/KAjR3qMb2gCE+9PZK1P+grfo1tvsN16jPNml1NXxBhWXfafUzumqpiI6p6J8OUFHizmaHo66Hqz+CsiPB3wKv/xyeOgPqNu2/TgoE/YxWv/Y8SnPISJJkeV6bE2/sg5js+AghIRAIUkSIigFAomxHMYHaCdyfAE4+uBiv08bhQ3OM82WlaS5Ner0KHX1uapMl4xp626aOWPcnPe6wsT3A1zubjO2FmS6KMzWLwra6NsNCoMRYKvT3CnctXsOtL3/Dxj0trK9uZmNNJPtUc0eQjmDIWBlMlWhN0Z+8n6wWo6h9qspO+yBaL/h/WkYohwe2fAAPT4PlD2spcgUCwV6hL3K4HVqQtvm57HXGpvje11iCtvdLDwQCQV9BuD8NABINBDGB2nEGLX2emet18tcfH4JNlhhbkkFxpht7OIf6sHwvIVWlqT1oqUVhNuVHREWs+5MuCiobTcXhgEy3gyF5XqqafGytbUVRVRRFNSa/uqjQ36/b3cKeZq0y9rznv4x7z3uafaQ5bCiKmnI6xuiYj+5qim92NhIIKRxa3uP1uXuE6Pva1dhBxS5N5J145NUw+mQtxmLLB/DWTbDmFTjjQSgYvc/7KhD0F1o6IqLCJkn4TOre67Kzp1mr87MvRUXiuA4hKwQCQWKEpWIAkGgwSiVQ21zl2i7LSEjYZdkQFAB2m8zY4kwtbsKEJMWxVMRzfwpP2nebREWG287pE0uYMkSbgG+ra0PFGjQdbalYsSV+dWqnXWZouFjdnmYfqmotsNcZ+vnNKXS7iqKoVDV2UNvip83fNUtJb2K+l+jYloY2v7Vx7jC45P/B6QvAmQHbP4VHj4YP79MqkgsEgi6jB2q77TYkyVqHIs1kBZb24WidKI2s0BQCgSAZQlQMABKmKJQlS8xtvJX76FV6rV2C80nR7yPn97j0qtzNLF1bHbdORVW48vaJ44p4ds6RjCxMZ8pQrabE7iYfiqJa+hNSVVRVs1z4gwort8avHJ3uslOQoaXB3dPckfC+EqF3Vf98ElX2TnoO0+sDNdA7ultxA/wlCaZcBnOXw4gTIOSDd34DC0+E3Wv2RTcFgn6F7v6U5rQhy5Ll2WSuqL3fQipEnQqBQJAiQlQMAJINBLolIV7Oc7CumkUfE3OdOMX0IpaKiKfdM59to8pklYiICq0exZjidMYPzkKSJIYXeAEtjWxIUa1iJOwKpaLyz+VbaPWHyE5zsODcSfzurIO5+pjhpDlkLjqynMJw/vbqsCtBV0SFfk373lgqDtBADKvYsfYx6apkdhlc+CKc+TC4s2DXKvj7MbDsTxAKJDlQIBCY0UWFyyFjkySLNdZpjwzR+9L9ySELS4VAIOg6QlQMAJL5weouPfEyP2kHx25K1Z1KMsVU5HqdXH7UUGOf2VVJn9/rQqMsJ5IScVB2GnYZgopKVVN7jKUipKpsqG7h0811yBJcecxw0l12SjLTmDI0l09uPoFjRhdQEBYVe7ohKvRBXr8XtRtRFckqiO9PzH2J7lanEwhJgkMvgLmfwphTQQloKWgfOw4q48e0CAQDiUBIiZM9zoru/pTmsCFL1meTyyQq9uWEviDDRVGmm9FFGVHXFapCIBAkRoiKAUCywUgXAvEyPwFMHJSFw279miQSINGbZVlCt94rCkwbkc95h5cBsGJbA8FwvEZIUQkoCrUtmg//kLyIqLDbZErCdTG21rYZmaL041QV1le3AHBoeTajizIsfXDabLjsNsNSURO+RnRBu2TosQZ2m9VSEQwpbK9rS6lCtyV2QUncbl/TI/omswTOewZ+sBDScmH315qwePd3EPT1wAUEgr6HP6iwbN0ePtlcm7RdS7gwp9ths8ShgdX9aV+uRUiSxITBWZTneaK277s+CASCvocQFQOApO5PYTGRyPqQl+7i2NEFZIcrYkPigMHomAxZirWSHDlci5H4dmcjb6/ZTXNHAEVRqW7yoQJuu2zUwdAZnK0NbNvr2i1iQFGgpSPA5nDa2OH56TF9sskSDptMUWbEUqFdM/49RKMoKkvW7ObLHQ3GqK53YXNNK+uqmhPGcljOo1otLPubjkCItVVNNHdEXJViLBVdWZWUJJjwQ7jmUzjoLFBD8MFfNJeoHSt7ptMCQR+irlVbwGjzJV90sKaUhXElGWS47UwcnIUsSwzOTaMgw4XXtX+SNYoq2gKBIFWEqBgAJEudarg/dZJe1bw/kaUiXqB2dNvhBenkeZ0EFZV1Vc20+IIoKuyo16pmD8pJQ46KBB+cq1kqHvzfBh7+3wZj+zsVuznsrndYta0BIG6qVodNs5ZkpTkZmudBBT7dXEcwRVXxv3XVPLpsE39buoE/vbkONfwPIpOGdn/nlgqzB8SB4P707a4mdtS1G9YhiHXrMv/qojNDJSS9EH78T/jxU+AtgD1rtSDut2+DQHtPdF0g6BOk+nceyf4kI0sSHqedI4bnUZipJZcYW5zJIWXZvdXNThF1KgQCQaoIUTEASClQuwuiIpFVI3q7JFkrwoLmNzyuRHNRWlvVRDCkxUVsr9MmnGW5aTF9KTfFWPzjw83GBPflVTst7c46tJSxJRH3J5tNQpIkMtyalWXayHwAPt5Yk7KlYntdm/H68631bNzTaqzom1cOzX7TTR0BNu1psfhGH2gxFU0dscHUybrVlRgUAA46E675DCb8WPP3+vgBLf3stk+62FOBoG+S6l+Mvjjhcdr3mzUiVUSdCoFAkAwhKvoxIwrTkWUYXZyRsE13LBWJCyNFHSNJZLgclm1uh40x4f6s292iFbRTVbaHLRWDczwxIuiQsmyLFUSvyu2NEiwepx27ycqhZ2sqzHAxsjCdK44ehk2CbXXt7GxoIxXq2qyT73cqdhuTBXNmllZT7YnPNtWxaU8rW2sj1bzNcRSpCpreJN6vMDalbIRuuWx5cuEHj8P5z0FGCdRugEXfhzd+Bf7Wzo8XCPowqVr36sOioiw3zfJMOVAQlgqBQJAqB94TTNBjDMv3MmN0IZluR8I2RqB2JytQFvenBAIk2lIhS1I493pkm8suM7ooE4Btta00tmsxFTtMoiL6PGW5Hh44/1BywnEdDe3aIKyv8AGcO6Uspm+6wJAkiaH5XoYVpFOcpblS7Yqq3p2IulYt0HhS2P3gi631RhyC2eKg+0WbMW870GIq4q04Rk+CzMaJverymFNg7idw6IWACp8+Cg9Phc3v78VJBYIDm2SFJXX8QYW2gPacGJ7v3Rfd6jLmmAphqBAIBMkQoqKfkyyeQttv/T9hO6nr7k/6e6ctYlFwhgOx89OdKCp8taORpvYAje3awDo4Jy2mL7KkVZvNT9eCretbA6io1IXjAe4+ezxnTioFrFXBHXFqbxSEz1HdlKqo0K4xrjiDwTlpKKoWjwBWi0NLR6yoMH8e0fU19jepWCosQmhv+5yWDWc+BBe+BFll0LAV/jkLXpsHHU17d26B4ADErCMS/fm0+0N0BLQHSXZUgooDBaulQqgKgUCQGCEqBjipuj+Z9yZqGhOoHf52mU36+qA0slDL1FSxq4ltYStFYYYTt92WUJzkhgfd+jY/LR1BAuGROtfrNMSTuYhfvHsqiCqC1xm6qEhPc3BYOBD8q+0NgHXS3RzHUmHmQAvUjqcLo1Pd9kocyMgT4Kcfw5TZ2vsVizSrxYZ3eub8AsEBgjnxQaK/n/ZAyEj04HUe2PEUICwVAoEgOUJUDHDSHDbL/4mwrFalWFHbEc6xbk5Hq6uTIXmaqf+76hYqdjUDWmYoiB/wDZCXromKhrYAdW3aZD/X48Qhy4bosVoqYr/eheHUsrsbO9jV0N6p1UAXFRkuO4cP1UTF1zsbAetEobPK46plgp70kvuEeCuO0dmfLCutPRkH4s6E0++FS/4f5AyFph3w7x/AK9dAe+fpeQWCvkAq7oON7QGC4YYZ7gNTVAgdIRAIUkWIigFOWY6Hw4bkWKpYxyOVrB9mw4DDLuMOC5Xh+V6Ks9xMKo8EXA/J1a63sbqFb3dpk/SDSzNjzmO+dq5HExV1rX7qW7W4hqIsTSTEs7jY47g/6fUqKps6WLOriU3hGhchReWbnY2sq2q2tNdTrmqiQquxsb2+nermjk6tD+aPzNw2pKj73QUqrqUiqku9Hgcy7BjNanHETwEJVv8bHjoS1r3R89cSCPYxibK/malu1twwJUga+7Y/ERmfBAJBqghRMcCRZcniPpSIVIYV88q8edXNbpMZPyiL/HSXMUDplVrr2vxsr9fSyR5cmqVdK8b9Sftfd39qaPcblgrdnUk/xpr9KfbrXRTO/a5nXKlt0dygVm9voKqxg+11bcaEX1FU6sPXSXfbyU13Uh6umbFs3Z4Y96BttW1sqYlkNUoUU7GlppX/rauOm9Y1GZWN7Zbz7w2pxFSoUUKoV3B64ZQ/wOVvQt5IaKmCZ8+DF+dAW13vXFOwz3n44YcZNmwYbrebyZMn88EHHyRt//TTT3PIIYfg8XgoKSnhsssuo7bWWpn6xRdf5KCDDsLlcnHQQQfx8ssv9+YtdBnz33wiTV4TdsPMdNs7fQbvL8y9EvpCIBAko1uioqsDxLJly5g8eTJut5vhw4fz6KOPJmz73HPPIUkSZ5111l5fV9BzpGapiLRJtOqmt3DbbYbVAKA8N804Jl4RPTCJiraAEaStB17rx9hkyRj47HEG6ZJMTRToYsFuk2hsDxgiAyKTgaaOgGFhSHfbsUkSEwdnA/BuRbXFpSkYUvludzMbqlsi92qxVMS6FpnbpsK3O5vYUN0SN9NUl4lrqbD2MWSZFPWyZaX8SLj6QzjqZ1rJ9q9fgIe+B9++0rvXFfQ6zz//PDfccAO33HILq1atYvr06Zxyyils27YtbvsPP/yQiy++mNmzZ/Ptt9/ywgsv8PnnnzNnzhyjzfLlyzn33HO56KKL+PLLL7nooov48Y9/zKeffrqvbqtTzEI82rVQpzq8qJHjOTCDtEEEagsEgtTpsqjo6gCxefNmTj31VKZPn86qVau4+eabuf7663nxxRdj2m7dupVf/OIXTJ8+fa+vK+hZMlPw9zUPPonam9sMM6VQPH5skfE6UUxFdnjgrW/1G7EOeWGhYclOFRYT8dyfirM0S0Vje5CgomCXZToC1orY+lxAv4bbLuO225AkyUgt+8H6PUbWFiBuhW6zpok3J+/K8Gx2lwrFid/oKvFjKqLemy0V+yK43JEGJ90Js9+BgnHQugdeuAT+czG0VPf+9QW9wr333svs2bOZM2cO48aNY8GCBZSVlfHII4/Ebf/JJ58wdOhQrr/+eoYNG8bRRx/NVVddxYoVK4w2CxYs4KSTTuKmm25i7Nix3HTTTZxwwgksWLBgH91V51jdn2L3B0MKjeHFjRzvgen6BNYFJWGpEAgEyeiyqOjqAPHoo49SXl7OggULGDduHHPmzOHyyy/nL3/5i6VdKBTiggsu4Le//S3Dhw/f6+sKepa8dBcHD8rke8NzE7YxT+zTE4gKc5vTJpQwY0wBvz5lLEeHq11HtzG/1wO+O4IKn23RXGPKwrEZ1voU2ut4gdr56U70zU3tQWyyFEdUaDMAI/OT244kaSJhSK6H7DQHrf4QFZWRVKjxA5njuz8Ze7swQgfMF+iBgT3epWOtKfspuHzwZLhqGRzzS5BssOb/4KEj4KsX9rJgxr5BUVR2N3XEfK8GIn6/n5UrVzJz5kzL9pkzZ/Lxxx/HPWbatGns2LGDxYsXo6oqu3fv5r///S+nnXaa0Wb58uUx5zz55JMTnnN/0Fn2NF9QocWnfUdyva6Y/QciQlQIBIJkdElUdGeASPTwX7FiBYFAxKf8zjvvpKCggNmzZ/fIdQU9T0lWWtJgQqddJsfrIDfdiSeF9IglWWlceMQQRoazPulIUd9KXRzYZMlIRasd72ZUoVadO80ZyV6lC4x47k8Ou0x2Wjjgu82H3SbhC1oVQayocGhuVUhIksSIcB92hGNBEtHZpLwrLtQWV4p9NLG2BKLv68ByuwuOvxWu/B8UT4D2OnhpDjx7PjRV7tu+dJHt9W18vaORz7eImJCamhpCoRBFRUWW7UVFRVRVVcU9Ztq0aTz99NOce+65OJ1OiouLyc7O5m9/+5vRpqqqqkvnBPD5fDQ1NVl+ehPL32ycRQdFVQ1XxuwD2P0JImJCuD8JBIJkdElUdGeASPTwDwaD1NTUAPDRRx+xcOFCHn/88R67Luz7QUQAk4fkGvUc4pHKSle0pcIVrnPhDypcecwwY/tJBxUZ1WjNKXGH5HnJz3DFHahtsmQU0dvd5EMi1lKhTwa21mpB0Zkuu9ancLf04/d0UuvCPA2Pa6nowgAdNLs/9cAEP26gdrLsT/srW1XJIXDF/+C4W0F2wHdvaFaLVf8+YK0W+vfCF+jJPLx9m2irnKqqCS11a9as4frrr+f2229n5cqVvPnmm2zevJmrr7662+cEuOeee8jKyjJ+ysrKunk3qWEJ1I4TU6Go0BoWFTmeA9f9CUyiQmgKgUCQhG4Fanf1YR6vvb69ubmZCy+8kMcff5z8/Px4h3f7uvt6EBF0TioT6egVfGfYUqGqUJjh5trjRnL6xBKmjcgz3I7MomJQdhqTyrLjFr+TJYnBOVqw9o66dhRVjWOp0P7fuEcTFV6X5v4kGaJCEyt6OshEmOe8cWMqujBAm+tg9ER8Q7y/m32eUjZVbA449pdw1ftQehj4GuH/rtFqWzRs33/9SoCYeEXIz8/HZrPFLP5UV1fHLBLp3HPPPRx11FH88pe/ZOLEiZx88sk8/PDDLFq0iMpKzUpVXFzcpXMC3HTTTTQ2Nho/27f37ncnZHqsxNPkqqrS0hEWFQdoNW0dYaEQCASp0CVR0Z0BItHD3263k5eXx8aNG9myZQuzZs3Cbrdjt9t56qmnePXVV7Hb7WzcuLFb14V9P4gIOqc7lgpZlnCErRXBkMqksmzOOWyQJWWs2f0pGTbZJCoa2lBUNcZSoYvetnCl28w0O76gYgyruqWipiW5paIzn+ouiQpTTEWPFqIzkaz4XW+6XH2zs5H1u5s7b1h0EMxeAif+Fmwu2PiuVo17xaLe+1AEe4XT6WTy5MksWbLEsn3JkiVMmzYt7jFtbW3IUemgbTbt71v/Hk6dOjXmnG+//XbCcwK4XC4yMzMtP71JZ3UqFBUjpiL7ALdU6AhpIRAIktElUdGdASLRw3/KlCk4HA7Gjh3L119/zerVq42fM844g+OOO47Vq1dTVlbWrevCvh9EBJ0TLRjiEa+JMyro2hyELcsRF6nOsMsSg8OF/nbUt6MomlsVYAgXfTLQEM7Mkul2EAqpRt/12hh7mv0kozNLRVfm6T1tqYg3yUnu/rTXl4xLRyBEVWMH2+raUjvAZoejb4CffgRlR4C/GV6bB0+dAXWbe6eTXSbxd7zFF2TVtnoa27tWo6QvM3/+fP7xj3+waNEiKioqmDdvHtu2bTPcmW666SYuvvhio/2sWbN46aWXeOSRR9i0aRMfffQR119/Pd/73vcoLS0F4Gc/+xlvv/02f/zjH1m7di1//OMfeeedd7jhhhv2xy3Gxfz309wRjLFsKqpKq0/7HhzIKWUB4ystrHACgSAZnUfTRjF//nwuuugipkyZwtSpU3nsscdiBoidO3fy1FNPAXD11Vfz4IMPMn/+fK644gqWL1/OwoULefbZZwFwu92MHz/eco3s7GwAy/bOrivoGyQak2w2yUiVGk94uBwyrSbDgMdpxxfQU77aUs6kJEkSg7I1S0VzR5Cqpg4ktMEyzWEjEFQMVwW9lkVmmh1Zjgyoer2MFl+QjmAIVVVZ9NEW3HaZ0yaWUByuhREdk7Biax3+oMK0EfnGtmga2wM0tQeMrFbm43V6Img6rqiIaRP/+j2J3g9V7dyd0UL+KLjsDfjsMXjnt7DlA3hkGpxwB3zvSk1p7ieS3cLqbQ10BELUttRx4kGJraz9iXPPPZfa2lruvPNOKisrGT9+PIsXL2bIkCEAVFZWWlKDX3rppTQ3N/Pggw/y85//nOzsbI4//nj++Mc/Gm2mTZvGc889x6233sptt93GiBEjeP755zniiCP2+f0lwvw3szFck2ZsiWIsalgDtQ9sS4UU55VAIBBE02VR0dUBYtiwYSxevJh58+bx0EMPUVpaygMPPMAPfvCDHr2uoG+QaMKV5rDREtIG2HiiItpSMSTPYxSss8dJHZsMp12mKMPF7mYf3+1uojQrjVxvJNWsPtFtaNNWEQsy3BxWnmP4Fac5bGS67TR1BKlt8fPmN1Ws2tYAaJW57z13Eo6oSW2rL8CjyzYBMKzAS0lmWlyLw+ebtYxBDpts1NQACJhMBT0ywU/g422mM/etnsCSYUqFOKVFEiPb4MifwuiT4dXrNWHx5q/g25fhzIcgf2RKp9lS00pIVRkRlYWsuyS7hYGaZnbu3LnMnTs37r4nn3wyZtt1113Hddddl/ScP/zhD/nhD3/YE93rFeL9fa+tbKY0Kw1ZllBVaA27Px3olgpN7KvCUiEQCJLSZVEBXR8gjj32WL744ouUzx/vHJ1dV9A3SLQSne6yG0GL8VKtmt2bvC67EdfQXcryPOxu9vHq6l2sr27h++OLmTNdq49iiIqwi8qhZdlke5zGhFBFZXBOGmsqm/l4Yw3LN9Ua520PKDS0BijIcKGoKoqisr66hS/CogPgm51NlGSmJbU41LX6LaLCLCRScX9q9QUJhBQy3Q6jGKCZoKKyta6VQdlpRmxK9GnVfSIqrNewdWclNHc4XPwqrHwCltwO2z+BR4+C426Gqddq4iPR9RWVjXtaUFUoz/XErW0iEHSHRH/fNa0+CjPcBEOKYanIPeADta3/CwQCQTzECCo4IPC6NH2rZVmKY6kwiYrMNK3tpPJs0pw2RhV2fYVZT3u7bncLigqLv67CFr6uomqT+OawyDFP7kGbfA8KB3u/9e1uAI4emWdkhWpo9xvtalv9bK9rY+OeFuP4r3Y0ANo1/EGF5Rtr2VzTarmGL2hd0Q6YYio6C5reXNPK8o21rNhSz9YEsQrvrq3md69V8MynEatiTEyFKY6it9yfzPn790q3yDIcPhvmLocRx0OwQxMYC0+C6oqEhymqaly3p+4xeSa8HrmE4ABHUdSE3+f2cAKI5o6gYTA84N2fxPdWIBCkgBAVggMCbzh7U6JAbpc9stqsF+DLT3dx1Mj8bqVjnFSWTZrD+vWvb/XT1BFAUVRqTZmdCsJWEb1vqgqHDM62HHv40Fwj13xj2G1KUVXDbWlrbWRyX1HZzB/fWkt9m5+Gdj+tviCVje0ETS5O/qg0txZLRSdB002mIOBEAcEvf7ETgPfX1xjbzNmfoldZe6tMRY+7WGWXw4UvwRkPgisLdq6Evx8D7/8ZQrGfhdX9qodEhfn8UR+cmJwNDMzWRBXV8t3SU1jXht03XXbZ8nw7kEk55kkgEAxIhKgQHBDoKWETjVkWS0WSqt6pMHVEHuW5Ho4YnmfZfumTn3PHq9/S2BagqknL1JLussft2w+nDLYcO6Y4g6xwpe768EReJZK1aUuUJWL97hbe/67G2B8IqZYCd+1RvvfmlLKdraibz9PmD8ZtY14Z1cWEmmSCvS/cn3rsEpIEh10E13wCo78PIT8svQsePx4qv0p4/d4QTtGfm8j3PzDQ/0YVVeWu1yq47tlV/PvTLYQUxXCj1IO3y3I8Cc9zoKB/b8W3VyAQJEOICsEBQbrLTkGGi/Lc+AOswxTBm+HuViiQgddlpyTLzQ8OG8SPJg8iKy1yvuaOIE9/tpXqJs1SkZVmN1bnzAOq12nn9IklAJwwthC7LJMVZalQVRV/SKE9EGR3uMrybaePM+pkbNzTYoiKYEjBbzJBBEPW+hlmoZBogl/d1MHW2laLxaMjEIrrLpVp+gybTCIocg1r+97L/mR+3cPXyCyF85+Dcx6HtByo+goePw6W3g1Bf8w1e0M47deigYL9hv5dqmrsYGtdG76gwnvrathc02pYKiqqmgAYWdQzCQJ6E1FRWyAQpIIQFYIDAkmSOKQsm+EJMvBkuB2U53kYU5wRN/C4q9hkiTSHnZMPLmFkVEzGf1fu4D8rtEKJWWmRFX2z6T+kqJxxSCk3nDiKS44aCkB2uG2jKaYiqCjsrG8HIMfjYEiul3OnaJXdN+xpIRC2QKhqbGagVl/EymCpU5Fggv/VjkbW724xYkFAi4uIrhgO4DMJj50NHeE+JJ5g95alItVg8HZ/iKaObtR2kCSY+GOY+ymMmwVKEN7/Ezx2LOxcaRU1PSScLOIs+qMXk7IBgf43uinKQlnV6DP+zr8LF3zsTkzYviYSqC2+wAKBIDFCVAj6DKOLMmLqN3QX8+Co15XQCYRU3l6jBWCbUz2ah9OQohXDmzIkF69TW/XXXYr0rFGKqokBfdJeGq6PMSRfu4faFr+lKnebzyoqOkxiwOL+FGfyHUgSaKEHhpoxC5Yd9Vq8h/msyYK2exLzPD7ZlH7V9no+21RHdVNHklZJyCiCc/8NP3oSPPlQvQb+cSKO//0WOeSL6cvekMz6IaZkAwP972VzTYtl++7mDnwBhVBIYf1ubd+Yoox93b2uIywVAoEgBYSoEOxzirPcuBwywwu8ABxUuu8rnUumb745vuD3Z4/nhLGFxnuPK+ImZB5Q9boUshRJgZsdjqloMLk/BUIKuxo0S4VedM/j0NyvANbsajTO+cxn23j1y13Ge/PKucX9Kc7s12yN+HpnI48s22BU8G2LsoAoimoRFeurWwgpStKYit5y47EEtCYRLrrg+mpHY0wQe5c4+Gy45jMY/0NQFVyfPMARb59BVs0XPWaNMf9+oj83Eeg6MNAXAfSsbqMKtWed/je5ta6Npo4gsgTD8r37p5NdQFgoBAJBKghRIdjnjB+UxdEj8xlekM700fnGCv6+xJxlalTYp9lhkyjMcPPT40YY+7LTzKJCilmpk2XJOJfuKlXbqlkgVDQxsDNKVEBkIvHal1WoqkplYzuPvb+JV7/cZUw8dBcKVVWNauPm7WZ8YeGwbnczD/1vAyu3NvC71yrYVNMSY6kIKYpRdAs0gfTIsk2YbQWKqhJSFF5ZvZMNe1oIhpROU9l2ByWFWJFoEZUoo1XKePPghwvhvGdQvEV4mzczZen5eJbeCv74KXi7gtX9SVgqBiKhcDzVjrDr4xHDtKQQeqzWJ+HaNoOy03A5DvzMT0ILCwSCVNi7iFeBoJvoK7b7K5WiOSxjcLaHhy84jPwMF01tAdx2G387fxJPLd/KGYcMshwXPe+VJQlb+GR6oLY/qPDrl77mxpPHMKks2xAVZvF0wrhCPttcx8pt9byztprdJree3U1acSx9kh2MmpjGc9PRLRUvrtxBUFFJc8i0B0Is/HAz44ozKcly43XZ2VDdzLqqZuOcp04oZvHXVayraoqpbv35lnpe+6qS176q5NELDyMQUnHa488uqho7CCoKg7uYySaZdUQn+v57TNyMPY2G/Cl0vPZrSre8hOeLx+C7VyFrMNicYHOE/0/02gF2V8z+vHo/npANVXZga88Gr8fYl1ndhl+1ocgOqK5Pfg0xk+uzKArsamhHUSHDZWdsiebitLvJh6qqvPF1JQCHlGUbz4++gPhKCgSCZAhRIRiQRNfDmDg4C1WFprYAiqoypjiTq44ZQUm2O8EZ9PNEBlqP0yqQ/v3pVspyPUbgtPlcQ3K9nHPYIP6zYgcfrq+xxFZUNXUwYVCWMcn2BxVWbq3D47QzriQz7uRbFxW6OPnZiaN57P2N7G7y8eqXO3HYZY4cnseWmjbDSmGT4YSxRSz+uor2gGIJBldVlT2mPq3YUs/RowosqX3NrKlsRFGgKNNNmz/Epj0tjCxMJ6OT9L+ppHQNRgV09KS9JOjKYs33/sDuslOZuOp2bC27oKVqr845PMm+SeY3Szo5UWeCJmabK/W29kRtTa99nfRPkJBASDH+Fouz3OSnu5Al8IcUdja2szxsqfje0Fz6gqYwArWFqhAIBEkQokIwIIkeG22yZExqq5t82MIpbO1ycg9BSYq4P0lIDM/3GhlfdjV08M/lWwDIT3fittuYWJaFhMSayiYmlWXznxU7DEuGzp5ma+DwN7saeWTZJuyyxILzJsWd2PuCIdoDQVrDrk7D8r3MO2kMt73yDSu3NnC6yeLS4tPch7wuO15XRAi1+iJuRYqqFQPUeadiN9ccNxJcsZ+BoqhGYGpIUfl8cx0AgVAz3xuWm/Tzs9SpSCAXot29zJpqc00rGW47+elxOpYC+rlqS46havz7DGr5RqvGHfJrxfJC/tjXQX/87eHXtU0tKEE/shIgw67glILG/rb2diQlgKT4cUuhyPFBHzFyST/v/sI7ovM2grgEFdUQFUWZLuyyTGGGi6omH89+uo2gAkPzPJRmpyUs+HkgES+ttkAgEEQjRIVgQBI9kMuSNRQxFFJJd9spyuzcUmE+1/UnjKLZF+DLbQ3894udrNhSD2huDgAZLgdpThu2KomCDBcZLjvN4aBpuywRVFQjw5E+mV5aUQ1oE5V1Vc0cNiQnph++gGJU6PU6bWS47UYMx54WHyHTar9uqfA67dhlGaddxh9ULKloVVW1WE+21LaxprKRo0YWxFzbLAzMAiBZRqrIsZHXibyaokWFfr2aFp9RQOzEg4o6vVb865ssJQ4vjDiuW+cxs2b9HnwB7d4PKs20uL2t2lBjxLjE9DkUTCpWYl93oW0nQih2mw/cg4BVe/15DESCIYXd4fgJ/RkytiSTqqY9rAtnfTr7UE3o9wFNIepUCASClBCiQjAgiScqolfKx5VkGtW0k53HZjIcjCpKJxhSKcpw882uJtZWNVOQ4YyZQMhy2LJR4OXLHVoGqOPGFLKkYrcxGdEn0x9tqDHO/+3ORiaFBYoZX1ChplkTFXnpThw2mfI8Lb6huSNImz9kBA3rmZ/Sw5mt0p026oIKTSZRETKJigyXjWZfiP/35a4EoiLy2iwkot3B4pFKnYqYmArTPXdGqy+IwyYndNvqjeJ75nNGC6KkczKbXfvhAKmw3NQEV2bt7170SQIh1Sh4WZSpWdGuOHo4Nkli6dpqfjRlMMeMLqC2xd83LBXG/wd+XwUCwf5DZH8SDEiih0ZZIqaontMW++cxNN+LwzRBtcmSxc+4ONPNyMJ0ZEniqmNHcOr4Yq4/fhTucEC6PoHQ/zcX+zt2TD6gWRb8IS3Fa1NHgDWVTUabr3Y20uILxmQV8gVD1IZFQH66C5sskel2GNmr9jT5jEJ7LX5NPHjDoiItXGej2VRcLhRSDcvHhUcOBeCtNbvjZp4yT8bNmZk6cx3Tjo3/2ow51gNSD9T2BUN8sqmWVdvqE19fiW9l2Rt6u0p3T9PmDxoud4KeIago7G7U3BoLw5aKzDQHD11wGA/+5FC+P77Y+G70BVEhEAgEqSBEhWBAEi0gbKbUsDrxVrdHFqZz5HBrnID5ONkUY5HhsnPOYYMpyUoz7df+t4dfHByu0TGuJIOiTLexun/7/33Dgne+44kPN6OokOt1YpehpsXPr/77FVtrI5V6VVXFH1SoadFEQH66yzj/kDwtde2elg5j0qy7P6WH4yn0/82Wit3NPkKKFsw965ASZAnqWwNxJ5/mibNeowNSq21hnsgnqmgdfR79rWTZFntsR0ATZq3+YMy+6HNBzxW/U5MFn5s63VMVvPeWjzfU8uX2BkOUCvaeulY/7WEXuMJ0TVTIMrjsMmlOG4oC7X5tf18I1NYXIDyuAz/9rUAg2H8IUSEYsJgX0rV6E5H3NllKmOrRbMEIKSo2s6iQY2tZWK6pBzyG/x+a5+WO0w/iqmNGICFxaNi1qabFz7trq7nvnfUAzBhdwO/PngholbZXbK0zzqlbNWpaI5YK3UpQHhYV1U0+w40o4v7kINvjIDNcX6PFXGW7TqvXUJjhJsPtIMNtD/crMvHUi9CZ58YNJkuFWTDsafZR1RhbDdsSqJ0opiLGUqFNyM2fczwrgz65V5TEVghL8b0eK36XuF9m95EDzYrRsLf1PwQGa3Zp1sU8r9NYnNCtmu5wXYqOcG2ZvpBRaWxxBtNH55PZSTY3gUAwsBGiQjBgkaItDCYR4Yjj+hTvuKCiWCa3na066m3NgmVYgdeIb7j/vEN5es4RRgyGztmHDeLHh5dx7GgtpmHd7hbWVTXT5g8ak1h9pTkv3Yk9nL1qSK7mn7+nxUcgLAJ0YZDtcTBlaC5FGdpKarNpUqlnpCrKdOG0y8ZkQk8zW93Uwfvf7WFDdbNlchwwxTmETCLmy+0NfLOz0ZhI6Zjn1YmyPwWiUspWN3ewdG01O+sjWbPiWUUSxXpY23Se0rYrRFsfooWD+btygBgqDA4wjdNn+WZnI39dsg6IWCIhsqDgcVpDGfuCpUKSpP1WU0ggEPQdhKgQDFjMbks2yer+lCiwN5pASLWIkXhuVGZ0QWK2bmS47YwrzWTi4CzyM1yMKc7gtAklTBmqZXkqSHdx5HCtIm9puNbF6u0NbK9rY/W2BmNCbXZ/0kWLHqy9p1mL0+gIhvh6pxYYrk94MsJxF2ZLxdZazVJRkp2GwyYb1oyasPvTut3NAGypaUNNEC+ti4oN4QxN0deA1Cb10av9uouVxdWqk1iPRKLCbJ3oiZiKaBGR7JwHmqWiZyuADFyWrq0mpGjxV/NPGm1sj4gK6+RcxFQIBIL+gsj+JBiwmFcIJQlsmC0VqQ30IUW1nEeWYt2f7DYpJtjY7Hpll2Uj/at+DoCLpw4h3+vilAnFhuWkNByfobsStflDhBSVtoCW4QnC2Z/CFxiWr7k/ba5pZXdjB19srScQUinKcDGyUAsS160Q5pSyG/ZoomF4njdsqdDdn/zGfeskmhwrqoovGLLEYbT5QpCuBQd7nPaUgpqjP7t4dC4qEvUxfvvuEt2N6HOm4u61vzjQ+tNX0WN4RhWkk5fuMv5mdKHvtgtRIRAI+idCVAgEaAO+eWKazP3JjNMuW2MqpFhLRa7XSXWTNQjW7FtfkGEt3Kafz+Ow88PJgy11DgblaK+rm3yoqookSSiKSm144pLusuG224zifZPKshmS52FrbRt/emudkeHpyBF5kYByt579KSIqNlZrgeAHlWbisEkmS4UmZlIRAyFFjUn72uILsnFPC5v3tDI03xtVpyKxOOkMJY4hwnxYKu5PPTGpjhER0ZfthRS2PcWB1Zu+S0v4b8zlsKYy1hcfHHbr80ES/gICgaCfIB5nggGLeU5nk6yB2Z25P00ZmkNeupODSzOjYjNi09V6XXaOGJ7L0aPyjW1FmS48LhvjooqjQewkw2w1Kcp0I0tacHZ92P0npEZERWE4PkLP/uSwyfzy5DEAfLyxlnVVmivSxEFZhttWlscaqF3f6jfSyR42JAenLWKpqNarfZsmywndllTVEmMBmoVi8x5NsGypabUIiTZ/iOrm2GBuXRDYkliP4sdURLb5E9S0SMXi0hWiTxGTuSpB/w4EDrDu9FlaOjSLodsuWxYndBEfnWpZWCoEAkF/QYgKgQA9+5M1NiIZ2R4nh5bn4HHaY7JGxaSrlSQy3A4j6wtAXrqLaSPyLW5P5vaW96bzuR02w7KxO1x5W1EiheqKszRRYfbbHlWYQUl4uz+kYJOhNCfN6Lfu/qRnhdIzS+V5nZRkpSFJEtkeJxBxfzKTyMIQCqmG25EeON7qtwZqmwVJdZOPr7Y3xqSt1Sf+jiR1L+K7P0VeRxfQi/TddI5uzqpVVY1kmuokpqI3Utj2FIkC5QVdo8WniX2PK/bZALH1b/pCoLZAIBCkghAVggFL9BTKPLjbuzDSR9LERgKxpTiTia6eL9KXyJ+pwyZRHC6mpYsKs6XioNJMjhyRR4Yp9aNNlozYCoBB2WnhCbp2new0TTDo7k9fhSt8jyiIHJPn1YRMTZxaBubJcasvyNc7G6lr9XPvknXMfXolK7bWGaIk2nIRb7W+oc0qXHRBYE9mqYgnKkzbUnN/6t6ketX2BpZvrCWkqDH3E31O88S9p1LY9hQHWHf6LHodGI/TFpNhDmK/x8JSIRAI+gsipkIwYIme1EldsFSY0S0TZguFJEUmaV0WFVFS3+z2Y5dlcrzaBF2vKxBSVCPVa3mux0hPGzlGYni+l4831obbaGIhYqmwZn/auEdzkRpZmGGcI8eriZR4BdLME+kF765nc00rg7LTjLS0FZVNfH98MW6HLSalbDxRYf49KIrK35aux2GTue74UTFtdeJZGcybErk/pVLRG7RK4c0dAQbneGL21YUFXW2rL8b1LVrsxLteMKSwcU8rzR0BJg7OTuh6Fwy7vOV5nTHWsJ5AiIqewVyx3pLEIfxrjRYVQlMIBIL+grBUCARxiPZ7ToYuGsxuS90VKBDPUmESFTaJnHDQtL6iHwypxmR/cE6sO5XdJjHMZHUYEk4zq18nOywY/CGFjXtaqA/HUxSFXaZAS2sLUN8WiDNR1t7vbu5gc40WL6ELCoD2gMJ76/bEtTTEm8iaP65vdjXy0YZa3lu3hzZf4srY8apTm4VGKpaKZOlfP99cx9rK5hhLjVmYLvpwM4ff/S6/+X/fsrWuNaYP0e31a3+9s5HtdW00tAVoaI91L9P5ckcDX25vsHy2e8uBUtW7P9FmFJe0W58D4dfRbnzCUiEQCPoLQlQIBizxplPZHgeyrKVlTRWv08bg3DRGhFO0gjVYu6uiIllMhV2WyAq7EtW3BWj2BTj7kY/YHi4EF28lXU9Zqwd867UrdDLcDg4q0awS855fTV1YVOR6Ii5UuWHriArUR7kn6fPk97/bE3Pt6SO14PRXV++KWcWHzi0Vq7Y1GK8/31LH8k01cX3/dUHQ0OZn1bZ6rShgnJSyHYEQ22rbDJGhdDFQuz1JTMhnm7VYlB317Tz+/iaCihKTDjdeoLb584yXxQo0MVLfqlmm4gWzdxfV8loIjJ5AjxvyOm0x6aaBGCuTEBUCgaC/INyfBAOXOHOoyUNyUNSuCQFJkhhbnGnZpk0UtAtEi4TOkGXJ4j5ltlQ4bDI5Ht1SEeC7qhZjsgnED/yWJeyyzKXThrK7qYPh4fgKfVIrAZceNYzbX/nGiKcADDcrgDSnnXSXnRZfkB2mStY2WTLOs9okALS+Svxw8mA+2FDDpppW/HGsBfGsA+aP/vMtdcbrB5ZuALQ8/4eW51jPE+7Dii31gBYXkm8ShsHwtbfVtbEtXNivPM8TldI2pisxfYx2TTILkUZTRfKqJh+vf1XJWVGV0Ym6nqKoFiGRKFjcXOgvK80Rt013OJDrZvRVdOGZ4XZYBEN0kUz9eyUCtQUCQX9BWCoEAhNSVGrZ7p8n8jpZKtREJMpEZbdFMjE1tPktrjDnTinD64pdJ9BFyRHD8jjjkEFGjQx9DilJErkep1EDQyffJCocNskolvfI/zYY2yUpslqvx3jo2W2G5nvxuux4w5moorM6QecT2RUmUaHzbWVjzLZocdLqC1oEgy5odAuF/j5RvQ2zu5Q5HiN6VTmeqDhhbCEA/++rSl5etdMSh2K2BiiqGiO0QgmK9O2JE8vSEwgh0fPoRSgzo8SfTY7/Ny0JS4VAIOgnCFEhGLD0pruHHMeXukvHW1yeZMvr7PBkpdUfYmut5rv/4ymDuerY4XHPlUjUqCZLBVgtE9r7SFE+h03m7EMHIQFvrdnNlvB19RVXf1AxCt0dNTIPgDFFmktVQThblV4F3Ew8lyNdDOxqaKeqKXYyXbGrOWZbvNSt5nMHQ2o49avRIub6+jnqWv0sW7eHDdXadXzBiMtT9HfG3P2mds2X/oRxhZwdtlC89lUl5zzycSTlbFhDBBWFW1/5hn9/stV6Hwlm+U0mK0hPhkFEixzB3uEPKka2smxPlKViLzLCCQQCQV9AiArBgKU351B7E1OR7HiHTcLjshnWhzWVTQCUZqclvE6i9Lj65FSf95hjKCQwxIt2XS0u47Cw29GmcEA2aOKkKVxF2C5LXDR1CHNnjODUCSUAFIaDvKuaYkVFvN+BPrndUN0St9+7m33URcV1xE8pa30fCEVSvurNzdfXX2+r09yjttRo/5stFapxnMr63c2G9cUfUgyrQ7rbzkVHljNn+jAAtta20R4IWYK0V29v4N2Kau56vSJu8HY05vvrye/tgVw3oy/S5o8kE8hOc1qEhIRZYAhRIRAI+h9CVAgGLPtqYbY7osLcNXuU24SERHZYAOgByKXZaQknKokyWelByrr7he5WBdrE2JytSQ/y1gPYG8LB3Iqq/eiiIjPNgddp58SDigw3qMJMTVR8uqmWjzZEAq1DisIzn27l2c+2sa2ulfXVzbzxTaXhAqTX4RhuqrGhoweFf7mjgV0N7QksHtZtgZBiTJwj4iJ2Qp/uihQONFtgIPKd2dnQztbaNtZVadYMPR2vTYI0hw1ZljlhbKExqWxoC1i+b22mgG+z2EqUjSmUgvDoDlZRIVTF3qIHadtliYw0a/anvaldIxAIBH0BISoEgl7APDfsjvuTefVajgrUBoxgbdAmsTkeR8LaBY5E7k/h//XumUVFhttuESn28HX1IOH6cOCwqmqr/7rrT2ZYjJjvuSjs/rRudwtPfLyFdVWaBeLbyiaWrtvDu2uruXtxBX98cx0vfrGTpeuqeW9dNZv2aNaQ4iw34wdlkut18oPDIm5Fz362jb8t3cDtr34b11IRPUfWRIXVDSnaAqCqKjaTCGto91tEhX68XuBMpyUsqtLdDiQkZAmcdpsR41Lf5rcIRT1tL1itPoncn1Kd/Hc1Rey+Ksb38MMPM2zYMNxuN5MnT+aDDz5I2PbSSy9FkqSYn4MPPtjSbsGCBYwZM4a0tDTKysqYN28eHR09lxmrO+jfA5ddxmWXsckSQ/O9lOd5cDsiYlWICoFA0B/plqjoygABsGzZMiZPnozb7Wb48OE8+uijlv0vvfQSU6ZMITs7G6/Xy6RJk/jXv/5lafOb3/wmZpApLi7uTvcFgl5HSSAKUiXR/E63WpgrZg/KcSMhJRQvkiTFLahmzv4EVqGS4bL6g+tWBz34VK+noMcuGJaKcMYb86Sp1FTvAmB7veZWtNOURcocr7y1ppVgSGVtlebale1x8KcfTOSln07jlPElRiD0u2urjWPi1aGInnz7Q0oktiH8f/TnrKjWyXVDW8Dq/mT013pgS1hk6FYOWZJw2CSjEKFmqYgcU9MSERWbayJuXolqZSRzf2psC7CzoZ2dDe0sXVtNdRw3s0SYL5eglMde8/zzz3PDDTdwyy23sGrVKqZPn84pp5zCtm3b4ra///77qaysNH62b99Obm4uP/rRj4w2Tz/9NL/+9a+54447qKioYOHChTz//PPcdNNNvXMTKaKnB3Y7bMbfzMjCdEYXZVjaCVEhEAj6I10WFV0dIDZv3sypp57K9OnTWbVqFTfffDPXX389L774otEmNzeXW265heXLl/PVV19x2WWXcdlll/HWW29ZznXwwQdbBpuvv/66q90XCPYJe+tKkiiI3G6TkSTINQVRHzdGm2Qnq9cXt0qzEVOhuz+ZREWa3XI+3dqRnRZJZ6v3U1WhuV13f7JjkyWLIBmca62LoReQ0zNXnTmplLMnlZr2axMzPZtUtseJx2knLZxFalJ5dsytNIb7o/e5sd3Ptc98wdtrqow2wZAaEzcQ/XtSVOsn39gesGRo0i0B0cfp7k+6ZUKSNKuSLjIa2gKW89a1RgLQN+9pM50/5tZirhf91fp8Sx0Vu5qo2KWJMHNa4HioanzrRG+5P917773Mnj2bOXPmMG7cOBYsWEBZWRmPPPJI3PZZWVkUFxcbPytWrKC+vp7LLrvMaLN8+XKOOuoofvKTnzB06FBmzpzJ+eefz4oVK3rlHlJF/7tIc8hJszp1x3opEAgEBzpdFhVdHSAeffRRysvLWbBgAePGjWPOnDlcfvnl/OUvfzHazJgxg7PPPptx48YxYsQIfvaznzFx4kQ+/PBDy7nsdrtlsCkoKOhq9wUCA331cGgcn/29ZW/nZ4kml6AJi2kjchlXksHcGSM4YpiWbSnZRCW+pUL7Xz8qOy3i/uSyyXHdn3ThYcRUKLqlQnd/0iwV5niMIbkehpk+Yz0LlG6pGJydxmkTS/nJ98qBSPpUfYKWnebAaZeNfpbFKfCnCxG9z/9bt4ft9e38Z8UOY7IcCCkWURAIhlBUlaAScYsKKarldxdSVHwBq6uTvt1Mi6mKst4Ph03G69LdxfyW89aa3J+217fRGg7wTeT+lCj1bVdpbA/w3nd72B4ORjffRm+ICr/fz8qVK5k5c6Zl+8yZM/n4449TOsfChQs58cQTGTJkiLHt6KOPZuXKlXz22WcAbNq0icWLF3Paaaf1XOe7yPrdzYagS4uT2tmMwy5EhUAg6H90SVR0Z4BYvnx5TPuTTz6ZFStWEAgEYtqrqsq7777LunXrOOaYYyz71q9fT2lpKcOGDeO8885j06ZNSfvr8/loamqy/AgEOuV5Ho4elW/UX+hJejPo1SFLlOd6+flJY4xsTJDcpUJ3xTCjN9e1g9sZ8fkOqWrcwG9dVLQHFTrCqVZDikpj2P0pI80RY6nI9Tp56afTuPH7YwAtAFtRVSrD4kKvj1GQoVlfdEtGY9iVJNvjwGWXDfWT7rJbMlMB1Lb5LJ+B2R1qc22rkaFKn0RXNnYw8bdv88RHW/jdaxXc9so3BBUlPPFPXEvCEB/Rlgo9piI8mbTJmvuTXqOjsT1gCQ7Xq5bnpztRVHhv7R7js4xGVa0F8vbmm7W5ppVQSDUCzM0nSyZku0tNTQ2hUIiioiLL9qKiIqqqqhIcFaGyspI33niDOXPmWLafd955/O53v+Poo4/G4XAwYsQIjjvuOH79618nPFdvjwdba9uM9MNeR3JRMSzfi8dl65Vnj0AgEOwvuiQqujNAVFVVxW0fDAapqakxtjU2NpKeno7T6eS0007jb3/7GyeddJKx/4gjjuCpp57irbfe4vHHH6eqqopp06ZRW1ubsL/33HMPWVlZxk9ZWVlXblcwADAHT/YkvZlIJ5F4SBa74XZE/tTz0p2kOW2MH5wFmFJdRq3QR2uKofle8tJd2gQfrfie3tZwf3LbLVW2QXNfyvE6ObQsG4DaFj+VjR0EFRWHTaIgnHJWr4Bd2+xDVSNCJTvNGbZUmF2qrIX66sKWCr1NnanK+OptDdy/dD3n/v0To0DdF9vqaQ8ofLyxlp0N7exu9lHfqk38LfN6NX5K2eiA6JawpSbdrU0m3Xab5v4Ufq8HZquofLm9QavaLsHlR2tpZ9+uqMIf1CwmqqpS1dhBR9hCksxlK1EMRiLMQfuKou6zOhXRrkCqqqZU9O3JJ58kOzubs846y7L9vffe4+677+bhhx/miy++4KWXXuK1117jd7/7XcJz9fZ4kOa04Qso4dfJh1aX3ca0Efm9YiUVCASC/UXy5ZQEdHWAiNc+entGRgarV6+mpaWFd999l/nz5zN8+HBmzJgBwCmnnGK0nTBhAlOnTmXEiBH885//ZP78+XGve9NNN1n2NTU1CWEh6PPY41gdoBP3J1tEPJVkpVFsCp7WD1NUlZJMF5VNPiaVZcdYKkYWpjOyMJ1sj4PdTT4qGzr4z+c7OGxIdoz7U4fJZUh3vcrzOnHbZTqCCqu31QNQmpVmPAfywuKiPahQ1dRhBA5nptlx2mXL5H5wtodvdkZWmhd9tIVtde385Ajt77vGVIH60821hsj4emcDh5bl4A/ELss3tvvDk/rItlDUe/1l0DSZ393cwdrwyr9uqfC6bCgqeJza+50N7azYUs8LK3bw9prd4TZ2pgzJJcfjoL4twMY9LeSmO9nV2EHFriZkGY4fWxS3sJ9OsIvmBZfJDa7FH+x196f8/HxsNlvMolN1dXXMYlM0qqqyaNEiLrroIpxOa2HG2267jYsuusiwYEyYMIHW1lauvPJKbrnlFuQ4AUa9PR7Y5cj3PjeqkKRAIBAMBLokKrozQBQXF8dtb7fbycvLM7bJsszIkSMBmDRpEhUVFdxzzz2GqIjG6/UyYcIE1q9fn7C/LpcLl8uVcL9A0BdxxQu6phP3J9Mx0c107aCq8OtTx7FpTyvjSzNj2ukMyfWwu8nHOxW7Wbe7ha92RgKD9exQ5Xke6lr9llgKWZYoynSzta6NT7fUATA4Jw2bTSIUUnHaZDLT7DS1B7nt/74FtNS2DpuM02YNfB1RELvC+07FbiYOzuSgkixqmiOiwmy10C0M5pgGncb2AIpiDZKPntBHUtJq/7/xTSUvr9ppTM4z3LqosNMRCBki441vqvhwQw3NHZHiaCMK0lFUlREF6azYWs/m2lYmD8kxrBq6Xoie7KtdsFRo1ojId8PcvKk9QJrJUtcbosLpdDJ58mSWLFnC2WefbWxfsmQJZ555ZtJjly1bxoYNG5g9e3bMvra2thjhYLPZwlXT499Hb48HigodYeEbXZ1eIBAIBgJdcn8yDxBmlixZwrRp0+IeM3Xq1Jj2b7/9NlOmTMHhcMQ9BrSB0+fzJdzv8/moMkLj4gAAOHRJREFUqKigpKSkC3cgEPQNbGE3lXhCYWRhOmOKM8hNt05cklXpNQuRaDcps1uR12lnwqAsJElKeL7B4UDpdbutFa9tMuR4HUiSZrE4ZnQBZabMT5IkUZqtuS3tatDiKYble3GZLC9pUe5owaCCIywozL2ZVJ7NT48dzukTrX//1U0+OoIhmsOB09Erxnrwt54S96CSDAaF+9TYESQUFb8QO6HX/g8oKnVtfl78Yqdlop4eDsx2O2yW7E+AISgOLs3k5IOLOHVCCS0dQYbmaZ/RlprWuLEsMX0wvdaLHybik021fLihxhBBQVP7pvYoS0UvpZSdP38+//jHP1i0aBEVFRXMmzePbdu2cfXVVwOaBeHiiy+OOW7hwoUcccQRjB8/PmbfrFmzeOSRR3juuefYvHkzS5Ys4bbbbuOMM87AZusdl8bOUFEN9ydvJ4HaAoFA0B/p8pNv/vz5XHTRRUyZMoWpU6fy2GOPxQwQO3fu5KmnngLg6quv5sEHH2T+/PlcccUVLF++nIULF/Lss88a57znnnuYMmUKI0aMwO/3s3jxYp566ilLRqlf/OIXzJo1i/Lycqqrq7nrrrtoamrikksu2dvPQCA44DisPIcN1c2MLMyI2ed22CjL9RjxATqpWiqi3aQSaZFEMRrjB2Xx0qqdxvsxRekMzfcyflAWHoedRL2QJYmxJRks3xSJgxqW78VhlyFciXj8oCx2N1XjsEkEQirjB2dRmu02jjf6jMTkIbnY5AbLNbbVtzGqRfvMPE4b4wdl8v53kdgtPU2tHgMxqTyHNl+QnavbaWwLaMHa5jiDqIm2vhIeDClGxe/CDCenjC/lyx0NjCnKMD5Pp13GEzW5POewQZw4tsjy+xhWoAXrVlQ2sXp7g1GHQyfGWqKkZqkIhBSjcnerP0iG22ERKC2+IPkZVtGlKGq36qok49xzz6W2tpY777yTyspKxo8fz+LFi41sTpWVlTEpyRsbG3nxxRe5//77457z1ltvRZIkbr31Vnbu3ElBQQGzZs3i7rvv7tG+dwkVI3mB1ylEhUAgGHh0+cnX1QFi2LBhLF68mHnz5vHQQw9RWlrKAw88wA9+8AOjTWtrK3PnzmXHjh2kpaUxduxY/v3vf3PuuecabXbs2MH5559PTU0NBQUFHHnkkXzyySeWNIMCQX8hK83B5CG5SdtEr2gnmwvGrVMRJt5hyWpenD6xhDtfW2O8nztjZEorsxIwriTTsm1QTpolM9UPJw/mqBF5lGan0dwR5KhReZRmeRJ2NMdjnRTvqm9nT9j1KT/dyYj8dIuo0IWYbskozXIb1gfD/SkqzkCSItYcFS2eQlW1gHPtOi5+NGUw00flA5Gq5w6bbMlSNSg7jVPHx1pWh4QtFe0Bhb++/R2KonJwqRZEHwwpscLG9DpZTIU5A5Zu0QhFCZLY4n8qckJZ2H3mzp3L3Llz4+578sknY7ZlZWXR1tYW2ziM3W7njjvu4I477uipLu41ihopfleYIdxuBQLBwKNbyyldHSCOPfZYvvjii4Tnu+uuu7jrrruSXvO5557rUh8Fgv6O2TJhk6WkyRIcpol7MGp12yZLeF12Wn0Rf/9krlSFmdYK2dGCIlE/ZEkiN0oE2GXZ0jeHLFOeq8VLnDKhxIhJ0M4be85cr9WFcmdDu1HnIj/dxfBCa+xFY5SlIjvNYQS+N7UHLJN0f0jh9v/7huIsNzecMBrQJt26C1Gt6TpyuGq5P6hQmBmZUOabJpe6m5PDLlOS5cbjtLG2shm33cbIQi8bqlsBWLW9wRAVgZAat0CfTjJLhTmwXU91ak6Fq6qxoiKkqt0bFASoqIbQ1NMkCwQCwUCiy8XvBAJB5+irz3qBvd7AnAQqFZeV4iw3HpctJs5AkiSOHJ5ridFIJioAHrtoMgCzJqYe06Sf8qAS7TOZFE4xm8iKEu2mFa9POV4nl0wdwgljC5ElbbX/qx0NAJRmp1Gc6eaU8cW4w9eIFhVZHqdRH6OxPWBZvd9Y3UJNi59vdjYZYkNVMepW6AX38tJdyDJMHpLD8AIvIwsitQf0dLmAkXHLLkuMLspgcI6HKUNzOHxoLreeehCFYVek7/QaEmjXiq6JYcn+lCSmwiwqOsK+/mbXKZXYyu29mQq5vxNSVCMBgB6nIxAIBAMJsSglEPQCo8KTxjRn7wWNmifZqbjBjx+UlXCfJEkWN6TORMXMg4v52/mTcNlj78+b4J71U151zAje+24Px44uAKy1E8xEu2DFa2WTJX575ng+21zHmsomKhs7qKjUJuXDC7xISNx/3qH8Z8V2bn3lGxrbA6iqalTAzvE4yApXEq9r87O9ro0Mt2b9MM+vmzqC5HqcYUuFNkGvbY24WcmSZu0ZXmAtZpZtsswMyfPG3Ie+/9gxBdxy6kH87PnV7GrsoNUXxOuya1XAwzN9WSbGPSva6mTGXLTPZypUqBNTk4PerVXR32n1BQ0hVypEhUAgGIAIUSEQ9BK9KSjA6v7kSFC7oit0VaTkel20+yP1KA4qzcQXVGLco3R0tyivy85pEyIWjkQB5qkElOt9tskwqjDdqNINMDQ8iZelSBaolo4gzb6gIRiyPU7yw9aE5o4gFy78jN+fPZ7CDLfFHayhLUCux4mqRuITDEuF15nw8/I4bVx4ZDk1zX7GhS008dzDJEkiI81OUYaL3c0+Nte2Mr40i0BIMeI5bLKMoigpuT8pimp1fwpbKsxWD0UlJv1qV4vpCSJUh2N58ryOXiuqKRCYCYVCBAKBzhsKBJ3gcDh6JHOeEBUCQR/FPP8zp27tLnaTxSCVasfRLfLSnXEtFzqJJt4Om8y0kXm0+IJ8tb3R1L5z9yd9myxJTB6Sw/vrtaDsHI+DzLDFQZYkcjxOJDTrg57ONs0h47LL5EWl5v14Yy1nTRpkWDMAGtv8gDcsKhRCimJUFM9LdyX8vNIcNmaMtmZzSphtS5IYmu/VREVNWFQEVfTnvF2WCGC1JiQK1A6pqiXd7O6mDl5ZtZPaVh/luR4kpLgxFUJTdB89nkJYKQS9jaqqVFVV0dDQsL+7IuhHZGdnU1xcnNL4nwghKgSCPorZzag0K751oCt01VIR/eDpzGUq0X67LOFx2rGb/J0kKbU4Eb2JLEmMKY7Er5jdgmRZwmHTLAFN7UF2NbQDkO62I0lSjJWnLuwXbxYVeipaFS1wur49gKJqFpLsNEfCe4tnrUp0VzZZYnBOGp9uhqqwxcUfUnBJsrFf60OERDEVoShLxSPvbWBVWLBdcEQ5x40pRFVjYygSFY4TdI5exb0kS4gKQe+iC4rCwkI8Hs9eTQIFAlVVaWtro7q6GmCv6r8JUSEQ9FEKMlyMK80kz+vskUHFHpVNqjOiL9mZqEi0V8++ZL5kvFPFdX+SI0UC7bLM1OG5LN9Ux5mHlIb3Y+zPdDtoag+yo15LVZrhchj3+aPJg3lh5Q4Attdp+1tM1a/1AG9V1SbsdeFV6dzwZ5/o44pX/TzR78oWtqiYr+cPKuxqaKOisolpI/PCfbC6Pz37+TbW7Gri6mNHMCg7jYZ2Pyfdt4wjhuXy/YNLaAsELVXPt9S0wpiIQDIj3J+6h6qaMj8JS4WgFwmFQoagyMvL29/dEfQT0tK051Z1dTWFhYXddoUSokIg6KNIktSjExizkEhFpHSlTkayc+qB2t2xdOh91vddNHUo00cVMKpQC5jWvYNsskRJVho76tsNF6l0l92I2/j5zDFMGZrLr178il2N7QQVxWqpCLs66cHNuuVCD7RO1HdJkphYlsWG6hbafKFw2wT3J0tkeRyW67X4gly86HMAhuVPwu2wR2V/Uni3QltduuPVb5l34ig2VLewva6d7XU7scsyH26owRSzbcSCKApESwihKbqHqkYC90U6WUFvosdQeDx77/IqEJjRv1OBQKDbokKklBUIBIBVVKTm/mR93ZkQSXROQxiYRU0cu4Z5iyRpKVqHhGNJ9HM4bTKHlGXHdc06PSr97eDcNOMeZFmreeG2y4QU2NPss4iK+jbd/UkTFrolISsct5Hs1gsz3EbQeLK2NlkiOywqGsPXe6eiythfUdWi9cEUYN0QVVX9vnfW8966Pcb75z7fzo56zd1rVLhmhz75hdhsT8L9qXuoaMH80DOuiAJBZwiXJ0FP0xPfKSEqBAIBYHV/6sxqoLUxv07FXSqBpSJe+e647k/WbFfjB2WRF87cZA6LiHcVu02z6sw8qAiAyUOyOX1CqSFk7LKMhGSsMu9saLe4P+kTxmBIZc2uJsOSkOnWjL1ds7IkLg6oV+BuDyr4giFe+7LS2L+5psV4rc/99WxXaXaZw4fmAJFq4dF8P1zNu67VT0hRqGpqp6qx3di/ra6Va5/9gorKpqT3IohFUVUjW1hOVB0YgUAgGCgI9yeBQABYLQWpiATz5DiV5sncfhKfOf626OuZBYckSdhsEiFTELPu5vSjKYM5flyhFoeCZGzX/x+UncbGPa3sauiwZn8KV9v+/eIKVm9vMLZnpUUyTCXDrJuSWSrcDptRmXtzTRu7TClyN1S3sKW2lRe/2IHLLjN9dAFVTdr+kuw0ThlfzOdb6mPO+9+rp1Ld3IFDlrHJEFJgR0M7d71WgQo8euFh2GWZO1+rAODXL37FS3OPSno/AiuqGgnsjy4uKRAIeo8ZM2YwadIkFixYsL+7IkCICoFAEMYud18k2ONZG6KI59LUFaQklhFzTQtZ0qwfoVCkhobuHiUhke91WdoC2MJxHXrNivo2v0VUtPiCvLJql0VQAGQYoiJ53839S5j9SZKQkMhJc7C72cd3u5uMcysqfLe7hbte1yb+973zHYeUZ7M7bGnIT3fFpBW+eOoQCjJcTBmay66GdtbsaiLP66S62c83OxqNeIr1u1sYbIoD2NXYzpc7GhiZLWotpEowpNAartmSa/p+CQQCjc5cay655BKefPLJLp/3pZdewuFwdLNXgp5GiAqBQAB03VJhHiRSyhbVBWfLRAXiJElbFY7ea4kHkSUcQEcg/n4zEfcn7X/d8lDd5MMXtNaA+HhTTczxevvO40k6/2z1PmZ6NFGxLhxDcWh5Nl/vaLJUyP5mZyOfbaplTzjoOj9Ds7wUZmiiAeCYUQVGDQ49bW5euovqZj/rqyOuVCu31VHVFBEkdlnWsluJLEYp0eoLGimAASMuRiAQRKisjLhyPv/889x+++2sW7fO2KZnH9IJBAIpiYXc3Nye6+QBRKr3f6AhYioEAgEQFVORwpPBYqmwpSAqutCXRG31+XhMIHZUkLkjKpVrool8pCK39r8+IdTTzkpAUYa28tzUHhurECmwl6DDcfqXCGe4z9lpmhD4bnczoNU9GD8oE4ARBVqwtT+kUt3sM2oj6BaWnx47kqIMF1cdM1zrX1j0OMOiQm/33e6IqPhiWwNf74iknK1t8aOiiqDtFFAUlc+31LE8LDjT7HKPVLcXCPobxcXFxk9WVhaSJBnvOzo6yM7O5j//+Q8zZszA7Xbz73//m9raWs4//3wGDx6Mx+NhwoQJPPvss5bzzpgxgxtuuMF4P3ToUH7/+99z+eWXk5GRQXl5OY899ljSvr355pscffTRZGdnk5eXx+mnn87GjRstbXbs2MF5551Hbm4uXq+XKVOm8Omnnxr7X331VaZMmYLb7SY/P59zzjnH2CdJEq+88orlfNnZ2YZlZsuWLUiS1K37VxSFP/7xj4wcORKXy0V5eTl33303AMcffzzXXnutpX1tbS0ul4ulS5cm/Uy6i3j6CQQCILXVdDNmd6ZULBWpxWmEz52gqX7N2BoZ5mMl8qL82hOJnuiYCr1OhO7Kku6yU5iZ2J0l5ZiKFILa9Ym/HqytT+mLMl1cfvQwnrzscG45dRzD8jVhsbmmlfpWzRyji4WyXA93nz2Bw4fmWvrnsIfdu8Kfi9nq0dQeZG1Vs/HeH1Jo8YVoD0TcxwTxCSoqwZBquMp53cL4L9j3qKpKmz+4X356cvHhV7/6Fddffz0VFRWcfPLJdHR0MHnyZF577TW++eYbrrzySi666CLLZD4ef/3rX5kyZQqrVq1i7ty5/PSnP2Xt2rUJ27e2tjJ//nw+//xz3n33XWRZ5uyzz0YJ5yRvaWnh2GOPZdeuXbz66qt8+eWX3Hjjjcb+119/nXPOOYfTTjuNVatW8e677zJlypR9cv833XQTf/zjH7nttttYs2YNzzzzDEVFWkKSOXPm8Mwzz+DzRTL+Pf3005SWlnLcccd1uX+pIJ6AAoEAsFoqUsE8N07lWPNk2maT8DhsDDGlWu3KNZPHVEiU53qQkMgNu//YEkzkdZcsWZaQZcjxWs3NXpeNgvSIqHA7ZHwBxZjwZ6Sllv3JlkK8imGpiHKfKchw4bbbOKgkkzWVTQzP97K5ppX11c3sadbcbvLT4wcH65YUffW8JDt+ulOzyACobfHR5hMBx52hp+Rt6dAEWIYQFYL9QHsgxEG3v7Vfrr3mzpPxOHvme3/DDTdYVvgBfvGLXxivr7vuOt58801eeOEFjjjiiITnOfXUU5k7dy6gTdTvu+8+3nvvPcaOHRu3/Q9+8APL+4ULF1JYWMiaNWsYP348zzzzDHv27OHzzz833K1GjhxptL/77rs577zz+O1vf2tsO+SQQ1K86whdvf/m5mbuv/9+HnzwQS655BIARowYwdFHH23c13XXXcf//d//8eMf/xiAJ554gksvvbTXUhILS4VAIACsE99UFp/Mz6SuVuD2OGwcMTyP4gQ5/RMFdesPwhhLRZT7kyRJlOd5SHfZY/abiRYjaQ47bpPrVGGGm/yMiKgoSHfhckT26+lwO4sXScVKoxcBjLay6FYIWZaQJInh4cJ+H6yvwR9SyfU44mYcSnPaDKGii4oheenWNo5IMLYswdA8LbaipsVHqz9+alpBBP3vRE8nm+7qez7QAsGBQvTqfigU4u6772bixInk5eWRnp7O22+/zbZt25KeZ+LEicZr3c2quro6YfuNGzfyk5/8hOHDh5OZmcmwYcMAjOusXr2aQw89NGH8xurVqznhhBNSusdkdPX+Kyoq8Pl8Ca/tcrm48MILWbRokdHPL7/8kksvvXSv+5oIsawiEAgAa5yCGlNrORbzRDml7E9mF6BOREhi9yf92tbt0eIgmkSWlOh7CIZC5HidRv2H4QVeY1IP2gS/uSNIR0BJeJ7OrpMIXQBMGJyFxyHTFr5GljviYiVLcGhZNqXZbnY1aH286pgRcT//0jiB1rleB5luO03hGhzjSzP5fKuWhjYv3UlhhpsttW3UtvhpE6KiUwxLhS4qhKVCsB9Ic9hYc+fJ++3aPYXXa7Vc//Wvf+W+++5jwYIFTJgwAa/Xyw033IDf7096nugAZ0mSDFeleMyaNYuysjIef/xxSktLURSF8ePHG9eJDiKPprP9kiTFuInpldHNdPX+O7suaC5QkyZNYseOHSxatIgTTjiBIUOGdHpcdxGWCoFAEEOS569BVy0V5ol1Z607C9SObtF5nYhElg9zG+1/s/vR8IJ0CsyWigyXMfm3Xj/p5VP6fPTzuuw27jhjPNkeB98bmmuIPT3lrNMmc81xI5HQ3J4umhoZIPTrZHkchtXBjIRksQ4dVJoZubd0l5EtqrbFR5tfxFR0hi4qdEtFphAVgv2AJEl4nPb98tOblb0/+OADzjzzTC688EIOOeQQhg8fzvr163v0GrW1tVRUVHDrrbdywgknMG7cOOrrrfV+Jk6cyOrVq6mrq4t7jokTJ/Luu+8mvEZBQYEl+9X69etpa2vrtG+d3f+oUaNIS0tLeu0JEyYwZcoUHn/8cZ555hkuv/zyTq+7N4gnoEAgiKHrloquZX/q7kCkXzP6cuaF+q5YKsyTfW21P2TZNjTfYzlfXrqLcw4bxKPLNnHahOKk17T2O/JaSeBb5jRlDSrJcvPQTw6j3TSxl+XIecYWZ3L7rIMYVZSOx2U3Uu0WZbopy00j3ZV4sNctHwDlJuFRmOmiKEMTHKu3N3D2hP6ZqrEnUcK/SsNS4RJDqkDQU4wcOZIXX3yRjz/+mJycHO69916qqqoYN25cj10jJyeHvLw8HnvsMUpKSti2bRu//vWvLW3OP/98fv/733PWWWdxzz33UFJSwqpVqygtLWXq1KnccccdnHDCCYwYMYLzzjuPYDDIG2+8wY033ghoWZgefPBBjjzySBRF4Ve/+lVK6WI7u3+3282vfvUrbrzxRpxOJ0cddRR79uzh22+/Zfbs2cZ55syZw7XXXovH4+Hss8/usc8uHsJSIRAIYkgppsL0usuWigTNh+Zrk9zRxRlJrxkTqN1JOtzOUsqaz9HYFjFLexx23HYbWeEV6Dyvk8lDcvjdWQdz86mRga0zjWRxLUvw2ZrbhBQ1JjWpXZaNDyAQUijL8VCc6bbch02WyHA7koq2iWVZxuvC9IjVoiDdxZRhORRkOKlrC/Dal5XxDheYUKPcn/QUvgKBYO+57bbbOOywwzj55JOZMWMGxcXFnHXWWT16DVmWee6551i5ciXjx49n3rx5/PnPf7a0cTqdvP322xQWFnLqqacyYcIE/vCHP2Czaa5fM2bM4IUXXuDVV19l0qRJHH/88ZYMTX/9618pKyvjmGOO4Sc/+Qm/+MUv8HhiLcnduf/bbruNn//859x+++2MGzeOc889NyZ+5Pzzz8dut/OTn/wEtzt+HGNPIZZVBAJBDKkFanctpWyyitg6IwszGJLnTZzrX4o9V/T54p07YfG7OC5cZxxSykPvbeTkg4qMfceNLeTDDTWMLExHQqIkM400U8aTrqTLTWSpMBNSVKI/ArtNMq4TDKnhPmuNZFmKe4yZbI+DhrYAZxxSiizJlGa7SXPayHDZafYFKchw47bbuGL6cP7wxlqjToYgMUpUoHaGW4gKgaAzLr30Ukuw8NChQ+Omps3NzY2p7xDNe++9Z3m/ZcuWmDarV69Oeo4TTzyRNWvWWLZF92fIkCH897//TXiOc845JyZzk05paSlvvWXNztXQ0GC83pv7l2WZW265hVtuuSVhm/r6ejo6OizWi95CiAqBQGCQl+6ktsXP4JzOA8C6mlI2WUVsM8mKh0XcnxLHVMQ7d8KUshYxov1/aHkOvz97PCML042Cd1ceM4LTJ5ZajjXXvuiKqEg1q3v0Oe1yRFQEwkEv+ucuG2IriYVicDa7mzooznLjD6mGRWbayDy+2dnE6CLNOjQ8P50Hf3IoU0rTKEo8TgkwB2prbmoipkIgEBwoBAIBKisr+fWvf82RRx7JYYcd1uvXFE9AgUBgMKksG19QwZ1CRo94rkOdoYuKrkzCLccn2G5JhxtnfyoVrc19KsxwW8RNdPE8myxFZZzq9PSR/qWoKsz3ZAunk9UvGTIsFZEg7uh7iMZplynL1Uzu5lY/mlzGzaems95UZfvY0YWEfJ0HEg50IoHamkDLFJYKgUBwgPDRRx9x3HHHMXr06KRWlp5EiAqBQGAgSVJKggKsE+lUUsrq5we10xiE5MfHs1REXne3wGvSOI3o65msBpLUtcDzVCvQWgLhbfED1HVLhTlDVCpE99ceZR2yyRIi91PnqKqW1EC3VIjidwKB4EBhxowZPVrxPBXEE1AgEHQLc4E6my3Fyaz+fzdFRcTNJ+q8XayxEffcUboomSXGJkm4HTL5Ga4u52lP1jvd/czjssVYKiBWDOgWmMj+1PoQ3c4RfX9drK4+UFFUFX9QIRgOrsgSgdoCgWAAI0SFQCDoFl2NqQB9oq4mrJid6jW76z6VjOhVfsukPo5lRJIkJpVld/k6ipJYVhxcmsWO+jZKs9PYVhdxP9JdsaLvOjqmIlUxkMwqA6lbPAY6igrN4SBtu5y6lU8gEAj6I0JUCASCvaark9kUvaXi0PmKfHetvTFWAHPMRLQVYy9W8pN1z2mXGV6QHnN9myEe4osBbTIbwBWnMF88onsvS5Lm8qSoWj0MYalICUVRjcxP3ijrkkAgEAw0hKgQCATdwjx5T9VSoc+Ju2upkCXr/531qytE17owz9/Nmasg9fuNRyopZaP74wirmmgxpceyjCnOoDQ7jRxPau430eJEkiJpaXvDCtRfUVVz4TtHt936BAKBoD8git8JBIJuETJNjlMNVJZSEAXJjzeiMhK2SRRTMX5QFjleZ0o1KyQky3vN3SleP7qO15naWo6tC5YKh00m1+vs8u8h8j6SzSrVoHuBJhD1IG2vy9ZtsSwQCAT9ATF6CASCbpHqirsZc8ak7qAf1h1LRXGWm8lDchLWwbBM2CXz1TSR0dVif9EcPjSXkmw3YxJUC4/GHPzuSBAI3113m9jigab0tMKFJ2UUNeL+lO6yC0uFQCAY0AhRIRAIukWygONERLI/7d1kON7xRZluAMrzPEnPkciSYY4jkCDWUmFq251A5iyPg4NLs1IO5u3MUiFJ3RcAse5PkhAV3UBRoaVDiAqBYF8xY8YMbrjhhv3dDUECuiUqHn74YYYNG4bb7Wby5Ml88MEHSdsvW7aMyZMn43a7GT58OI8++qhl/0svvcSUKVPIzs7G6/UyadIk/vWvf+31dQUCQe/RHWGgT9y7O/eKVNSO3TdhcBbHjinotABZIkuGzTJhl6Im8FLSwO3ewHwNI/uT6b73ZvIfIyroXUtFV57dl156aTiGxfpz8MEHW9o1NDRwzTXXUFJSgtvtZty4cSxevLjH+54MNdpSIdyfBIK4zJo1ixNPPDHuvuXLlyNJEl988cU+7pWgp+ny0Pj8889zww03cMstt7Bq1SqmT5/OKaecwrZt2+K237x5M6eeeirTp09n1apV3HzzzVx//fW8+OKLRpvc3FxuueUWli9fzldffcVll13GZZddxltvvdXt6woEgt6lPNdDhtvO6KLU3HnA7L7UvclXVpoDWYaMBMIhkWuTGb2/0RaNZDEU0e/3xWp+Z5aKvelDrPtT71kquvrsvv/++6msrDR+tm/fTm5uLj/60Y+MNn6/n5NOOoktW7bw3//+l3Xr1vH4448zaNCgHu17ZyimQG2vy75PxKZA0BeZPXs2S5cuZevWrTH7Fi1axKRJkzjssMP2Q896H7/fv7+7sM/o8iPw3nvvZfbs2cyZM4dx48axYMECysrKeOSRR+K2f/TRRykvL2fBggWMGzeOOXPmcPnll/OXv/zFaDNjxgzOPvtsxo0bx4gRI/jZz37GxIkT+fDDD7t9XYFA0Ls47TJHDM/r1N3IjG7d6K6bSFmuhxmjC8n1Ort3ArTYiumj82PEkNnyogVqJ7ZU7Is6DubJfbyK2ntnqbC+lyRMgdo9e29dfXZnZWVRXFxs/KxYsYL6+nouu+wyo82iRYuoq6vjlVde4aijjmLIkCEcffTRHHLIIT3a987QArUjlgqROUsgiM/pp59OYWEhTz75pGV7W1sbzz//PLNnz6a2tpbzzz+fwYMH4/F4mDBhAs8++2yXrrNx40bOPPNMioqKSE9P5/DDD+edd96xtPH5fNx4442UlZXhcrkYNWoUCxcuNPZ/++23nHbaaWRmZpKRkcH06dPZuHEjEN/96qyzzuLSSy813g8dOpS77rqLSy+9lKysLK644goAfvWrXzF69Gg8Hg/Dhw/ntttuIxAIWM716quvMmXKFNxuN/n5+ZxzzjkA3HnnnUyYMCHmfidPnsztt9/epc+oN+mSqPD7/axcuZKZM2dats+cOZOPP/447jHLly+PaX/yySezYsWKmA8TNHPyu+++y7p16zjmmGO6fV2BQHDg0RPF63qihoLLHhvXYJ6kxwtkNm9KxSKyt5jv0x4npezeZWmKTSmbKMPU3tATz+6FCxdy4oknMmTIEGPbq6++ytSpU7nmmmsoKipi/Pjx/P73vycUCiU8j8/no6mpyfKzt5gDtb0uu3B+EuwfVBX8rfvnJ8WEHXa7nYsvvpgnn3wS1XTMCy+8gN/v54ILLqCjo4PJkyfz2muv8c0333DllVdy0UUX8emnn6b8UbS0tHDqqafyzjvvsGrVKk4++WRmzZplsYxefPHFPPfcczzwwANUVFTw6KOPkp6u1QfauXMnxxxzDG63m6VLl7Jy5Uouv/xygsFgyn0A+POf/8z48eNZuXIlt912GwAZGRk8+eSTrFmzhvvvv5/HH3+c++67zzjm9ddf55xzzuG0005j1apVvPvuu0yZMgWAyy+/nDVr1vD5558b7b/66itWrVplETT7my7VqaipqSEUClFUVGTZXlRURFVVVdxjqqqq4rYPBoPU1NRQUlICQGNjI4MGDcLn82Gz2Xj44Yc56aSTun1d0AYRn89nvO+JQUQgEHSfA3kl15pSNjpw25r9yePs/crJZmuIbqkw9yHD3f0yQ9G6zCHLhqiwJ8g01R26++zWqays5I033uCZZ56xbN+0aRNLly7lggsuYPHixaxfv55rrrmGYDCYcNXunnvu4be//W33byYOljoVbpsI1BbsHwJt8PvS/XPtm3eB05tS08svv5w///nPvPfeexx33HGAZnU855xzyMnJIScnh1/84hdG++uuu44333yTF154gSOOOCKlaxxyyCEWi+Vdd93Fyy+/zKuvvsq1117Ld999x3/+8x+WLFlixHgMHz7caP/QQw+RlZXFc889h8OhudmOHj06pWubOf744y33AnDrrbcar4cOHcrPf/5znn/+eW688UYA7r77bs477zzLc0q/l8GDB3PyySfzxBNPcPjhhwPwxBNPcOyxx1r6v7/p1lJXdICmqqpJgzbjtY/enpGRwerVq/n888+5++67mT9/Pu+9995eXfeee+4hKyvL+CkrK0t6XwKBoHfJT3ficshkp1ikbV9iERGSNeRWiiqG594XosJiqdBe+wKKsa0sN3W3s2jMz029gnZe+HeTtxeuZalcDzp/dus8+eSTZGdnc9ZZZ1m2K4pCYWEhjz32GJMnT+a8887jlltuSeoOe9NNN9HY2Gj8bN++vVv3YulHtKVCqAqBICFjx45l2rRpLFq0CNBclT744AMuv/xyAEKhEHfffTcTJ04kLy+P9PR03n777S7Fzra2tnLjjTdy0EEHkZ2dTXp6OmvXrjXOsXr1amw2G8cee2zc41evXs306dMNQdFddAuDmf/+978cffTRFBcXk56ezm233Wa5t9WrV3PCCSckPOcVV1zBs88+S0dHB4FAgKefftr47A4UurTUlZ+fj81mi1lhqq6ujlmJ0ikuLo7b3m63k5eXZ2yTZZmRI0cCMGnSJCoqKrjnnnuYMWNGt64L2iAyf/58431TU5MQFgLBfmRwjofBOd2fDPcm0elaLe+x1uVISzEt7N4QLzDcbCFJd/WMpUJ3o8pPdzF9VEG3zxmP7j67QRMeixYt4qKLLsLptAqdkpISHA4HNlvk8xg3bhxVVVX4/f6Y9gAulwuXy7UXdxOLP6jQHhZ66S6HcH8S7B8cHs1isL+u3QVmz57Ntddey0MPPcQTTzzBkCFDjIn0X//6V+677z4WLFjAhAkT8Hq93HDDDV0KdP7lL3/JW2+9xV/+8hdGjhxJWloaP/zhD41zpKWlJT2+s/2yLFvct4C4rvxer9V688knnxhWiJNPPtmwhvz1r39N+dqzZs3C5XLx8ssv43K58Pl8/OAHP0h6zL6mS5YKp9PJ5MmTWbJkiWX7kiVLmDZtWtxjpk6dGtP+7bffZsqUKUmVoKqqhutSd64L2iCSmZlp+REIBIJ4RLs/WbM/SfiCESvBvoipcJhiJvTXg3PSGF2UwfTR+Xt1bjmOa1Vv0N1nN2ipyDds2MDs2bNj9h111FFs2LABRYn8Tr777jtKSkriCoreoqkj4mftddqEpUKwf5AkzQVpf/x08Tv/4x//GJvNxjPPPMM///lPLrvsMuPv5oMPPuDMM8/kwgsv5JBDDmH48OGsX7++S+f/4IMPuPTSSzn77LOZMGECxcXFbNmyxdg/YcIEFEVh2bJlcY+fOHEiH3zwQVyhAFBQUEBlZaXxPhQK8c0333Tar48++oghQ4Zwyy23MGXKFEaNGhWTCWvixIm8++67Cc9ht9u55JJLeOKJJ3jiiSc477zz8HgOrEW6Lo+M8+fP5x//+AeLFi2ioqKCefPmsW3bNq6++mpAsw5cfPHFRvurr76arVu3Mn/+fCoqKli0aBELFy60+Jrdc889LFmyhE2bNrF27VruvfdennrqKS688MKUrysQCAR7g7UOhRQjKkKhrhf726v+yBLTR+czfXS+4Zplt8mU53niBpp3l70L+O6cro4ZOgsXLuSII45g/PjxMft++tOfUltby89+9jO+++47Xn/9dX7/+99zzTXX9Oq9RNPUrq1+ep02ZElKWuldIBBAeno65557LjfffDO7du2yBBmPHDmSJUuW8PHHH1NRUcFVV12VUuyVmZEjR/LSSy+xevVqvvzyS37yk59YFh+GDh3KJZdcwuWXX84rr7zC5s2bee+99/jPf/4DwLXXXktTUxPnnXceK1asYP369fzrX/9i3bp1gBYr8frrr/P666+zdu1a5s6dS0NDQ0r92rZtG8899xwbN27kgQce4OWXX7a0ueOOO3j22We54447qKio4Ouvv+ZPf/qTpc2cOXNYunQpb7zxxgHn+gRddH8COPfcc6mtreXOO++ksrKS8ePHs3jxYiMzR2VlpcVHbNiwYSxevJh58+bx0EMPUVpaygMPPGAx2bS2tjJ37lx27NhBWloaY8eO5d///jfnnntuytcVCASCvSG2IFzibFD7ip4UD2bkOOlqe4uujhmgJe548cUXuf/+++Oes6ysjLfffpt58+YxceJEBg0axM9+9jN+9atf9co9+IMKQUXB47QOmU3tkXSygCh+JxCkwOzZs1m4cCEzZ86kvLzc2H7bbbexefNmTj75ZDweD1deeSVnnXUWjY2NKZ/7vvvu4/LLL2fatGnk5+fzq1/9KiZJzyOPPMLNN9/M3Llzqa2tpby8nJtvvhmAvLw8li5dyi9/+UuOPfZYbDYbkyZN4qijjgK0YPMvv/ySiy++GLvdzrx584yg82SceeaZzJs3j2uvvRafz8dpp53Gbbfdxm9+8xujzYwZM3jhhRf43e9+xx/+8AcyMzONLKg6o0aNYtq0adTW1qYcvL4vkdRo57B+TFNTE1lZWTQ2NgpXKIFAYCGkqPxvbTUAmWkORhR4WbWtAYDpo/P54LsaQKvPcczono092NdUNrbz7U5toC3KdDNhcFZMm/7+vOzK/S1duxtFgaNG5pNmimv529L1/PXt7xie7+XmU8cxpjhjrwLoBYLO6OjoYPPmzUaFesHAQlVVxo4dy1VXXWWJGe4Jkn23Un1edj/STyAQCPoRZtcVRVWt7lCShCyDokBBRs8G++4PzCvq+6I6eF9H956obfUx2BkRDc0dmt+1YakQH6VAIOglqqur+de//sXOnTstxUAPJISoEAgEAqxpT1XVOkGUgMOH5rK7ycfQLlQQP1Ax6whHL7s/9SfMwfoQCdT2ujTrxYFch0UgEPRtioqKyM/P57HHHiMnJ2d/dycuQlQIBAJBFCrWOgqyJJHhdpDhPvDqa3QLc0rZfZDJqr9grhMC0NyuWyq074XQFAKBoLfoC9EKYjQRCASCaKItFf1ssmhJKSvcn1LGFwxZ3rd0aO/Tw5YKEagtEAgGMkJUCAQCQRQqkYm3JMVWhO7rmO+mt7M/9Sf8Ue5PzT7NUuENx1QIfSYQCAYyQlQIBAJBFIqqGhPvfqYngGhLhRgGUiU6pqKlw5pSVhgqBPsKc+0FgaAn6InvlIipEAgEgihUVUsdK8vgdvROrYj9iVlUiEDt1Im1VIQDtd1iKBXsG5xOJ7Iss2vXLgoKCnA6nf3OkirYt6iqit/vZ8+ePciyjNPp7Pa5xJNQIBAIolABh03myOF5/TPlqumW+uX99SLvf7eHSeXZZLjstPiiLBUHfhyloI8jyzLDhg2jsrKSXbt27e/uCPoRHo+H8vJy5L2wXgtRIRAIBFHoWTaiKyj3F6wpZYX7UzKiM674gwq7GtoZWZBOa5SoEJpCsC9wOp2Ul5cTDAYJhUKdHyAQdILNZsNut++11at/jpgCgUCwF/T3yaH5/kT2p66jKCqPf7AJJfxB6oHaAsG+QpIkHA4HDkc/SXMt6BeIJSqBQCCIpp+rCkWJ3KBwf0pOvNTwH2+s5Y9vrjPeO8PWnj6QRl4gEAh6DSEqBAKBIAq1n6uKTLcDt8NGtschgjw7wfxNGJqvVVPfuKfV2Faa7Ta17d/fG4FAIEiGsNkKBAJBFP09W6MsS0wbkdcv0+X2JnomsF0N7QCMKUrnsqOGYbdJBEMqOZ7uZ00RCASCvs6AEhV6wF1TU9N+7olAIDgQaW1pNl43NaXtx57sf/TnZHSgcn8h1fEgGFKM74WvVaK1pZmtVTUovjamlheSZQ9ySJGLQEjB396Kv73Xuy4QCAT7lFTHgwElKmprawEoKyvbzz0RCASCvkFzczNZWVn7uxs9Tk+MB79d0EOdEQgEgj5AZ+PBgBIVubm5AGzbtq1fDpKgqcmysjK2b99OZmbm/u5OryDusf8wEO6zr96jqqo0NzdTWlq6v7vSK4jxoH8wEO4RBsZ9ins8cEl1PBhQokIv6JGVldWnfpndITMzU9xjP2Ag3CMMjPvsi/fYXyfbIMaD/sZAuEcYGPcp7vHAJJXxQGR/EggEAoFAIBAIBHuFEBUCgUAgEAgEAoFgrxhQosLlcnHHHXfgcrn2d1d6DXGP/YOBcI8wMO5zINxjX2Qg/F7EPfYfBsJ9invs+0hqf80XKBAIBAKBQCAQCPYJA8pSIRAIBAKBQCAQCHoeISoEAoFAIBAIBALBXiFEhUAgEAgEAoFAINgrhKgQCAQCgUAgEAgEe8WAERUPP/www4YNw+12M3nyZD744IP93aVu85vf/AZJkiw/xcXFxn5VVfnNb35DaWkpaWlpzJgxg2+//XY/9rhz3n//fWbNmkVpaSmSJPHKK69Y9qdyTz6fj+uuu478/Hy8Xi9nnHEGO3bs2Id30Tmd3eell14a87s98sgjLW0O5Pu85557OPzww8nIyKCwsJCzzjqLdevWWdr0h99lKvfZ13+X/RkxHhzY4wEMjDGhv48HMDDGBDEeRBgQouL555/nhhtu4JZbbmHVqlVMnz6dU045hW3btu3vrnWbgw8+mMrKSuPn66+/Nvb96U9/4t577+XBBx/k888/p7i4mJNOOonm5ub92OPktLa2csghh/Dggw/G3Z/KPd1www28/PLLPPfcc3z44Ye0tLRw+umnEwqF9tVtdEpn9wnw/e9/3/K7Xbx4sWX/gXyfy5Yt45prruGTTz5hyZIlBINBZs6cSWtrq9GmP/wuU7lP6Nu/y/6KGA8O/PEABsaY0N/HAxgYY4IYD0yoA4Dvfe976tVXX23ZNnbsWPXXv/71furR3nHHHXeohxxySNx9iqKoxcXF6h/+8AdjW0dHh5qVlaU++uij+6iHewegvvzyy8b7VO6poaFBdTgc6nPPPWe02blzpyrLsvrmm2/us753hej7VFVVveSSS9Qzzzwz4TF97T6rq6tVQF22bJmqqv33dxl9n6ra/36X/QUxHvSt8UBVB8aYMBDGA1UdGGPCQB4P+r2lwu/3s3LlSmbOnGnZPnPmTD7++OP91Ku9Z/369ZSWljJs2DDOO+88Nm3aBMDmzZupqqqy3K/L5eLYY4/ts/ebyj2tXLmSQCBgaVNaWsr48eP73H2/9957FBYWMnr0aK644gqqq6uNfX3tPhsbGwHIzc0F+u/vMvo+dfrT77I/IMaDvj8eQP99jsSjvz1DBsKYMJDHg34vKmpqagiFQhQVFVm2FxUVUVVVtZ96tXccccQRPPXUU7z11ls8/vjjVFVVMW3aNGpra4176k/3m8o9VVVV4XQ6ycnJSdimL3DKKafw9NNPs3TpUv7617/y+eefc/zxx+Pz+YC+dZ+qqjJ//nyOPvpoxo8fD/TP32W8+4T+9bvsL4jxAON9X71f6J/PkXj0t2fIQBgTBvp4YN/fHdhXSJJkea+qasy2vsIpp5xivJ4wYQJTp05lxIgR/POf/zQCf/rT/ep055762n2fe+65xuvx48czZcoUhgwZwuuvv84555yT8LgD8T6vvfZavvrqKz788MOYff3pd5noPvvT77K/0Z+ejwN1PID+9RyJR397hgyEMWGgjwf93lKRn5+PzWaLUXrV1dUxyriv4vV6mTBhAuvXrzeyfvSn+03lnoqLi/H7/dTX1yds0xcpKSlhyJAhrF+/Hug793ndddfx6quv8r///Y/Bgwcb2/vb7zLRfcajr/4u+xNiPNDo6/fb354jqdKXnyEDYUwQ48EAEBVOp5PJkyezZMkSy/YlS5Ywbdq0/dSrnsXn81FRUUFJSQnDhg2juLjYcr9+v59ly5b12ftN5Z4mT56Mw+GwtKmsrOSbb77ps/cNUFtby/bt2ykpKQEO/PtUVZVrr72Wl156iaVLlzJs2DDL/v7yu+zsPuPR136X/RExHvT98QD6z3Okq/TFZ8hAGBPEeGBi38WE7z+ee+451eFwqAsXLlTXrFmj3nDDDarX61W3bNmyv7vWLX7+85+r7733nrrp/7dzh66qwwEUx/fCFIMMBMGp6J8g2CwGg2nJIqZVhdv8A6wmm9n/wL9AmElMLgzrFS12BS3iec3Lw3evF/YeerfvB5a2sMMvHA7i3t+1XC7lOI7S6fQtz3A4lGVZmk6nCoJAnU5Htm3rcDg8+c0/dzwe5fu+fN+XYRgajUbyfV/b7VbS9zJ1u10Vi0XNZjOtVis1Gg1VKhVdLpdnxbrzVc7j8ah+v6/FYqHNZiPP81Sr1VQoFH5Mzl6vJ8uyNJ/Ptd/vb9fpdLo9E4WzfJQzCmcZVfTB6/eBFI9OiHofSPHoBPrgQyxGhSSNx2OVy2UlEglVq9U/PvX107Tbbdm2LdM0lc/n1Wq1tF6vb/ev16sGg4FyuZySyaTq9bqCIHjiGz/meZ4Mw7i7XNeV9L1M5/NZb29vymQySqVSchxHu93uCWk+91XO0+mkZrOpbDYr0zRVKpXkuu5dhlfO+bdshmFoMpncnonCWT7KGYWzjDL64LX7QIpHJ0S9D6R4dAJ98OGXJP373z8AAAAAxEXk/1MBAAAA4P9iVAAAAAAIhVEBAAAAIBRGBQAAAIBQGBUAAAAAQmFUAAAAAAiFUQEAAAAgFEYFAAAAgFAYFQAAAABCYVQAAAAACIVRAQAAACAURgUAAACAUH4DIYo46FQXntsAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxUAAAGGCAYAAAANcKzOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAovElEQVR4nO3dfXBV9Z0/8E8kJMGHxPIUiEKMVCtblA6hUqjUStu0+LBibQWdFh+3za6WInZH0VaQ2Wm6OrVPCrYVtE5tS221665UmxaLWLRVDNUqtW5BAxpEUBNQTATO7w9/ZjZNkIQvQpL7es3cGe73fs+93w+HOR/e9557bl6WZVkAAADsoQP29wIAAICeTagAAACSCBUAAEASoQIAAEgiVAAAAEmECgAAIIlQAQAAJBEqAACAJEIFAACQRKiAd3DrrbdGXl5ePPvss/t7KQAA3ZZQAQAAJBEqAAC6kddff31/LwG6TKiALlq0aFGMHj06ioqKon///nHGGWfE6tWr28xZs2ZNTJs2LcrKyqKwsDBKS0vjYx/7WKxatap1ztKlS+OjH/1oDBgwIPr16xfDhw+PM888UzMB2Mv+93//N84///w46qij4sADD4zDDjssTjvttHjiiSfazX311VfjsssuiyOPPDIKCwtj8ODBcfLJJ8df//rX1jnNzc0xb968GDlyZBQVFcWAAQPipJNOihUrVkRExLPPPht5eXlx6623tnv+vLy8mDt3buv9uXPnRl5eXjz22GPxmc98Jt7znvfEiBEjIiLi0UcfjWnTpsURRxwR/fr1iyOOOCLOPvvseO6559o97/PPPx9f+MIXYtiwYVFQUBBlZWXxmc98Jl588cXYunVrHHroofHFL36x3XbPPvts9OnTJ6677rqu/rVCG/n7ewHQk9TU1MSVV14ZZ599dtTU1MTmzZtj7ty5MX78+HjkkUfiqKOOioiIk08+OXbs2BHXXnttDB8+PDZt2hQrVqyIV199NSLeOoifcsopMXHixFi0aFEceuih8fzzz8e9994bLS0tceCBB+7HKgF6lxdeeCEGDBgQ3/jGN2LQoEHx8ssvx49+9KMYN25c1NXVxfve976IiNiyZUuccMIJ8eyzz8bll18e48aNi61bt8YDDzwQDQ0Nccwxx8T27dtj8uTJsXz58pg5c2ZMmjQptm/fHg8//HDU19fHhAkT9miNn/70p2PatGlRXV0dr732WkS81Sve9773xbRp06J///7R0NAQCxYsiA9+8IPx1FNPxcCBAyPirUDxwQ9+MN5888248sor47jjjovNmzfHfffdF6+88kqUlpbGBRdcED/4wQ/i2muvjZKSktbXnT9/fhQUFMQFF1yQ+LdMzsuAXbrllluyiMjWrl2bvfLKK1m/fv2yk08+uc2c+vr6rLCwMDvnnHOyLMuyTZs2ZRGRffvb397l8/7iF7/IIiJbtWrVu7p+ANrbvn171tLSkh111FHZpZde2jo+b968LCKy2traXW572223ZRGR/fCHP9zlnLVr12YRkd1yyy3tHouIbM6cOa3358yZk0VEdvXVV3dq3Vu3bs0OOuig7Dvf+U7r+AUXXJD17ds3e+qpp3a57d///vfsgAMOyL71rW+1jm3bti0bMGBAdv755+/2tWF3nP4EnfTQQw/Ftm3b4rzzzmszPmzYsJg0aVL87ne/i4iI/v37x4gRI+K6666L66+/Purq6mLnzp1ttvnABz4QBQUF8YUvfCF+9KMfxZo1a/ZVGQA5Z/v27fH1r389/umf/ikKCgoiPz8/CgoK4plnnmlz+uqvf/3rOProo+PjH//4Lp/r17/+dRQVFe31d/bPPPPMdmNbt26Nyy+/PN773vdGfn5+5Ofnx8EHHxyvvfZau3WfdNJJMXLkyF0+/5FHHhmnnnpqzJ8/P7Isi4iIn/zkJ7F58+a45JJL9mot5CahAjpp8+bNERExdOjQdo+VlZW1Pp6Xlxe/+93v4pOf/GRce+21MWbMmBg0aFDMmDEjtmzZEhERI0aMiN/+9rcxePDguPjii2PEiBExYsSI+M53vrPvCgLIEbNmzYqvfe1rMWXKlPjv//7v+OMf/xiPPPJIjB49OrZt29Y676WXXorDDz/8HZ/rpZdeirKysjjggL37X6iOess555wTN9xwQ1x00UVx3333xZ/+9Kd45JFHYtCgQV1ed0TEl7/85XjmmWeitrY2IiJuvPHGGD9+fIwZM2bvFULO8p0K6KQBAwZERERDQ0O7x1544YXWc1sjIsrLy2PhwoUREfG3v/0tfv7zn8fcuXOjpaUlbrrppoiImDhxYkycODF27NgRjz76aHzve9+LmTNnRmlpaUybNm0fVASQG3784x/H9OnT4+tf/3qb8U2bNsWhhx7aen/QoEGxfv36d3yuQYMGxYMPPhg7d+7cZbAoKiqKiLe+0P1/vf3mU0fy8vLa3G9sbIz/+Z//iTlz5sQVV1zROt7c3Bwvv/xyuzXtbt0REZMmTYpRo0bFDTfcEAcffHA89thj8eMf/3i320Fn+KQCOmn8+PHRr1+/dgfg9evXx9KlS+NjH/tYh9sdffTR8dWvfjWOPfbYeOyxx9o93qdPnxg3blzceOONEREdzgFgz+Xl5UVhYWGbsXvuuSeef/75NmOTJ0+Ov/3tb7F06dJdPtfkyZPjjTfe6PDKTm8rLS2NoqKiePzxx9uM/9d//VeX1pxlWbt133zzzbFjx452a7r//vvj6aef3u3zzpgxI+65556YPXt2lJaWxmc/+9lOrwneiU8qoJMOPfTQ+NrXvhZXXnllTJ8+Pc4+++zYvHlzXHPNNVFUVBRz5syJiIjHH388LrnkkvjsZz8bRx11VBQUFMTSpUvj8ccfb3236aabboqlS5fGKaecEsOHD4833ngjFi1aFBHxjufyAtB1p556atx6661xzDHHxHHHHRcrV66M6667rt0pQzNnzozFixfH6aefHldccUUcf/zxsW3btli2bFmceuqpcdJJJ8XZZ58dt9xyS1RXV8fTTz8dJ510UuzcuTP++Mc/xsiRI2PatGmRl5cXn/vc52LRokUxYsSIGD16dPzpT3+Kn/zkJ51ec3FxcXzkIx+J6667LgYOHBhHHHFELFu2LBYuXNjm05WIiHnz5sWvf/3r+MhHPhJXXnllHHvssfHqq6/GvffeG7NmzYpjjjmmde7nPve5mD17djzwwAPx1a9+NQoKCpL+bqHV/v6mOHRn//fqT2+7+eabs+OOOy4rKCjISkpKstNPPz178sknWx9/8cUXs/POOy875phjsoMOOig7+OCDs+OOOy771re+lW3fvj3Lsix76KGHsjPOOCMrLy/PCgsLswEDBmQnnnhidvfdd+/rEgF6vVdeeSW78MILs8GDB2cHHnhgdsIJJ2TLly/PTjzxxOzEE09sN/fLX/5yNnz48Kxv377Z4MGDs1NOOSX761//2jpn27Zt2dVXX50dddRRWUFBQTZgwIBs0qRJ2YoVK1rnNDY2ZhdddFFWWlqaHXTQQdlpp52WPfvss7u8+tNLL73Ubt3r16/PzjzzzOw973lPdsghh2Sf+tSnsr/85S9ZeXl5du6557aZu27duuyCCy7IhgwZkvXt2zcrKyvLzjrrrOzFF19s97znnXdelp+fn61fv37P/kKhA3lZ9v8vAQAAQK/W0tISRxxxRJxwwgnx85//fH8vh17E6U8AAL3cSy+9FE8//XTccsst8eKLL7b58jfsDUIFAEAvd88998T5558fQ4cOjfnz57uMLHud058AAIAkXb6k7AMPPBCnnXZalJWVRV5eXvzqV7/a7TbLli2LysrKKCoqiiOPPLL1Ov0A9B76A0Du6nKoeO2112L06NFxww03dGr+2rVr4+STT46JEydGXV1dXHnllTFjxoz45S9/2eXFAtB96Q8AuSvp9Ke8vLy46667YsqUKbucc/nll8fdd98dq1evbh2rrq6OP//5z/HQQw/t6UsD0I3pDwC55V3/ovZDDz0UVVVVbcY++clPxsKFC+PNN9+Mvn37ttumubm5zU/b79y5M15++eUYMGBAu5+xByBNlmWxZcuWKCsriwMO6PIH2HtMfwDo/jrbI971ULFhw4YoLS1tM1ZaWhrbt2+PTZs2xdChQ9ttU1NTE9dcc827vTQA/o9169a1+4Xhd5P+ANBz7K5H7JNLyv7ju0dvn3G1q3eVZs+eHbNmzWq939jYGMOHD49169ZFcXHxu7dQgBzU1NQUw4YNi0MOOWSfv7b+ANC9dbZHvOuhYsiQIbFhw4Y2Yxs3boz8/PwYMGBAh9sUFhZGYWFhu/Hi4mJNA+Bdsq9PH9IfAHqO3fWId/3k2fHjx0dtbW2bsd/85jcxduzYDs+XBSA36A8AvUeXQ8XWrVtj1apVsWrVqoh465KAq1ativr6+oh466Pp6dOnt86vrq6O5557LmbNmhWrV6+ORYsWxcKFC+MrX/nK3qkAgG5BfwDIXV0+/enRRx+Nk046qfX+2+e2nnvuuXHrrbdGQ0NDawOJiKioqIglS5bEpZdeGjfeeGOUlZXFd7/73TjzzDP3wvIB6C70B4DclfQ7FftKU1NTlJSURGNjo3NmAfaynnyM7clrB+gJOnuc3XcXJAcAAHoloQIAAEgiVAAAAEmECgAAIIlQAQAAJBEqAACAJEIFAACQRKgAAACSCBUAAEASoQIAAEgiVAAAAEmECgAAIIlQAQAAJBEqAACAJEIFAACQRKgAAACSCBUAAEASoQIAAEgiVAAAAEmECgAAIIlQAQAAJBEqAACAJEIFAACQRKgAAACSCBUAAEASoQIAAEgiVAAAAEmECgAAIIlQAQAAJBEqAACAJEIFAACQRKgAAACSCBUAAEASoQIAAEgiVAAAAEmECgAAIIlQAQAAJBEqAACAJEIFAACQRKgAAACSCBUAAEASoQIAAEgiVAAAAEmECgAAIIlQAQAAJBEqAACAJEIFAACQRKgAAACSCBUAAEASoQIAAEgiVAAAAEmECgAAIIlQAQAAJBEqAACAJHsUKubPnx8VFRVRVFQUlZWVsXz58necf/vtt8fo0aPjwAMPjKFDh8b5558fmzdv3qMFA9B96Q8AuanLoWLx4sUxc+bMuOqqq6Kuri4mTpwYkydPjvr6+g7nP/jggzF9+vS48MIL48knn4w77rgjHnnkkbjooouSFw9A96E/AOSuLoeK66+/Pi688MK46KKLYuTIkfHtb387hg0bFgsWLOhw/sMPPxxHHHFEzJgxIyoqKuKEE06IL37xi/Hoo48mLx6A7kN/AMhdXQoVLS0tsXLlyqiqqmozXlVVFStWrOhwmwkTJsT69etjyZIlkWVZvPjii/GLX/wiTjnllD1fNQDdiv4AkNu6FCo2bdoUO3bsiNLS0jbjpaWlsWHDhg63mTBhQtx+++0xderUKCgoiCFDhsShhx4a3/ve93b5Os3NzdHU1NTmBkD3pT8A5LY9+qJ2Xl5em/tZlrUbe9tTTz0VM2bMiKuvvjpWrlwZ9957b6xduzaqq6t3+fw1NTVRUlLSehs2bNieLBOAfUx/AMhNeVmWZZ2d3NLSEgceeGDccccdccYZZ7SOf/nLX45Vq1bFsmXL2m3z+c9/Pt5444244447WscefPDBmDhxYrzwwgsxdOjQdts0NzdHc3Nz6/2mpqYYNmxYNDY2RnFxcaeLA2D3mpqaoqSkJOkYqz8A9E6d7RFd+qSioKAgKisro7a2ts14bW1tTJgwocNtXn/99TjggLYv06dPn4h46x2sjhQWFkZxcXGbGwDdl/4AkNu6fPrTrFmz4uabb45FixbF6tWr49JLL436+vrWj6tnz54d06dPb51/2mmnxZ133hkLFiyINWvWxB/+8IeYMWNGHH/88VFWVrb3KgFgv9IfAHJXflc3mDp1amzevDnmzZsXDQ0NMWrUqFiyZEmUl5dHRERDQ0Oba5Kfd955sWXLlrjhhhvisssui0MPPTQmTZoU//mf/7n3qgBgv9MfAHJXl75Tsb/sjfN9AehYTz7G9uS1A/QE78p3KgAAAP6RUAEAACQRKgAAgCRCBQAAkESoAAAAkggVAABAEqECAABIIlQAAABJhAoAACCJUAEAACQRKgAAgCRCBQAAkESoAAAAkggVAABAEqECAABIIlQAAABJhAoAACCJUAEAACQRKgAAgCRCBQAAkESoAAAAkggVAABAEqECAABIIlQAAABJhAoAACCJUAEAACQRKgAAgCRCBQAAkESoAAAAkggVAABAEqECAABIIlQAAABJhAoAACCJUAEAACQRKgAAgCRCBQAAkESoAAAAkggVAABAEqECAABIIlQAAABJhAoAACCJUAEAACQRKgAAgCRCBQAAkESoAAAAkggVAABAEqECAABIIlQAAABJhAoAACCJUAEAACQRKgAAgCRCBQAAkESoAAAAkggVAABAkj0KFfPnz4+KioooKiqKysrKWL58+TvOb25ujquuuirKy8ujsLAwRowYEYsWLdqjBQPQfekPALkpv6sbLF68OGbOnBnz58+PD3/4w/H9738/Jk+eHE899VQMHz68w23OOuusePHFF2PhwoXx3ve+NzZu3Bjbt29PXjwA3Yf+AJC78rIsy7qywbhx42LMmDGxYMGC1rGRI0fGlClToqampt38e++9N6ZNmxZr1qyJ/v3779Eim5qaoqSkJBobG6O4uHiPngOAju2tY6z+AND7dPY426XTn1paWmLlypVRVVXVZryqqipWrFjR4TZ33313jB07Nq699to47LDD4uijj46vfOUrsW3btq68NADdmP4AkNu6dPrTpk2bYseOHVFaWtpmvLS0NDZs2NDhNmvWrIkHH3wwioqK4q677opNmzbFv/3bv8XLL7+8y/Nmm5ubo7m5ufV+U1NTV5YJwD6mPwDktj36onZeXl6b+1mWtRt7286dOyMvLy9uv/32OP744+Pkk0+O66+/Pm699dZdvhtVU1MTJSUlrbdhw4btyTIB2Mf0B4Dc1KVQMXDgwOjTp0+7d502btzY7t2ptw0dOjQOO+ywKCkpaR0bOXJkZFkW69ev73Cb2bNnR2NjY+tt3bp1XVkmAPuY/gCQ27oUKgoKCqKysjJqa2vbjNfW1saECRM63ObDH/5wvPDCC7F169bWsb/97W9xwAEHxOGHH97hNoWFhVFcXNzmBkD3pT8A5LYun/40a9asuPnmm2PRokWxevXquPTSS6O+vj6qq6sj4q13kaZPn946/5xzzokBAwbE+eefH0899VQ88MAD8e///u9xwQUXRL9+/fZeJQDsV/oDQO7q8u9UTJ06NTZv3hzz5s2LhoaGGDVqVCxZsiTKy8sjIqKhoSHq6+tb5x988MFRW1sbX/rSl2Ls2LExYMCAOOuss+I//uM/9l4VAOx3+gNA7ury71TsD65DDvDu6cnH2J68doCe4F35nQoAAIB/JFQAAABJhAoAACCJUAEAACQRKgAAgCRCBQAAkESoAAAAkggVAABAEqECAABIIlQAAABJhAoAACCJUAEAACQRKgAAgCRCBQAAkESoAAAAkggVAABAEqECAABIIlQAAABJhAoAACCJUAEAACQRKgAAgCRCBQAAkESoAAAAkggVAABAEqECAABIIlQAAABJhAoAACCJUAEAACQRKgAAgCRCBQAAkESoAAAAkggVAABAEqECAABIIlQAAABJhAoAACCJUAEAACQRKgAAgCRCBQAAkESoAAAAkggVAABAEqECAABIIlQAAABJhAoAACCJUAEAACQRKgAAgCRCBQAAkESoAAAAkggVAABAEqECAABIIlQAAABJhAoAACCJUAEAACQRKgAAgCRCBQAAkGSPQsX8+fOjoqIiioqKorKyMpYvX96p7f7whz9Efn5+fOADH9iTlwWgm9MfAHJTl0PF4sWLY+bMmXHVVVdFXV1dTJw4MSZPnhz19fXvuF1jY2NMnz49Pvaxj+3xYgHovvQHgNyVl2VZ1pUNxo0bF2PGjIkFCxa0jo0cOTKmTJkSNTU1u9xu2rRpcdRRR0WfPn3iV7/6VaxatarTr9nU1BQlJSXR2NgYxcXFXVkuALuxt46x+gNA79PZ42yXPqloaWmJlStXRlVVVZvxqqqqWLFixS63u+WWW+Lvf/97zJkzpysvB0APoT8A5Lb8rkzetGlT7NixI0pLS9uMl5aWxoYNGzrc5plnnokrrrgili9fHvn5nXu55ubmaG5ubr3f1NTUlWUCsI/pDwC5bY++qJ2Xl9fmfpZl7cYiInbs2BHnnHNOXHPNNXH00Ud3+vlramqipKSk9TZs2LA9WSYA+5j+AJCbuhQqBg4cGH369Gn3rtPGjRvbvTsVEbFly5Z49NFH45JLLon8/PzIz8+PefPmxZ///OfIz8+PpUuXdvg6s2fPjsbGxtbbunXrurJMAPYx/QEgt3Xp9KeCgoKorKyM2traOOOMM1rHa2tr4/TTT283v7i4OJ544ok2Y/Pnz4+lS5fGL37xi6ioqOjwdQoLC6OwsLArSwNgP9IfAHJbl0JFRMSsWbPi85//fIwdOzbGjx8fP/jBD6K+vj6qq6sj4q13kZ5//vm47bbb4oADDohRo0a12X7w4MFRVFTUbhyAnk1/AMhdXQ4VU6dOjc2bN8e8efOioaEhRo0aFUuWLIny8vKIiGhoaNjtNckB6H30B4Dc1eXfqdgfXIcc4N3Tk4+xPXntAD3Bu/I7FQAAAP9IqAAAAJIIFQAAQBKhAgAASCJUAAAASYQKAAAgiVABAAAkESoAAIAkQgUAAJBEqAAAAJIIFQAAQBKhAgAASCJUAAAASYQKAAAgiVABAAAkESoAAIAkQgUAAJBEqAAAAJIIFQAAQBKhAgAASCJUAAAASYQKAAAgiVABAAAkESoAAIAkQgUAAJBEqAAAAJIIFQAAQBKhAgAASCJUAAAASYQKAAAgiVABAAAkESoAAIAkQgUAAJBEqAAAAJIIFQAAQBKhAgAASCJUAAAASYQKAAAgiVABAAAkESoAAIAkQgUAAJBEqAAAAJIIFQAAQBKhAgAASCJUAAAASYQKAAAgiVABAAAkESoAAIAkQgUAAJBEqAAAAJIIFQAAQBKhAgAASCJUAAAASYQKAAAgiVABAAAk2aNQMX/+/KioqIiioqKorKyM5cuX73LunXfeGZ/4xCdi0KBBUVxcHOPHj4/77rtvjxcMQPelPwDkpi6HisWLF8fMmTPjqquuirq6upg4cWJMnjw56uvrO5z/wAMPxCc+8YlYsmRJrFy5Mk466aQ47bTToq6uLnnxAHQf+gNA7srLsizrygbjxo2LMWPGxIIFC1rHRo4cGVOmTImamppOPcf73//+mDp1alx99dWdmt/U1BQlJSXR2NgYxcXFXVkuALuxt46x+gNA79PZ42yXPqloaWmJlStXRlVVVZvxqqqqWLFiRaeeY+fOnbFly5bo37//Luc0NzdHU1NTmxsA3Zf+AJDbuhQqNm3aFDt27IjS0tI246WlpbFhw4ZOPcc3v/nNeO211+Kss87a5ZyampooKSlpvQ0bNqwrywRgH9MfAHLbHn1ROy8vr839LMvajXXkpz/9acydOzcWL14cgwcP3uW82bNnR2NjY+tt3bp1e7JMAPYx/QEgN+V3ZfLAgQOjT58+7d512rhxY7t3p/7R4sWL48ILL4w77rgjPv7xj7/j3MLCwigsLOzK0gDYj/QHgNzWpU8qCgoKorKyMmpra9uM19bWxoQJE3a53U9/+tM477zz4ic/+Umccsope7ZSALot/QEgt3Xpk4qIiFmzZsXnP//5GDt2bIwfPz5+8IMfRH19fVRXV0fEWx9NP//883HbbbdFxFsNY/r06fGd73wnPvShD7W+i9WvX78oKSnZi6UAsD/pDwC5q8uhYurUqbF58+aYN29eNDQ0xKhRo2LJkiVRXl4eERENDQ1trkn+/e9/P7Zv3x4XX3xxXHzxxa3j5557btx6663pFQDQLegPALmry79TsT+4DjnAu6cnH2N78toBeoJ35XcqAAAA/pFQAQAAJBEqAACAJEIFAACQRKgAAACSCBUAAEASoQIAAEgiVAAAAEmECgAAIIlQAQAAJBEqAACAJEIFAACQRKgAAACSCBUAAEASoQIAAEgiVAAAAEmECgAAIIlQAQAAJBEqAACAJEIFAACQRKgAAACSCBUAAEASoQIAAEgiVAAAAEmECgAAIIlQAQAAJBEqAACAJEIFAACQRKgAAACSCBUAAEASoQIAAEgiVAAAAEmECgAAIIlQAQAAJBEqAACAJEIFAACQRKgAAACSCBUAAEASoQIAAEgiVAAAAEmECgAAIIlQAQAAJBEqAACAJEIFAACQRKgAAACSCBUAAEASoQIAAEgiVAAAAEmECgAAIIlQAQAAJBEqAACAJEIFAACQRKgAAACSCBUAAECSPQoV8+fPj4qKiigqKorKyspYvnz5O85ftmxZVFZWRlFRURx55JFx00037dFiAeje9AeA3NTlULF48eKYOXNmXHXVVVFXVxcTJ06MyZMnR319fYfz165dGyeffHJMnDgx6urq4sorr4wZM2bEL3/5y+TFA9B96A8AuSsvy7KsKxuMGzcuxowZEwsWLGgdGzlyZEyZMiVqamrazb/88svj7rvvjtWrV7eOVVdXx5///Od46KGHOvWaTU1NUVJSEo2NjVFcXNyV5QKwG3vrGKs/APQ+nT3O5nflSVtaWmLlypVxxRVXtBmvqqqKFStWdLjNQw89FFVVVW3GPvnJT8bChQvjzTffjL59+7bbprm5OZqbm1vvNzY2RsRbRQGwd719bO3ie0xt6A8AvVNne0SXQsWmTZtix44dUVpa2ma8tLQ0NmzY0OE2GzZs6HD+9u3bY9OmTTF06NB229TU1MQ111zTbnzYsGFdWS4AXbBly5YoKSnZo231B4DebXc9okuh4m15eXlt7mdZ1m5sd/M7Gn/b7NmzY9asWa33X3311SgvL4/6+vo9bng9RVNTUwwbNizWrVuXEx/lq7f3yqVaI3p2vVmWxZYtW6KsrCz5ufSHd1dP/ne2J3Kp3lyqNSK36u3ptXa2R3QpVAwcODD69OnT7l2njRs3tnu36W1DhgzpcH5+fn4MGDCgw20KCwujsLCw3XhJSUmP3Bl7ori4OGdqjVBvb5ZLtUb03HpT/0OuP+xbPfXf2Z7KpXpzqdaI3Kq3J9famR7Rpas/FRQURGVlZdTW1rYZr62tjQkTJnS4zfjx49vN/81vfhNjx47t8HxZAHoe/QEgt3X5krKzZs2Km2++ORYtWhSrV6+OSy+9NOrr66O6ujoi3vpoevr06a3zq6ur47nnnotZs2bF6tWrY9GiRbFw4cL4yle+sveqAGC/0x8AcleXv1MxderU2Lx5c8ybNy8aGhpi1KhRsWTJkigvL4+IiIaGhjbXJK+oqIglS5bEpZdeGjfeeGOUlZXFd7/73TjzzDM7/ZqFhYUxZ86cDj/y7m1yqdYI9fZmuVRrRO7V2xH94d2n3t4rl2qNyK16c6XWLv9OBQAAwP/V5dOfAAAA/i+hAgAASCJUAAAASYQKAAAgSbcPFfPnz4+KioooKiqKysrKWL58+f5e0l4xd+7cyMvLa3MbMmRI6+NZlsXcuXOjrKws+vXrFx/96EfjySef3I8r7rwHHnggTjvttCgrK4u8vLz41a9+1ebxztTW3NwcX/rSl2LgwIFx0EEHxT//8z/H+vXr92EVnbe7es8777x2+/pDH/pQmzk9pd6ampr44Ac/GIccckgMHjw4pkyZEk8//XSbOb1p/3am3t60f3ui3tgjenN/iMitHqE/6A+9Zf92RrcOFYsXL46ZM2fGVVddFXV1dTFx4sSYPHlym0sS9mTvf//7o6GhofX2xBNPtD527bXXxvXXXx833HBDPPLIIzFkyJD4xCc+EVu2bNmPK+6c1157LUaPHh033HBDh493praZM2fGXXfdFT/72c/iwQcfjK1bt8app54aO3bs2FdldNru6o2I+NSnPtVmXy9ZsqTN4z2l3mXLlsXFF18cDz/8cNTW1sb27dujqqoqXnvttdY5vWn/dqbeiN6zf3ua3twjemt/iMitHqE/6A+9Zf92StaNHX/88Vl1dXWbsWOOOSa74oor9tOK9p45c+Zko0eP7vCxnTt3ZkOGDMm+8Y1vtI698cYbWUlJSXbTTTftoxXuHRGR3XXXXa33O1Pbq6++mvXt2zf72c9+1jrn+eefzw444IDs3nvv3Wdr3xP/WG+WZdm5556bnX766bvcpifXu3HjxiwismXLlmVZ1vv37z/Wm2W9e/92d721R+RKf8iy3OoR+kPv3bdZpj9kWZZ1208qWlpaYuXKlVFVVdVmvKqqKlasWLGfVrV3PfPMM1FWVhYVFRUxbdq0WLNmTURErF27NjZs2NCm9sLCwjjxxBN7fO2dqW3lypXx5ptvtplTVlYWo0aN6rH1//73v4/BgwfH0UcfHf/yL/8SGzdubH2sJ9fb2NgYERH9+/ePiN6/f/+x3rf11v3bnfX2HpGL/SGi9x9DOtJbjx/6w1t66/7tSLcNFZs2bYodO3ZEaWlpm/HS0tLYsGHDflrV3jNu3Li47bbb4r777osf/vCHsWHDhpgwYUJs3ry5tb7eWHtnatuwYUMUFBTEe97znl3O6UkmT54ct99+eyxdujS++c1vxiOPPBKTJk2K5ubmiOi59WZZFrNmzYoTTjghRo0aFRG9e/92VG9E792/3V1v7hG52h8ievcxpCO99fihP7ylt+7fXcnf3wvYnby8vDb3syxrN9YTTZ48ufXPxx57bIwfPz5GjBgRP/rRj1q/xNNba4/Ys9p6av1Tp05t/fOoUaNi7NixUV5eHvfcc098+tOf3uV23b3eSy65JB5//PF48MEH2z3WG/fvrurtrfu3p+iNx8lc7w8RvfMY0pHeevzQH97SW/fvrnTbTyoGDhwYffr0aZfUNm7c2C7l9gYHHXRQHHvssfHMM8+0XuWjN9bemdqGDBkSLS0t8corr+xyTk82dOjQKC8vj2eeeSYiema9X/rSl+Luu++O+++/Pw4//PDW8d66f3dVb0d6w/7tCXKpR+RKf4jovceQzuoNxw/9Ydd6w/59J902VBQUFERlZWXU1ta2Ga+trY0JEybsp1W9e5qbm2P16tUxdOjQqKioiCFDhrSpvaWlJZYtW9bja+9MbZWVldG3b982cxoaGuIvf/lLj68/ImLz5s2xbt26GDp0aET0rHqzLItLLrkk7rzzzli6dGlUVFS0eby37d/d1duRnrx/e5Jc6hG50h8iet8xpKt68vFDf9AfuvXVn372s59lffv2zRYuXJg99dRT2cyZM7ODDjooe/bZZ/f30pJddtll2e9///tszZo12cMPP5ydeuqp2SGHHNJa2ze+8Y2spKQku/POO7MnnngiO/vss7OhQ4dmTU1N+3nlu7dly5asrq4uq6uryyIiu/7667O6urrsueeey7Ksc7VVV1dnhx9+ePbb3/42e+yxx7JJkyZlo0ePzrZv376/ytqld6p3y5Yt2WWXXZatWLEiW7t2bXb//fdn48ePzw477LAeWe+//uu/ZiUlJdnvf//7rKGhofX2+uuvt87pTft3d/X2tv3b0/TWHtGb+0OW5VaP0B/0h96yfzujW4eKLMuyG2+8MSsvL88KCgqyMWPGtLlUV082derUbOjQoVnfvn2zsrKy7NOf/nT25JNPtj6+c+fObM6cOdmQIUOywsLC7CMf+Uj2xBNP7McVd97999+fRUS727nnnptlWedq27ZtW3bJJZdk/fv3z/r165edeuqpWX19/X6oZvfeqd7XX389q6qqygYNGpT17ds3Gz58eHbuuee2q6Wn1NtRnRGR3XLLLa1zetP+3V29vW3/9kS9sUf05v6QZbnVI/QH/aG37N/OyMuyLNv7n38AAAC5ott+pwIAAOgZhAoAACCJUAEAACQRKgAAgCRCBQAAkESoAAAAkggVAABAEqECAABIIlQAAABJhAoAACCJUAEAACQRKgAAgCT/D5IFw/6K4u2eAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -565,11 +570,6 @@ "data": { "text/html": [ "
Ep  | loss       accuracy   | val_loss   val_accuracy\n",
-       "0   | 0.050017   0.830078   | 0.049117   0.822591    \n",
-       "1   | 0.040737   0.852539   | 0.043656   0.843750    \n",
-       "2   | 0.043558   0.836914   | 0.043343   0.844076    \n",
-       "3   | 0.031844   0.884766   | 0.038179   0.864366    \n",
-       "4   | 0.032076   0.883789   | 0.037992   0.864041    \n",
        "
" ], "text/plain": [ @@ -578,6 +578,28 @@ }, "metadata": {}, "output_type": "display_data" + }, + { + "ename": "IndexError", + "evalue": "arrays used as indices must be of integer (or boolean) type", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/06_training.ipynb Cell 18\u001b[0m line \u001b[0;36m2\n\u001b[1;32m 1\u001b[0m \u001b[39m#| eval: false\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m student\u001b[39m.\u001b[39;49mfit(epochs\u001b[39m=\u001b[39;49m\u001b[39m5\u001b[39;49m)\n", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/06_training.ipynb Cell 18\u001b[0m line \u001b[0;36m2\n\u001b[1;32m 26\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstep \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstep \u001b[39mif\u001b[39;00m start_step \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39melse\u001b[39;00m start_step\n\u001b[1;32m 27\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mepochs \u001b[39m=\u001b[39m \u001b[39mrange\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstart_epoch, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstart_epoch \u001b[39m+\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mn_epochs)\n\u001b[0;32m---> 28\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdo_fit()\n", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/06_training.ipynb Cell 18\u001b[0m line \u001b[0;36m1\n\u001b[1;32m 11\u001b[0m \u001b[39mfor\u001b[39;00m callback \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallbacks:\n\u001b[1;32m 12\u001b[0m \u001b[39mgetattr\u001b[39m(callback, pre_name, noop)(\u001b[39mself\u001b[39m)\n\u001b[0;32m---> 14\u001b[0m func(\u001b[39mself\u001b[39;49m)\n\u001b[1;32m 15\u001b[0m \u001b[39mfor\u001b[39;00m callback \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallbacks:\n\u001b[1;32m 16\u001b[0m \u001b[39mgetattr\u001b[39m(callback, post_name, noop)(\u001b[39mself\u001b[39m)\n", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/06_training.ipynb Cell 18\u001b[0m line \u001b[0;36m3\n\u001b[1;32m 32\u001b[0m \u001b[39mfor\u001b[39;00m e \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mepochs:\n\u001b[1;32m 33\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mepoch \u001b[39m=\u001b[39m e\n\u001b[0;32m---> 34\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdo_epoch()\n", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/06_training.ipynb Cell 18\u001b[0m line \u001b[0;36m1\n\u001b[1;32m 11\u001b[0m \u001b[39mfor\u001b[39;00m callback \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallbacks:\n\u001b[1;32m 12\u001b[0m \u001b[39mgetattr\u001b[39m(callback, pre_name, noop)(\u001b[39mself\u001b[39m)\n\u001b[0;32m---> 14\u001b[0m func(\u001b[39mself\u001b[39;49m)\n\u001b[1;32m 15\u001b[0m \u001b[39mfor\u001b[39;00m callback \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallbacks:\n\u001b[1;32m 16\u001b[0m \u001b[39mgetattr\u001b[39m(callback, post_name, noop)(\u001b[39mself\u001b[39m)\n", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/06_training.ipynb Cell 18\u001b[0m line \u001b[0;36m4\n\u001b[1;32m 38\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtraining \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[1;32m 39\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdl \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdataloaders\u001b[39m.\u001b[39mtrain\n\u001b[0;32m---> 40\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdo_all_batches()\n\u001b[1;32m 41\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdl \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdataloaders\u001b[39m.\u001b[39mtest\n\u001b[1;32m 42\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtraining \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/06_training.ipynb Cell 18\u001b[0m line \u001b[0;36m1\n\u001b[1;32m 11\u001b[0m \u001b[39mfor\u001b[39;00m callback \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallbacks:\n\u001b[1;32m 12\u001b[0m \u001b[39mgetattr\u001b[39m(callback, pre_name, noop)(\u001b[39mself\u001b[39m)\n\u001b[0;32m---> 14\u001b[0m func(\u001b[39mself\u001b[39;49m)\n\u001b[1;32m 15\u001b[0m \u001b[39mfor\u001b[39;00m callback \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallbacks:\n\u001b[1;32m 16\u001b[0m \u001b[39mgetattr\u001b[39m(callback, post_name, noop)(\u001b[39mself\u001b[39m)\n", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/06_training.ipynb Cell 18\u001b[0m line \u001b[0;36m5\n\u001b[1;32m 49\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbatch \u001b[39m=\u001b[39m batch\n\u001b[1;32m 50\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdo_batch_forward()\n\u001b[0;32m---> 51\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdo_calc_loss()\n\u001b[1;32m 52\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtraining: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdo_batch_backward()\n", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/06_training.ipynb Cell 18\u001b[0m line \u001b[0;36m1\n\u001b[1;32m 11\u001b[0m \u001b[39mfor\u001b[39;00m callback \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallbacks:\n\u001b[1;32m 12\u001b[0m \u001b[39mgetattr\u001b[39m(callback, pre_name, noop)(\u001b[39mself\u001b[39m)\n\u001b[0;32m---> 14\u001b[0m func(\u001b[39mself\u001b[39;49m)\n\u001b[1;32m 15\u001b[0m \u001b[39mfor\u001b[39;00m callback \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallbacks:\n\u001b[1;32m 16\u001b[0m \u001b[39mgetattr\u001b[39m(callback, post_name, noop)(\u001b[39mself\u001b[39m)\n", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/06_training.ipynb Cell 18\u001b[0m line \u001b[0;36m5\n\u001b[1;32m 54\u001b[0m \u001b[39m@add_callbacks\u001b[39m\n\u001b[1;32m 55\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdo_calc_loss\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 56\u001b[0m _, y \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbatch\n\u001b[0;32m---> 57\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mloss \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mloss_func(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mpreds, y)\n", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/06_training.ipynb Cell 18\u001b[0m line \u001b[0;36m2\n\u001b[1;32m 21\u001b[0m MM_func \u001b[39m=\u001b[39m partial(linear_model, params\u001b[39m=\u001b[39m[w1, b1, w2])\n\u001b[1;32m 22\u001b[0m optimizer \u001b[39m=\u001b[39m Adam([w1, b1, w2], lr\u001b[39m=\u001b[39m\u001b[39m0.005\u001b[39m)\n\u001b[0;32m---> 24\u001b[0m loss_f \u001b[39m=\u001b[39m \u001b[39mlambda\u001b[39;00m preds, targets: CrossEntropy_loss(preds, one_hot_encode_batch(targets\u001b[39m.\u001b[39;49mdata, n_classes\u001b[39m=\u001b[39;49m\u001b[39m10\u001b[39;49m))\n\u001b[1;32m 25\u001b[0m \u001b[39m# loss_f = lambda preds, targets: CrossEntropy_loss(preds, one_hot_encode_batch(targets.data, 10))\u001b[39;00m\n\u001b[1;32m 27\u001b[0m student \u001b[39m=\u001b[39m Learner(\n\u001b[1;32m 28\u001b[0m dataloaders\u001b[39m=\u001b[39mDataLoaders(mnist_train, mnist_test),\n\u001b[1;32m 29\u001b[0m model\u001b[39m=\u001b[39mMM_func,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 35\u001b[0m ], plot_train_skip_ylim\u001b[39m=\u001b[39m\u001b[39m15\u001b[39m, plot_smooth_training\u001b[39m=\u001b[39m\u001b[39m5\u001b[39m)],\n\u001b[1;32m 36\u001b[0m )\n", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/06_training.ipynb Cell 18\u001b[0m line \u001b[0;36m5\n\u001b[1;32m 3\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mone_hot_encode_batch\u001b[39m(y, n_classes):\n\u001b[1;32m 4\u001b[0m diag \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39meye(n_classes)\n\u001b[0;32m----> 5\u001b[0m \u001b[39mreturn\u001b[39;00m Tensor(diag[y])\n", + "\u001b[0;31mIndexError\u001b[0m: arrays used as indices must be of integer (or boolean) type" + ] } ], "source": [ diff --git a/nbs/10_utils.grad_check.ipynb b/nbs/10_utils.grad_check.ipynb index 3d16cc5..f09cd87 100644 --- a/nbs/10_utils.grad_check.ipynb +++ b/nbs/10_utils.grad_check.ipynb @@ -30,7 +30,7 @@ "source": [ "# | export\n", "import numpy as np\n", - "import tidygrad as tg\n" + "import tidygrad as tg" ] }, { @@ -55,7 +55,8 @@ " grad_view = p.grad.reshape(-1)\n", "\n", " slow_grad = np.zeros_like(p.grad)\n", - " slow_grad_view = slow_grad.reshape(-1)\n", + "\n", + " scaled_slow_grad_view = slow_grad.reshape(-1)\n", "\n", " indices = np.random.choice(np.arange(grad_view.size), size=min(n, grad_view.size), replace=False)\n", " good_indices = []\n", @@ -73,16 +74,26 @@ " data_view[idx] = old_val + eps\n", " loss_plus_h = func(inputs, params)\n", "\n", - " slow_grad_view[idx] = (loss_plus_h.data - loss.data) / eps\n", + " scaled_slow_grad_view[idx] = (loss_plus_h.data - loss.data) / eps\n", + " # slow_grad_view[idx] =\n", + "\n", + " # (loss_plus_h.data - loss.data) / eps\n", + "\n", " if verbose:\n", - " print(f\"{idx}: loss_plus_h: {loss_plus_h.data}, loss: {loss.data}, diff: {loss_plus_h.data - loss.data}, grad: {grad_view[idx]}, slow_grad: {slow_grad_view[idx]}\")\n", + " print(\n", + " f\"{idx}: loss_plus_h: {loss_plus_h.data}, loss: {loss.data}, diff: {loss_plus_h.data - loss.data}, grad: {grad_view[idx]}, slow_grad: {scaled_slow_grad_view[idx] / eps}\"\n", + " )\n", " data_view[idx] = old_val\n", "\n", - " if abs(slow_grad_view[idx]) > eps:\n", + " if abs(scaled_slow_grad_view[idx]) > eps:\n", " good_indices.append(idx)\n", "\n", + " differences = ( (scaled_slow_grad_view[good_indices] - grad_view[good_indices])\n", + " / (grad_view[good_indices])\n", + " )\n", + "\n", + " # slow_grad /= eps\n", "\n", - " differences = (slow_grad_view[good_indices] - grad_view[good_indices]) / slow_grad_view[good_indices]\n", " max_grad_diff = np.max(np.abs(differences))\n", " print(f\"Max fractional gradient difference for {p.name}: {max_grad_diff*100:.4f}%\")\n", " if max_grad_diff > 1e-2:\n", @@ -92,7 +103,6 @@ " print(\"Fast grad: \", p.grad)\n", " print(\"Differences: \", differences)\n", "\n", - "\n", " if grad_failed: raise ValueError(f\"Gradient check failed for {p.name}: Max error: {max_grad_diff*100:.4f}\")" ] }, @@ -114,7 +124,7 @@ { "data": { "text/plain": [ - "array[32, 10] n=320 (2.5Kb) x∈[-8.202, 7.625] μ=-0.278 σ=2.839" + "array[32, 10] n=320 (2.5Kb) x∈[-7.871, 6.829] μ=-0.001 σ=2.903" ] }, "execution_count": null, @@ -145,8 +155,8 @@ "output_type": "stream", "text": [ "Max fractional gradient difference for w2: 0.0011%\n", - "Max fractional gradient difference for b1: 0.0011%\n", - "Max fractional gradient difference for w1: 0.0156%\n" + "Max fractional gradient difference for b1: 0.0010%\n", + "Max fractional gradient difference for w1: 0.0159%\n" ] } ], @@ -173,7 +183,6 @@ "\n", " loss = -tg.BCE_loss(z2, y).sum(\"loss\")\n", "\n", - "\n", " return loss\n", "\n", "debug = []\n", diff --git a/nbs/examples/gpt2.ipynb b/nbs/examples/gpt2.ipynb deleted file mode 100644 index 9470fe3..0000000 --- a/nbs/examples/gpt2.ipynb +++ /dev/null @@ -1,626 +0,0 @@ -{ - "cells": [ - { - "cell_type": "raw", - "metadata": {}, - "source": [ - "---\n", - "skip_exec: true\n", - "---" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tidygrad.tensor import Tensor\n", - "from tidygrad.functional import Embedding, embedding\n", - "import numpy as np\n", - "from lovely_numpy import Lo\n", - "\n", - "from transformers import GPT2Tokenizer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from safetensors import safe_open" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "text = \"In a hole in the ground there lived a\"\n", - "tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n", - "\n", - "# tokens = tokenizer.encode(text) # returns a list of integers\n", - "# tokens = Tensor(tokens)\n", - "\n", - "tokens = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tokens" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model = safe_open(\"model.safetensors\", framework=\"np\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['h.9.ln_2.bias',\n", - " 'h.9.ln_2.weight',\n", - " 'h.9.mlp.c_fc.bias',\n", - " 'h.9.mlp.c_fc.weight',\n", - " 'h.9.mlp.c_proj.bias',\n", - " 'h.9.mlp.c_proj.weight',\n", - " 'ln_f.bias',\n", - " 'ln_f.weight',\n", - " 'wpe.weight',\n", - " 'wte.weight']" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.keys()[-10:]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tensor[1024, 768](name=\"?\" op=Load):\n", - " v=array[1024, 768] f32 n=786432 (3Mb) x∈[-4.538, 4.065] μ=-0.001 σ=0.123\n", - " \n", - "Tensor[50257, 768](name=\"?\" op=Load):\n", - " v=array[50257, 768] f32 n=38597376 (0.1Gb) x∈[-1.270, 1.785] μ=0.000 σ=0.144\n", - " \n" - ] - } - ], - "source": [ - "wte = Tensor(model.get_tensor(\"wte.weight\"))\n", - "wpe = Tensor(model.get_tensor(\"wpe.weight\"))\n", - "\n", - "print(wpe)\n", - "print(wte)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import tidygrad" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Tensor[10, 768](name=\"(embedding(?)+embedding(?))\" op=Add):\n", - " v=array[10, 768] f32 n=7680 (30Kb) x∈[-4.511, 3.938] μ=-9.411e-05 σ=0.219\n", - " " - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "token_embeddings = embedding(wte, tokens)\n", - "\n", - "positions = np.arange(len(tokens))\n", - "position_embeddings = embedding(wpe, positions)\n", - "\n", - "embeddings = token_embeddings + position_embeddings\n", - "Lo(embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ln_1_w = model.get_tensor(\"h.0.ln_1.weight\")\n", - "ln_1_b = model.get_tensor(\"h.0.ln_1.bias\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def layer_norm(x, w, b, eps=1e-5):\n", - " mu = x.mean(axis=-1, keepdims=True)\n", - " sigma = x.std(axis=-1, keepdims=True, correction=0)\n", - "\n", - " return (\n", - " (x - mu) / (sigma + eps)\n", - " ) * w + b # tensor[10, 768] n=7680 (30Kb) x∈[-0.788, 0.579] μ=-0.005 σ=0.106" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Tensor[10, 768](name=\"(((((embedding(?)+embedding(?))-(sum((embedding(?)+embedding(?)))/?))/(pow((sum(var)/?),0.5)+?))*?)+?)\" op=Add):\n", - " v=array[10, 768] f32 n=7680 (30Kb) x∈[-0.788, 0.579] μ=-0.005 σ=0.106\n", - " " - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ln_1 = layer_norm(embeddings, ln_1_w, ln_1_b)\n", - "ln_1\n", - "\n", - "# tensor[10, 768] n=7680 (30Kb) x∈[-0.788, 0.579] μ=-0.005 σ=0.106" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "attn_w_qkv = model.get_tensor(\"h.0.attn.c_attn.weight\")\n", - "attn_b_qkv = model.get_tensor(\"h.0.attn.c_attn.bias\")\n", - "\n", - "attn_w_q, attn_w_k, attn_w_v = np.split(attn_w_qkv, 3, axis=-1)\n", - "attn_b_q, attn_b_k, attn_b_v = np.split(attn_b_qkv, 3, axis=-1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "array[12, 10, 64] f32 n=7680 (30Kb) x∈[-4.234, 4.473] μ=-0.064 σ=0.971\n", - "array[12, 10, 64] f32 n=7680 (30Kb) x∈[-6.097, 6.787] μ=0.034 σ=1.350\n", - "array[12, 64, 10] f32 n=7680 (30Kb) x∈[-6.097, 6.787] μ=0.034 σ=1.350\n" - ] - }, - { - "data": { - "text/plain": [ - "array[12, 10, 10] f32 n=1200 (4.7Kb) x∈[-7.848, 11.893] μ=-0.591 σ=2.526" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "q = ln_1.mmul(attn_w_q) + attn_b_q\n", - "k = ln_1.mmul(attn_w_k) + attn_b_k\n", - "v = ln_1.mmul(attn_w_v) + attn_b_v\n", - "\n", - "q_chunked_np = np.array_split(q.data, 12, axis=-1)\n", - "k_chunked_np = np.array_split(k.data, 12, axis=-1)\n", - "v_chunked_np = np.array_split(v.data, 12, axis=-1)\n", - "\n", - "q_chunked_np = np.stack(q_chunked_np, axis=0)\n", - "k_chunked_np = np.stack(k_chunked_np, axis=0)\n", - "v_chunked_np = np.stack(v_chunked_np, axis=0)\n", - "\n", - "# q_chunked = Tensor(q_chunked_np, name=\"q_chunked\")\n", - "# k_chunked = Tensor(k_chunked_np, name=\"k_chunked\")\n", - "# v_chunked = Tensor(v_chunked_np, name=\"v_chunked\")\n", - "\n", - "# attention = q_chunked_np.mmul(k_chunked_np.transpose(-1, -2)) / np.sqrt(64)\n", - "\n", - "print(Lo(q_chunked_np))\n", - "print(Lo(k_chunked_np))\n", - "print(Lo(k_chunked_np.swapaxes(-1, -2)))\n", - "\n", - "attention = np.matmul(q_chunked_np, k_chunked_np.swapaxes(-1, -2)) / np.sqrt(64)\n", - "Lo(attention)\n", - "\n", - "# Lo(q_chunked_np).chans(scale=5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array[10, 768] n=7680 (60Kb) x∈[-1.057, 1.432] μ=0.003 σ=0.166" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "mask = np.tril(np.ones(attention.shape), k=0) # * (np.finfo(float).min)\n", - "ee = np.exp(attention) * mask\n", - "\n", - "softmaxed = ee / ee.sum(axis=-1, keepdims=True)\n", - "\n", - "# print(Lo(softmaxed))\n", - "\n", - "attention_output = np.matmul(softmaxed, v_chunked_np)\n", - "# print(Lo(attention_output))\n", - "\n", - "attention_chunks = attention_output[:]\n", - "Lo(attention_chunks[0])\n", - "attention_reshaped_np = np.concatenate(attention_chunks, axis=-1)\n", - "Lo(attention_reshaped_np)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tensor[10, 768](name=\"((?@?)+?)\" op=Add):\n", - " v=array[10, 768] n=7680 (60Kb) x∈[-14.188, 14.257] μ=0.011 σ=1.083\n", - " \n", - "Tensor[10, 768](name=\"(((?@?)+?)+(embedding(?)+embedding(?)))\" op=Add):\n", - " v=array[10, 768] n=7680 (60Kb) x∈[-14.241, 14.485] μ=0.011 σ=1.123\n", - " \n", - "Tensor[10, 768](name=\"(((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)\" op=Add):\n", - " v=array[10, 768] n=7680 (60Kb) x∈[-2.793, 1.674] μ=0.005 σ=0.160\n", - " \n" - ] - } - ], - "source": [ - "cproj_w_np = model.get_tensor(\"h.0.attn.c_proj.weight\")\n", - "cproj_b_np = model.get_tensor(\"h.0.attn.c_proj.bias\")\n", - "\n", - "cproj_w = Tensor(cproj_w_np)\n", - "cproj_b = Tensor(cproj_b_np)\n", - "\n", - "attention_reshaped = Tensor(attention_reshaped_np)\n", - "\n", - "crosstalk = attention_reshaped.mmul(cproj_w) + cproj_b\n", - "print(crosstalk)\n", - "\n", - "after_residual = crosstalk + embeddings\n", - "print(after_residual)\n", - "\n", - "ln2_w = Tensor(model.get_tensor(\"h.0.ln_2.weight\"), name=\"ln2_w\")\n", - "ln2_b = Tensor(model.get_tensor(\"h.0.ln_2.bias\"), name=\"ln2_b\")\n", - "\n", - "after_ln2 = layer_norm(after_residual, ln2_w, ln2_b)\n", - "\n", - "print(after_ln2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tensor[10, 3072](name=\"(((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b)\" op=Add):\n", - " v=array[10, 3072] n=30720 (0.2Mb) x∈[-6.346, 10.617] μ=-1.086 σ=0.855\n", - " \n" - ] - } - ], - "source": [ - "mlp_c_fc_w = Tensor(model.get_tensor(\"h.0.mlp.c_fc.weight\"), name=\"fc_w\")\n", - "mlp_c_fc_b = Tensor(model.get_tensor(\"h.0.mlp.c_fc.bias\"), name=\"fc_b\")\n", - "\n", - "after_up = after_ln2.mmul(mlp_c_fc_w) + mlp_c_fc_b\n", - "\n", - "print(after_up)\n", - "# mlp_c_fca = gelu(mlp_c_fc)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tidygrad.functional import sigmoid, tanh\n", - "import math" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def gelu(x: Tensor):\n", - " return x * sigmoid(1.702 * x)\n", - "\n", - "def new_gelu(input):\n", - " return (0.5 * input * (1.0 + tanh(math.sqrt(2.0 / math.pi) * (input + 0.044715 * input.pow(3)))))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Tensor[10, 768](name=\"((((((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b)*?)*(tanh((((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b)+(pow((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b),3)*?))*?))+?))@proj_w)+proj_b)+(((?@?)+?)+(embedding(?)+embedding(?))))\" op=Add):\n", - " v=array[10, 768] n=7680 (60Kb) x∈[-67.477, 97.448] μ=0.023 σ=2.375\n", - " " - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "after_up_a = new_gelu(after_up)\n", - "\n", - "mlp_c_proj_w = Tensor(model.get_tensor(\"h.0.mlp.c_proj.weight\"), name=\"proj_w\")\n", - "mlp_c_proj_b = Tensor(model.get_tensor(\"h.0.mlp.c_proj.bias\"), name=\"proj_b\")\n", - "\n", - "after_down = after_up_a.mmul(mlp_c_proj_w) + mlp_c_proj_b\n", - "\n", - "attention_output = after_down + after_residual\n", - "attention_output" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " === Block 0 ===\n", - "ln_1 Tensor[10, 768](name=\"(((((embedding(?)+embedding(?))-(sum((embedding(?)+embedding(?)))/?))/(pow((sum(var)/?),0.5)+?))*?)+?)\" op=Add):\n", - " v=array[10, 768] f32 n=7680 (30Kb) x∈[-0.788, 0.579] μ=-0.005 σ=0.106\n", - " \n" - ] - } - ], - "source": [ - "def transformer_block(model_weigts, i, inputs):\n", - "\n", - " print(f\" === Block {i} ===\")\n", - "\n", - " ln_1_w = model.get_tensor(f\"h.{i}.ln_1.weight\")\n", - " ln_1_b = model.get_tensor(f\"h.{i}.ln_1.bias\")\n", - "\n", - " # ln_1 = embeddings\n", - " ln_1 = layer_norm(embeddings, ln_1_w, ln_1_b)\n", - " print(\"ln_1\", ln_1)\n", - "\n", - " attn_w_qkv = model.get_tensor(f\"h.{i}.attn.c_attn.weight\")\n", - " attn_b_qkv = model.get_tensor(f\"h.{i}.attn.c_attn.bias\")\n", - "\n", - " attn_w_q, attn_w_k, attn_w_v = np.split(attn_w_qkv, 3, axis=-1)\n", - " attn_b_q, attn_b_k, attn_b_v = np.split(attn_b_qkv, 3, axis=-1)\n", - "\n", - " q = ln_1.mmul(attn_w_q) + attn_b_q\n", - " k = ln_1.mmul(attn_w_k) + attn_b_k\n", - " v = ln_1.mmul(attn_w_v) + attn_b_v\n", - "\n", - " q_chunked_np = np.array_split(q.data, 12, axis=-1)\n", - " k_chunked_np = np.array_split(k.data, 12, axis=-1)\n", - " v_chunked_np = np.array_split(v.data, 12, axis=-1)\n", - "\n", - " q_chunked_np = np.stack(q_chunked_np, axis=0)\n", - " k_chunked_np = np.stack(k_chunked_np, axis=0)\n", - " v_chunked_np = np.stack(v_chunked_np, axis=0)\n", - "\n", - " attention = np.matmul(q_chunked_np, k_chunked_np.swapaxes(-1, -2)) / np.sqrt(64)\n", - "\n", - " mask = np.tril(np.ones(attention.shape), k=0) # * (np.finfo(float).min)\n", - " ee = np.exp(attention) * mask\n", - "\n", - " softmaxed = ee / ee.sum(axis=-1, keepdims=True)\n", - "\n", - " attention_output = np.matmul(softmaxed, v_chunked_np)\n", - " attention_chunks = attention_output[:]\n", - " attention_reshaped_np = np.concatenate(attention_chunks, axis=-1)\n", - "\n", - " cproj_w = Tensor(model.get_tensor(f\"h.{i}.attn.c_proj.weight\"))\n", - " cproj_b = Tensor(model.get_tensor(f\"h.{i}.attn.c_proj.bias\"))\n", - "\n", - " attention_reshaped = Tensor(attention_reshaped_np)\n", - "\n", - " crosstalk = attention_reshaped.mmul(cproj_w) + cproj_b\n", - "\n", - " after_residual = crosstalk + embeddings\n", - "\n", - " ln2_w = Tensor(model.get_tensor(f\"h.{i}.ln_2.weight\"), name=\"ln2_w\")\n", - " ln2_b = Tensor(model.get_tensor(f\"h.{i}.ln_2.bias\"), name=\"ln2_b\")\n", - "\n", - " after_ln2 = layer_norm(after_residual, ln2_w, ln2_b)\n", - "\n", - " mlp_c_fc_w = Tensor(model.get_tensor(f\"h.{i}.mlp.c_fc.weight\"), name=\"fc_w\")\n", - " mlp_c_fc_b = Tensor(model.get_tensor(f\"h.{i}.mlp.c_fc.bias\"), name=\"fc_b\")\n", - "\n", - " after_up = after_ln2.mmul(mlp_c_fc_w) + mlp_c_fc_b\n", - "\n", - " after_up_a = new_gelu(after_up)\n", - "\n", - " mlp_c_proj_w = Tensor(model.get_tensor(f\"h.{i}.mlp.c_proj.weight\"), name=\"proj_w\")\n", - " mlp_c_proj_b = Tensor(model.get_tensor(f\"h.{i}.mlp.c_proj.bias\"), name=\"proj_b\")\n", - "\n", - " after_down = after_up_a.mmul(mlp_c_proj_w) + mlp_c_proj_b\n", - "\n", - " output = after_down + after_residual\n", - " return output\n", - "\n", - "\n", - "res = transformer_block(model, 0, embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " === Block 0 ===\n", - "ln_1 Tensor[10, 768](name=\"(((((embedding(?)+embedding(?))-(sum((embedding(?)+embedding(?)))/?))/(pow((sum(var)/?),0.5)+?))*?)+?)\" op=Add):\n", - " v=array[10, 768] f32 n=7680 (30Kb) x∈[-0.788, 0.579] μ=-0.005 σ=0.106\n", - " \n", - "Embedding out: Tensor[10, 768](name=\"((((((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b)*?)*(tanh((((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b)+(pow((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b),3)*?))*?))+?))@proj_w)+proj_b)+(((?@?)+?)+(embedding(?)+embedding(?))))\" op=Add):\n", - " v=array[10, 768] n=7680 (60Kb) x∈[-67.477, 97.448] μ=0.023 σ=2.375\n", - " \n", - " === Block 1 ===\n", - "ln_1 Tensor[10, 768](name=\"(((((embedding(?)+embedding(?))-(sum((embedding(?)+embedding(?)))/?))/(pow((sum(var)/?),0.5)+?))*?)+?)\" op=Add):\n", - " v=array[10, 768] f32 n=7680 (30Kb) x∈[-3.292, 2.614] μ=-0.005 σ=0.247\n", - " \n", - "Embedding out: Tensor[10, 768](name=\"((((((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b)*?)*(tanh((((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b)+(pow((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b),3)*?))*?))+?))@proj_w)+proj_b)+(((?@?)+?)+(embedding(?)+embedding(?))))\" op=Add):\n", - " v=array[10, 768] n=7680 (60Kb) x∈[-33.394, 360.662] μ=0.620 σ=5.143\n", - " \n" - ] - }, - { - "data": { - "text/plain": [ - "Tensor[10, 768](name=\"((((((((((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b)*?)*(tanh((((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b)+(pow((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b),3)*?))*?))+?))@proj_w)+proj_b)+(((?@?)+?)+(embedding(?)+embedding(?))))-(sum(((((((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b)*?)*(tanh((((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b)+(pow((((((((((?@?)+?)+(embedding(?)+embedding(?)))-(sum((((?@?)+?)+(embedding(?)+embedding(?))))/?))/(pow((sum(var)/?),0.5)+?))*ln2_w)+ln2_b)@fc_w)+fc_b),3)*?))*?))+?))@proj_w)+proj_b)+(((?@?)+?)+(embedding(?)+embedding(?)))))/?))/(pow((sum(var)/?),0.5)+?))*?)+?)\" op=Add):\n", - " v=array[10, 768] n=7680 (60Kb) x∈[-10.000, 24.001] μ=-0.032 σ=0.977\n", - " " - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def transformer(model, token_ids):\n", - " wte = Tensor(model.get_tensor(\"wte.weight\"))\n", - " wpe = Tensor(model.get_tensor(\"wpe.weight\"))\n", - "\n", - " token_embeddings = embedding(wte, tokens)\n", - "\n", - " positions = np.arange(len(tokens))\n", - " position_embeddings = embedding(wpe, positions)\n", - "\n", - " embeddings = token_embeddings + position_embeddings\n", - "\n", - " for i in range(2):\n", - " embeddings = transformer_block(model, i, embeddings)\n", - " print(\"Embedding out:\", embeddings)\n", - "\n", - " ln_f_w = Tensor(model.get_tensor(\"ln_f.weight\"))\n", - " ln_f_b = Tensor(model.get_tensor(\"ln_f.bias\"))\n", - "\n", - " res = layer_norm(embeddings, ln_f_w, ln_f_b)\n", - "\n", - " return res\n", - "\n", - "tidygrad.tensor._grad = True\n", - "\n", - "res = transformer(model, tokens)\n", - "\n", - "# def gpt2_language_model(model, token_ids):\n", - "# res = transformer(model, token_ids)\n", - "\n", - "# wte = Tensor(model.get_tensor(\"wte.weight\").swapaxes(-1, -2))\n", - "# logits = res.mmul(wte)\n", - "# return logits\n", - "\n", - "# res = gpt2_language_model(model, tokens)\n", - "res\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "python3", - "language": "python", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/nbs/examples/gpt2_training.ipynb b/nbs/examples/gpt2_training.ipynb new file mode 100644 index 0000000..8548617 --- /dev/null +++ b/nbs/examples/gpt2_training.ipynb @@ -0,0 +1,331 @@ +{ + "cells": [ + { + "cell_type": "raw", + "metadata": {}, + "source": [ + "---\n", + "skip_exec: true\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tidygrad as tg\n", + "from tidygrad import Tensor\n", + "import numpy as np\n", + "\n", + "import huggingface_hub\n", + "\n", + "import datasets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ds = datasets.load_dataset(\"roneneldan/TinyStories\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "n_vocab = 1024\n", + "n_layers = 2\n", + "n_heads = 4\n", + "ndim = 512\n", + "ctx_len = 128" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def gpt2_new(n_vocab, n_layers, n_heads, ndim):\n", + " shape_dict = {\n", + " \"wte\": [n_vocab, ndim],\n", + " \"wpe\": [ctx_len, ndim],\n", + " \"ln_f.weight\": [ndim],\n", + " \"ln_f.bias\": [ndim],\n", + " }\n", + "\n", + " for i in range(n_layers):\n", + " shape_dict[f\"h.{i}.ln_1.weight\"] = [ndim]\n", + " shape_dict[f\"h.{i}.ln_1.bias\"] = [ndim]\n", + "\n", + " shape_dict[f\"h.{i}.attn.c_attn.weight\"] = [ndim, 3 * ndim]\n", + " shape_dict[f\"h.{i}.attn.c_attn.bias\"] = [3 * ndim]\n", + "\n", + " shape_dict[f\"h.{i}.attn.c_proj.weight\"] = [ndim, ndim]\n", + " shape_dict[f\"h.{i}.attn.c_proj.bias\"] = [ndim]\n", + "\n", + " shape_dict[f\"h.{i}.ln_2.weight\"] = [ndim]\n", + " shape_dict[f\"h.{i}.ln_2.bias\"] = [ndim]\n", + "\n", + " shape_dict[f\"h.{i}.mlp.c_fc.weight\"] = [ndim, 4 * ndim]\n", + " shape_dict[f\"h.{i}.mlp.c_fc.bias\"] = [4 * ndim]\n", + "\n", + " shape_dict[f\"h.{i}.mlp.c_proj.weight\"] = [4 * ndim, ndim]\n", + " shape_dict[f\"h.{i}.mlp.c_proj.bias\"] = [ndim]\n", + "\n", + " return tg.model.Model(shape_dict)\n", + "\n", + "model = gpt2_new(n_vocab=n_vocab, n_layers=n_layers, n_heads=n_heads, ndim=ndim)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "t = Tensor(123, requires_grad=False)\n", + "t1 = t + t\n", + "\n", + "t1.requires_grad is False\n", + "t1.parents is []\n", + "\n", + "\n", + "t1.requires_grad(True)\n", + "\n", + "t1.requires_grad is True\n", + "\n", + "But it has no parents!!!1\n", + "\n", + "t1.op should be Load, not Add\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def gpt2_init(model):\n", + " for k in model.params.keys():\n", + " if k.endswith(\".weight\"):\n", + " model.params[k] = Tensor(np.random.randn(*model.params[k].shape), name=k) * 0.02\n", + " elif k.endswith(\".bias\"):\n", + " model.params[k] = Tensor(np.zeros(model.params[k].shape), name=k)\n", + "\n", + " model.params[\"wte\"] = Tensor(np.random.randn(*model.params[\"wte\"].shape), name=\"wte\") * 0.02\n", + " model.params[\"wpe\"] = Tensor(np.random.randn(*model.params[\"wpe\"].shape), name=\"wpe\") * 0.01\n", + " \n", + "\n", + "gpt2_init(model)\n", + "model.requires_grad(True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tidygrad.func as F" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def gpt2_transformer_block(model: tg.model.Model, x, n_heads, i):\n", + " def get_params(s):\n", + " return model.params[f\"h.{i}.{s}\"]\n", + "\n", + " ln_1 = F.layer_norm(x, get_params(\"ln_1.weight\"), get_params(\"ln_1.bias\"))\n", + "\n", + " attn_w_qkv = get_params(\"attn.c_attn.weight\")\n", + " attn_b_qkv = get_params(\"attn.c_attn.bias\")\n", + "\n", + " attn_w_q, attn_w_k, attn_w_v = attn_w_qkv.split(3, axis=-1)\n", + " attn_b_q, attn_b_k, attn_b_v = attn_b_qkv.split(3, axis=-1)\n", + "\n", + " q = ln_1.mmul(attn_w_q) + attn_b_q\n", + " k = ln_1.mmul(attn_w_k) + attn_b_k\n", + " v = ln_1.mmul(attn_w_v) + attn_b_v\n", + "\n", + "\n", + "\n", + " q_chunked = F.stack(q.split(n=n_heads, axis=-1), axis=0)\n", + " k_chunked = F.stack(k.split(n=n_heads, axis=-1), axis=0)\n", + " v_chunked = F.stack(v.split(n=n_heads, axis=-1), axis=0)\n", + "\n", + " dim = q_chunked.shape[-1]\n", + " attention = q_chunked.mmul(k_chunked.transpose(-1, -2)) / np.sqrt(dim / n_heads)\n", + "\n", + " mask = np.tril(np.ones(attention.shape), k=0)\n", + " ee = np.exp(attention) * mask\n", + "\n", + " softmaxed = ee / ee.sum(axis=-1, keepdims=True)\n", + "\n", + " attention_output = softmaxed.mmul(v_chunked)\n", + " attention_chunks = attention_output.split(axis=0, n=n_heads)\n", + " # print(\"attention_chunks\", attention_chunks)\n", + "\n", + " attention_reshaped = F.concat(attention_chunks, axis=-1)\n", + " attention_reshaped = attention_reshaped[0]\n", + " # print(\"attention_reshaped\", attention_reshaped)\n", + "\n", + " cproj_w = get_params(\"attn.c_proj.weight\")\n", + " cproj_b = get_params(\"attn.c_proj.bias\")\n", + " # attention_reshaped = Tensor(attention_reshaped_np)\n", + "\n", + " crosstalk = attention_reshaped.mmul(cproj_w) + cproj_b\n", + "\n", + " after_residual = crosstalk + x\n", + " # print(\"after_residual\", after_residual)\n", + " ln2_w = get_params(\"ln_2.weight\")\n", + " ln2_b = get_params(\"ln_2.bias\")\n", + "\n", + " after_ln2 = F.layer_norm(after_residual, ln2_w, ln2_b)\n", + "\n", + " mlp_c_fc_w = get_params(\"mlp.c_fc.weight\")\n", + " mlp_c_fc_b = get_params(\"mlp.c_fc.bias\")\n", + "\n", + " after_up = after_ln2.mmul(mlp_c_fc_w) + mlp_c_fc_b\n", + " # print(\"after_up\", after_up)\n", + "\n", + " after_up_a = F.gelu(after_up)\n", + " # print(\"after_up_a\", after_up_a)\n", + "\n", + " mlp_c_proj_w = get_params(\"mlp.c_proj.weight\")\n", + " mlp_c_proj_b = get_params(\"mlp.c_proj.bias\")\n", + "\n", + " after_down = after_up_a.mmul(mlp_c_proj_w) + mlp_c_proj_b\n", + "\n", + " output = after_down + after_residual\n", + " return output\n", + "\n", + "def gpt2(model, input, n_layers, n_heads):\n", + " def get_params(s):\n", + " return model.params[s]\n", + "\n", + " token_embeddings = F.embedding(get_params(\"wte\"), input)\n", + " position_embeddings = F.embedding(get_params(\"wpe\"), np.arange(input.shape[-1]))\n", + "\n", + " x = token_embeddings + position_embeddings\n", + "\n", + " # print(\"first embedding\", x)\n", + "\n", + " for i in range(n_layers):\n", + " print(\"layer\", i)\n", + " x = gpt2_transformer_block(model=model, x=x, n_heads=n_heads, i=i)\n", + "\n", + "\n", + " return F.layer_norm(x, w=get_params(\"ln_f.weight\"), b=get_params(\"ln_f.bias\"))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# res = gpt2(model, np.arange(256).reshape(2, -1), n_layers=n_layers, n_heads=n_heads)\n", + "# res.sum().backward()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# from tidygrad.training import one_hot_encode_batch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def one_hot_encode_batch(y, n_classes):\n", + " diag = np.eye(n_classes)\n", + " return Tensor(diag[y])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "layer 0\n", + "layer 1\n" + ] + } + ], + "source": [ + "def language_modeling_loss(model, input, target, n_layers, n_heads):\n", + " res = gpt2(model, input, n_layers, n_heads)\n", + " # print(\"res\", res)\n", + " # print(\"wte\", model.params[\"wte\"])\n", + " logits = res.mmul(model.params[\"wte\"].transpose(-1, -2), name=\"logits\")\n", + "\n", + " # print(\"logits\", logits)\n", + " loss = F.CrossEntropy_loss(logits, one_hot_encode_batch(target, n_classes=n_vocab))\n", + " return loss\n", + "\n", + "\n", + "loss = language_modeling_loss(\n", + " model,\n", + " input=np.random.randint(0, n_vocab, size=(2, ctx_len)),\n", + " target=np.random.randint(0, n_vocab, size=(2, ctx_len)),\n", + " n_layers=n_layers,\n", + " n_heads=n_heads\n", + ")\n", + "\n", + "# print(\"loss\", loss)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tensor[2, 128, 1](name=\"\" op=Div parents=[,]):\n", + " v=array[2, 128, 1] n=256 (2Kb) x∈[0.007, 0.007] μ=0.007 σ=9.689e-06\n", + " ∇=array[2, 128, 1] n=256 (2Kb) \u001b[38;2;127;127;127mall_zeros\u001b[0m\n" + ] + } + ], + "source": [ + "np.seterr(all=\"raise\")\n", + "l = loss.sum()\n", + "print(loss)\n", + "\n", + "l.backward()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/nbs/examples/gpt2_v2.ipynb b/nbs/examples/gpt2_v2.ipynb index b1bb365..2e05550 100644 --- a/nbs/examples/gpt2_v2.ipynb +++ b/nbs/examples/gpt2_v2.ipynb @@ -41,11 +41,10 @@ "metadata": {}, "outputs": [], "source": [ - "# Download the model weights if needed\n", - "# !wget -c https://huggingface.co/gpt2/resolve/main/model.safetensors -O gpt2.safetensors\n", - "# !wget -c https://huggingface.co/gpt2-medium/resolve/main/model.safetensors -O gpt2-medium.safetensors\n", - "# !wget -c https://huggingface.co/gpt2-large/resolve/main/model.safetensors -O gpt2-large.safetensors\n", - "# !wget -c https://huggingface.co/gpt2-xl/resolve/main/model.safetensors -O gpt2-xl.safetensors" + "# !wget -c https://huggingface.co/gpt2/resolve/main/model.safetensors -O ./downloaded_weights/gpt2.safetensors\n", + "# !wget -c https://huggingface.co/gpt2-medium/resolve/main/model.safetensors -O ./downloaded_weights/gpt2-medium.safetensors\n", + "# !wget -c https://huggingface.co/gpt2-large/resolve/main/model.safetensors -O ./downloaded_weights/gpt2-large.safetensors\n", + "# !wget -c https://huggingface.co/gpt2-xl/resolve/main/model.safetensors -O ./downloaded_weights/gpt2-xl.safetensors" ] }, { @@ -68,6 +67,7 @@ "}\n", "\n", "gpt2_variant = \"gpt2-xl\"\n", + "weights_dir = \"./downloaded_weights/\"\n", "\n", "text = \"In a hole in the ground there lived a\"\n", "tokenizer = GPT2Tokenizer.from_pretrained(gpt2_variant)\n", @@ -84,7 +84,38 @@ "metadata": {}, "outputs": [], "source": [ - "model = safe_open(gpt2_variants[gpt2_variant].weight_file, framework=\"np\")" + "model = safe_open(weights_dir + gpt2_variants[gpt2_variant].weight_file, framework=\"np\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Tensor[50257, 1600](name=\"\" op=Load):\n", + " v=array[50257, 1600] f32 n=80411200 (0.3Gb) x∈[-0.325, 0.385] μ=-0.000 σ=0.048\n", + " " + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Tensor(model.get_tensor(\"wte.weight\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import safetensors" ] }, { @@ -96,6 +127,96 @@ "import tidygrad.func as F" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "nn.Module capabilities:\n", + "\n", + "0. Abstract neural network \"modules\", like Linear of Conv2D.\n", + "\n", + "1. Assignment tracks parameters\n", + "\n", + "class MyModel(Module):\n", + " def __init__():\n", + " self.w1 = Tensort(...)\n", + " self.b2 = Tens....\n", + "\n", + " # w1, b1 are tracked as parameters\n", + "\n", + "Then you can call model.parameters() to get a list of parameters.\n", + "\n", + "\n", + "2. Save / load weights. Also, count weights.\n", + "\n", + "3. Fun forward/backward pass on the model.\n", + "\n", + "\n", + "#### Pytorch\n", + "\n", + "class nn.Linear():\n", + " ....\n", + "\n", + "class Model(nn.Module):\n", + " __init__:\n", + " self.l1 = nn.Linear(...)\n", + " self.ln = ...\n", + " \n", + " forward(x):\n", + " x = self.l1(x) \n", + " x = self.conv(x)\n", + " ....\n", + " return x\n", + "\n", + "model = Model(...)\n", + "\n", + "y = model(x)\n", + "\n", + "#### TidyGrad\n", + "\n", + "y = x.mmul(w) + b\n", + "\n", + "\n", + "\n", + "\n", + "class ModelTensors(Dict):\n", + " __init__\n", + "\n", + "\n", + " load(st: safetensor):\n", + " for k in st.keys():\n", + " self.params[k] = st.get_tensor(k)\n", + "\n", + " save():\n", + " .....\n", + " return st\n", + "\n", + "\n", + "model = ModelTensors\n", + " \n", + " \n", + "a = model[\"h0.ln1.w\"] # Returns Tensor\n", + "a = models.h0.ln1.w\n", + "\n", + "\n", + "model.parameters() ==> Return list of params\n", + "\n", + "\n", + "optim = SGD(model.params(), lr=9000)\n", + "\n", + "def transformer()...\n", + "\n", + "loss = transformer(X, y, model)\n", + "loss.backwards()\n", + "\n", + "optim.step()\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ] + }, { "cell_type": "code", "execution_count": null, @@ -184,23 +305,6 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tensor[10, 1600](\" op=Add):\n", - " v=array[10, 1600] f32 n=16000 (62Kb) x∈[-5.412, 10.720] μ=0.017 σ=1.065\n", - " \n" - ] - } - ], "source": [ "def transformer(model, tokens, n_layer, n_head):\n", " wte = Tensor(model.get_tensor(\"wte.weight\"))\n", @@ -214,6 +318,7 @@ " embeddings = token_embeddings + position_embeddings\n", "\n", " for i in range(n_layer):\n", + " # print(\"Layer\", i)\n", " embeddings = transformer_block(model, i, embeddings, n_head)\n", " # print(\"Embedding out:\", embeddings)\n", " # print(tidygrad.tensor._num_tensors)\n", @@ -226,20 +331,36 @@ "\n", " return res\n", "\n", - "tokens = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n", + "# tokens = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n", + "\n", + "# with tidygrad.no_grad():\n", + "# res = transformer(model, tokens, gpt2_variants[gpt2_variant].n_layer, gpt2_variants[gpt2_variant].n_head)\n", + "# print(res)\n", "\n", - "with tidygrad.no_grad():\n", - " res = transformer(model, tokens, gpt2_variants[gpt2_variant].n_layer, gpt2_variants[gpt2_variant].n_head)\n", - " print(res)" + "# import gc\n", + "# del res\n", + "\n", + "# gc.collect()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(50257, 1600)" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "wte = Tensor(model.get_tensor(\"wte.weight\").swapaxes(-1, -2))" + "model.get_tensor(\"wte.weight\").shape" ] }, { @@ -251,8 +372,22 @@ "name": "stdout", "output_type": "stream", "text": [ - "[818, 257, 7604, 287, 262, 2323, 612, 5615, 257]\n", - "Tensor[1600](\" op=Slice):\n", + "[818, 257, 7604, 287, 262, 2323, 612, 5615, 257]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/xl0/work/projects/grads/tidygrad/tidygrad/ops/activation.py:33: RuntimeWarning: overflow encountered in exp\n", + " self.set_out(1 / (1 + np.exp(-self.args[0].data)))\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tensor[1600](name=\"\" op=Slice):\n", " v=array[1600] f32 6.2Kb x∈[-5.825, 4.088] μ=0.007 σ=1.243\n", " \n" ] @@ -274,9 +409,10 @@ "\n", "tokens = tokenizer.encode(text) # returns a list of integers\n", "print(tokens)\n", - "# tokens = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n", + "# tokens = list(range(1000))\n", "\n", - "def gpt2_language_model(model, token_ids, wte, n_layer, n_head):\n", + "def gpt2_language_model(model, token_ids, n_layer, n_head):\n", + " wte = Tensor(model.get_tensor(\"wte.weight\").swapaxes(-1, -2))\n", " res = transformer(model, token_ids, n_layer, n_head)\n", "\n", " res = res[-1, :]\n", @@ -284,9 +420,52 @@ " return logits, res\n", "\n", "with tidygrad.no_grad():\n", - " logits, res = gpt2_language_model(model, tokens, wte, n_layer=gpt2_variants[gpt2_variant].n_layer, n_head=gpt2_variants[gpt2_variant].n_head)\n", + " logits, res = gpt2_language_model(model, tokens, n_layer=gpt2_variants[gpt2_variant].n_layer, n_head=gpt2_variants[gpt2_variant].n_head)\n", " print(res)\n", - "tokenizer.decode(logits.data.argmax(axis=-1))" + "tokenizer.decode(logits.data.argmax(axis=-1))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dtype('float32')" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "res.data.dtype" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'gc' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/examples/gpt2_v2.ipynb Cell 16\u001b[0m line \u001b[0;36m2\n\u001b[1;32m 1\u001b[0m \u001b[39mdel\u001b[39;00m logits, res\n\u001b[0;32m----> 2\u001b[0m gc\u001b[39m.\u001b[39mcollect()\n", + "\u001b[0;31mNameError\u001b[0m: name 'gc' is not defined" + ] + } + ], + "source": [ + "\n", + "del logits, res\n", + "gc.collect()" ] }, { @@ -298,6 +477,58 @@ "from tqdm.auto import tqdm" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dtype('float32')" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Tensor(np.random.randn(5,5)).data.dtype" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dtype('float32')" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = np.random.randn(5, 5).astype(np.float32)\n", + "b = np.random.randn(5, 5).astype(np.float32)\n", + "\n", + "(a+b).dtype" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "a = np.zeros((1000_000, 1000))" + ] + }, { "cell_type": "code", "execution_count": null, @@ -314,12 +545,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f49317d500b64c9a9a86bb75c17174c6", + "model_id": "936355aa1482433f88b369374375e9f2", "version_major": 2, "version_minor": 0 }, "text/plain": [ - " 0%| | 0/10 [00:007\u001b[0m \u001b[39mwith\u001b[39;00m tidygrad\u001b[39m.\u001b[39mno_grad():\n\u001b[1;32m 8\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m tqdm(\u001b[39mrange\u001b[39m(\u001b[39m100\u001b[39m)):\n\u001b[0;32m----> 9\u001b[0m logits, res \u001b[39m=\u001b[39m gpt2_language_model(model, tokens, n_layer\u001b[39m=\u001b[39;49mgpt2_variants[gpt2_variant]\u001b[39m.\u001b[39;49mn_layer, n_head\u001b[39m=\u001b[39;49mgpt2_variants[gpt2_variant]\u001b[39m.\u001b[39;49mn_head)\n\u001b[1;32m 10\u001b[0m tokens\u001b[39m.\u001b[39mappend(logits\u001b[39m.\u001b[39mdata\u001b[39m.\u001b[39margmax(axis\u001b[39m=\u001b[39m\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m))\n\u001b[1;32m 11\u001b[0m \u001b[39mdel\u001b[39;00m logits, res\n", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/examples/gpt2_v2.ipynb Cell 20\u001b[0m line \u001b[0;36m1\n\u001b[1;32m 8\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mgpt2_language_model\u001b[39m(model, token_ids, n_layer, n_head):\n\u001b[1;32m 9\u001b[0m wte \u001b[39m=\u001b[39m Tensor(model\u001b[39m.\u001b[39mget_tensor(\u001b[39m\"\u001b[39m\u001b[39mwte.weight\u001b[39m\u001b[39m\"\u001b[39m)\u001b[39m.\u001b[39mswapaxes(\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m, \u001b[39m-\u001b[39m\u001b[39m2\u001b[39m))\n\u001b[0;32m---> 10\u001b[0m res \u001b[39m=\u001b[39m transformer(model, token_ids, n_layer, n_head)\n\u001b[1;32m 12\u001b[0m res \u001b[39m=\u001b[39m res[\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m, :]\n\u001b[1;32m 13\u001b[0m logits \u001b[39m=\u001b[39m res\u001b[39m.\u001b[39mmmul(wte)\n", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/examples/gpt2_v2.ipynb Cell 20\u001b[0m line \u001b[0;36m1\n\u001b[1;32m 10\u001b[0m embeddings \u001b[39m=\u001b[39m token_embeddings \u001b[39m+\u001b[39m position_embeddings\n\u001b[1;32m 12\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(n_layer):\n\u001b[0;32m---> 13\u001b[0m embeddings \u001b[39m=\u001b[39m transformer_block(model, i, embeddings, n_head)\n\u001b[1;32m 14\u001b[0m \u001b[39m# print(\"Embedding out:\", embeddings)\u001b[39;00m\n\u001b[1;32m 15\u001b[0m \u001b[39m# print(tidygrad.tensor._num_tensors)\u001b[39;00m\n\u001b[1;32m 16\u001b[0m \u001b[39m# print(tidygrad.tensor._num_ops)\u001b[39;00m\n\u001b[1;32m 18\u001b[0m ln_f_w \u001b[39m=\u001b[39m Tensor(model\u001b[39m.\u001b[39mget_tensor(\u001b[39m\"\u001b[39m\u001b[39mln_f.weight\u001b[39m\u001b[39m\"\u001b[39m))\n", + "\u001b[1;32m/home/xl0/work/projects/grads/tidygrad/nbs/examples/gpt2_v2.ipynb Cell 20\u001b[0m line \u001b[0;36m1\n\u001b[1;32m 8\u001b[0m ln_1 \u001b[39m=\u001b[39m F\u001b[39m.\u001b[39mlayer_norm(\u001b[39minput\u001b[39m, ln_1_w, ln_1_b)\n\u001b[1;32m 9\u001b[0m \u001b[39m# ln_1.ad\u001b[39;00m\n\u001b[0;32m---> 11\u001b[0m attn_w_qkv \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39;49mget_tensor(\u001b[39mf\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mh.\u001b[39;49m\u001b[39m{\u001b[39;49;00mi\u001b[39m}\u001b[39;49;00m\u001b[39m.attn.c_attn.weight\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n\u001b[1;32m 12\u001b[0m attn_b_qkv \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39mget_tensor(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mh.\u001b[39m\u001b[39m{\u001b[39;00mi\u001b[39m}\u001b[39;00m\u001b[39m.attn.c_attn.bias\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 14\u001b[0m attn_w_q, attn_w_k, attn_w_v \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39msplit(attn_w_qkv, \u001b[39m3\u001b[39m, axis\u001b[39m=\u001b[39m\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m)\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], @@ -361,12 +938,13 @@ "\n", "print(\"=== Generating ===\")\n", "print(\"Input: \", tokenizer.decode(tokens))\n", - "wte = Tensor(model.get_tensor(\"wte.weight\").swapaxes(-1, -2))\n", "\n", "with tidygrad.no_grad():\n", - " for i in tqdm(range(10)):\n", - " logits, res = gpt2_language_model(model, tokens, wte, n_layer=gpt2_variants[gpt2_variant].n_layer, n_head=gpt2_variants[gpt2_variant].n_head)\n", + " for i in tqdm(range(100)):\n", + " logits, res = gpt2_language_model(model, tokens, n_layer=gpt2_variants[gpt2_variant].n_layer, n_head=gpt2_variants[gpt2_variant].n_head)\n", " tokens.append(logits.data.argmax(axis=-1))\n", + " del logits, res\n", + " # gc.collect()\n", " print(\"Output:\", tokenizer.decode(tokens))" ] } diff --git a/nbs/tests/01_test_ops.ipynb b/nbs/tests/01_test_ops.ipynb index a48c04b..9b52bf1 100644 --- a/nbs/tests/01_test_ops.ipynb +++ b/nbs/tests/01_test_ops.ipynb @@ -29,7 +29,7 @@ "\n", " t = func(inputs=None, params=(a, b))\n", " t.backward()\n", - " grad_check(func=func, inputs=None, params=(a, b))" + " grad_check(func=func, inputs=None, params=(a, b), verbose=False)" ] }, { @@ -61,6 +61,22 @@ "### Binary elementwise ops\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "a = Tensor(np.random.randn(2, 3), name=\"a\", requires_grad=True)\n", + "b = Tensor(np.random.randn(2, 3), name=\"b\", requires_grad=True)\n", + "\n", + "c = a + b\n", + "\n", + "loss = c.sum()\n", + "\n", + "loss.backward()\n" + ] + }, { "cell_type": "code", "execution_count": null, @@ -78,10 +94,10 @@ "source": [ "def add_func(inputs, params: tuple = ()):\n", " a, b = params\n", - " loss = a.add(b, \"t\").sum(\"loss\")\n", + " loss = a.add(b, \"t\").sum()\n", " return loss\n", "\n", - "run_test_binary_elementwise(add_func, (100, 100))" + "run_test_binary_elementwise(add_func, (1, 1))" ] }, { @@ -117,7 +133,7 @@ "output_type": "stream", "text": [ "Max fractional gradient difference for b: 0.0001%\n", - "Max fractional gradient difference for a: 0.0000%\n" + "Max fractional gradient difference for a: 0.0002%\n" ] } ], @@ -174,7 +190,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Max fractional gradient difference for a: 0.4744%\n" + "Max fractional gradient difference for a: 0.1248%\n" ] } ], @@ -197,7 +213,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Max fractional gradient difference for a: 0.0006%\n" + "Max fractional gradient difference for a: 0.0028%\n" ] } ], @@ -231,7 +247,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Max fractional gradient difference for a: 0.0006%\n" + "Max fractional gradient difference for a: 0.0005%\n" ] } ], @@ -315,7 +331,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Max fractional gradient difference for a: 0.0006%\n" + "Max fractional gradient difference for a: 0.0007%\n" ] } ], @@ -402,7 +418,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Max fractional gradient difference for b: 0.0001%\n", + "Max fractional gradient difference for b: 0.0000%\n", "Max fractional gradient difference for a: 0.0000%\n" ] } @@ -457,7 +473,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Max fractional gradient difference for x: 0.0028%\n" + "Max fractional gradient difference for x: 0.0029%\n" ] } ], @@ -651,9 +667,9 @@ { "data": { "text/plain": [ - "Tensor[100, 100](\" op=Pow parents=[a]):\n", + "Tensor[100, 100](name=\"\" op=Pow parents=[a]):\n", " v=array[100, 100] n=10000 (78Kb) x∈[-41.412, 47.474] μ=0.066 σ=3.739\n", - " ∇=array[100, 100] f32 n=10000 (39Kb) \u001b[38;2;127;127;127mall_zeros\u001b[0m" + " ∇=array[100, 100] n=10000 (78Kb) \u001b[38;2;127;127;127mall_zeros\u001b[0m" ] }, "execution_count": null, @@ -721,6 +737,46 @@ "\n", "run_test_binary_elementwise(concat_test, (100, 100), (100, 100))" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tidygrad.func import layer_norm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Max fractional gradient difference for b: 0.0000%\n", + "Max fractional gradient difference for w: 0.0000%\n", + "Max fractional gradient difference for a: 0.0074%\n" + ] + } + ], + "source": [ + "def layer_norm_test(inputs, params):\n", + " a, w, b = params\n", + " t = layer_norm(a, w, b)\n", + " return t.sum(\"loss\")\n", + "\n", + "a = Tensor(np.random.randn(2, 100, 100), name=\"a\", requires_grad=True)\n", + "w = Tensor(np.random.randn(100), name=\"w\", requires_grad=True)\n", + "b = Tensor(np.random.randn(100), name=\"b\", requires_grad=True)\n", + "\n", + "t = layer_norm_test(inputs=None, params=(a, w, b))\n", + "t.backward()\n", + "\n", + "grad_check(func=layer_norm_test, inputs=None, params=(a, w, b))" + ] } ], "metadata": { diff --git a/tidygrad/__init__.py b/tidygrad/__init__.py index 20d0587..0656b40 100644 --- a/tidygrad/__init__.py +++ b/tidygrad/__init__.py @@ -1,11 +1,13 @@ __version__ = "0.0.1" import numpy as np -np.seterr(under="ignore") +np.seterr(under="raise") del np from .utils import datasets, data from .tensor import Tensor, no_grad from .func import * +from . import model + # __all__ = [datasets, data, no_grad, Tensor] diff --git a/tidygrad/_modidx.py b/tidygrad/_modidx.py index b228511..4e82d73 100644 --- a/tidygrad/_modidx.py +++ b/tidygrad/_modidx.py @@ -32,6 +32,12 @@ 'tidygrad.func.sum': ('func.html#sum', 'tidygrad/func.py'), 'tidygrad.func.tanh': ('func.html#tanh', 'tidygrad/func.py'), 'tidygrad.func.transpose': ('func.html#transpose', 'tidygrad/func.py')}, + 'tidygrad.model': { 'tidygrad.model.Model': ('model.html#model', 'tidygrad/model.py'), + 'tidygrad.model.Model.__init__': ('model.html#model.__init__', 'tidygrad/model.py'), + 'tidygrad.model.Model.__repr__': ('model.html#model.__repr__', 'tidygrad/model.py'), + 'tidygrad.model.Model.parameter_list': ('model.html#model.parameter_list', 'tidygrad/model.py'), + 'tidygrad.model.Model.requires_grad': ('model.html#model.requires_grad', 'tidygrad/model.py'), + 'tidygrad.model.Model.save': ('model.html#model.save', 'tidygrad/model.py')}, 'tidygrad.ops.activation': { 'tidygrad.ops.activation.Relu': ('ops.activation.html#relu', 'tidygrad/ops/activation.py'), 'tidygrad.ops.activation.Relu.__init__': ( 'ops.activation.html#relu.__init__', 'tidygrad/ops/activation.py'), diff --git a/tidygrad/func.py b/tidygrad/func.py index ce5f8f3..1ac1606 100644 --- a/tidygrad/func.py +++ b/tidygrad/func.py @@ -123,6 +123,13 @@ def softmax(input: Tensor, name=None) -> Tensor: def layer_norm(x: Tensor, w: Tensor, b: Tensor, eps=1e-5) -> Tensor: mu = x.mean(axis=-1, keepdims=True) sigma = x.std(axis=-1, keepdims=True, correction=0) + if sigma.data.any() == 0: + print("x", x) + print("w", w) + print("b", b) + print("mu", mu) + print("sigma", sigma) + raise ValueError("sigma is zero") return ( (x - mu) / (sigma + eps) @@ -152,7 +159,7 @@ def CrossEntropy_loss(logits: Tensor, target: Tensor, reduction="mean"): sm = softmax(logits) loss = -target * sm.log() if reduction == "mean": - return loss.mean() + return loss.mean(axis=-1, keepdims=True) if reduction == "sum": - return loss.sum() + return loss.sum(axis=-1, keepdims=True) assert 0, "Invalid reduction" diff --git a/tidygrad/model.py b/tidygrad/model.py new file mode 100644 index 0000000..d67ae1c --- /dev/null +++ b/tidygrad/model.py @@ -0,0 +1,47 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/04_model.ipynb. + +# %% auto 0 +__all__ = ['Model'] + +# %% ../nbs/04_model.ipynb 1 +import os + +from lovely_numpy import Lo + +import numpy as np +from tidygrad.tensor import Tensor +import safetensors +import safetensors.numpy + +# %% ../nbs/04_model.ipynb 2 +class Model: + def __init__(self, params: dict[str, tuple] | str | os.PathLike): + self.params = {} + + if isinstance(params, dict): + for name, shape in params.items(): + self.params[name] = Tensor(np.zeros(shape)) + + elif isinstance(params, (str, os.PathLike)): + model = safetensors.safe_open(params, framework="numpy") + for name in model.keys(): + self.params[name] = Tensor(model.get_tensor(name), name=name) + + else: + raise TypeError("params must be a dict or a path") + + def __repr__(self): + return f"Model with params:\n" + "\n".join( + [f"\t{name}: {param.shape}" for name, param in self.params.items()] + ) + + def save(self, filename: str): + d = {key: self.params[key].data for key in self.params.keys()} + safetensors.numpy.save_file(d, filename) + + def requires_grad(self, value): + for name, param in self.params.items(): + param.requires_grad = value + + def parameter_list(self): + return list(self.params.values()) diff --git a/tidygrad/ops/__init__.py b/tidygrad/ops/__init__.py index 9a03388..565e71a 100644 --- a/tidygrad/ops/__init__.py +++ b/tidygrad/ops/__init__.py @@ -3,3 +3,5 @@ from .activation import * from .conv import * from .loss import * + + diff --git a/tidygrad/ops/common.py b/tidygrad/ops/common.py index c230e18..4d2b8c0 100644 --- a/tidygrad/ops/common.py +++ b/tidygrad/ops/common.py @@ -63,7 +63,7 @@ def maybe_broadcast_matmul(a, b): return a, b -# %% ../../nbs/02_ops.common.ipynb 6 +# %% ../../nbs/02_ops.common.ipynb 7 _num_ops = 0 @@ -96,8 +96,9 @@ def __init__(self, *args, name: str = None): def set_out(self, data): from tidygrad.tensor import Tensor + op = self if self.requires_grad else None self.out = Tensor( - data=data, requires_grad=self.requires_grad, name=self.name, op=self + data=data, requires_grad=self.requires_grad, name=self.name, op=op ) def check_backward(self): @@ -130,7 +131,7 @@ def __init__(self, a, name=None): if self.requires_grad: self.parents = self.args -# %% ../../nbs/02_ops.common.ipynb 7 +# %% ../../nbs/02_ops.common.ipynb 8 class Load(BaseOp): """Load a tensor""" @@ -139,7 +140,7 @@ class Load(BaseOp): def __init__(self, name=None): super().__init__(name=name) -# %% ../../nbs/02_ops.common.ipynb 8 +# %% ../../nbs/02_ops.common.ipynb 9 class Add(BinaryElementwiseOp): """Add two tensors""" @@ -157,7 +158,7 @@ def backward(self): self.parents[0].accum_grad(self.out.grad) self.parents[1].accum_grad(self.out.grad) -# %% ../../nbs/02_ops.common.ipynb 9 +# %% ../../nbs/02_ops.common.ipynb 10 class Sub(BinaryElementwiseOp): """Subtract two tensors""" @@ -172,7 +173,7 @@ def backward(self): self.parents[0].accum_grad(self.out.grad) self.parents[1].accum_grad(-self.out.grad) -# %% ../../nbs/02_ops.common.ipynb 10 +# %% ../../nbs/02_ops.common.ipynb 11 class Mul(BinaryElementwiseOp): """Multiply two tensors""" @@ -184,10 +185,11 @@ def __init__(self, a, b, name=None): def backward(self): self.check_backward() + self.parents[0].accum_grad(self.out.grad * self.parents[1].data) self.parents[1].accum_grad(self.out.grad * self.parents[0].data) -# %% ../../nbs/02_ops.common.ipynb 11 +# %% ../../nbs/02_ops.common.ipynb 12 class Div(BinaryElementwiseOp): """Divide two tensors""" @@ -204,7 +206,7 @@ def backward(self): -self.out.grad * self.parents[0].data / (self.parents[1].data ** 2) ) -# %% ../../nbs/02_ops.common.ipynb 12 +# %% ../../nbs/02_ops.common.ipynb 13 class Neg(UnaryElementwiseOp): """Negate a tensor""" @@ -218,7 +220,7 @@ def backward(self): self.check_backward() self.parents[0].accum_grad(-self.out.grad) -# %% ../../nbs/02_ops.common.ipynb 13 +# %% ../../nbs/02_ops.common.ipynb 14 class Pow(UnaryElementwiseOp): """Raise a tensor to a power""" @@ -230,11 +232,12 @@ def __init__(self, a, power, name=None): def backward(self): self.check_backward() - self.parents[0].accum_grad( - (self.out.grad * self.power * self.parents[0].data ** (self.power - 1)) - ) + with np.errstate(divide="ignore"): + self.parents[0].accum_grad( + (self.out.grad * self.power * self.parents[0].data ** (self.power - 1)) + ) -# %% ../../nbs/02_ops.common.ipynb 14 +# %% ../../nbs/02_ops.common.ipynb 15 class Log(UnaryElementwiseOp): """Take the natural logarithm of a tensor""" @@ -248,7 +251,7 @@ def backward(self): self.check_backward() self.parents[0].accum_grad(self.out.grad / self.parents[0].data) -# %% ../../nbs/02_ops.common.ipynb 15 +# %% ../../nbs/02_ops.common.ipynb 16 class Exp(UnaryElementwiseOp): """Exponentiate a tensor""" @@ -262,7 +265,7 @@ def backward(self): self.check_backward() self.parents[0].accum_grad(self.out.grad * self.out.data) -# %% ../../nbs/02_ops.common.ipynb 16 +# %% ../../nbs/02_ops.common.ipynb 17 class ExpLog(UnaryElementwiseOp): """Exponentiate a tensor""" @@ -282,7 +285,7 @@ def backward(self): self.out.grad * (1 - 1 / (1 + np.exp(self.parents[0].data))) ) -# %% ../../nbs/02_ops.common.ipynb 17 +# %% ../../nbs/02_ops.common.ipynb 18 class Matmul(BaseOp): """Matrix multiplication of two tensors""" @@ -305,13 +308,19 @@ def backward(self): np.matmul(self.parents[0].data.swapaxes(-1, -2), self.out.grad) ) -# %% ../../nbs/02_ops.common.ipynb 18 +# %% ../../nbs/02_ops.common.ipynb 19 class Sum(BaseOp): """Sum-reduce a tensor along the given axis (int or tuple of ints)""" name_template = "sum({})" - def __init__(self, a, name=None, axis=None, keepdims=False): + def __init__( + self, + a, + axis=None, + keepdims=False, + name=None, + ): super().__init__(a, name=name) self.parents = self.args if self.requires_grad else [] self.set_out(np.sum(self.args[0].data, axis=axis, keepdims=keepdims)) @@ -320,7 +329,7 @@ def backward(self): self.check_backward() self.parents[0].accum_grad(self.out.grad) # This will broadcast correctly -# %% ../../nbs/02_ops.common.ipynb 19 +# %% ../../nbs/02_ops.common.ipynb 20 class Broadcast(BaseOp): """Broadcast a tensor to the given shape""" @@ -371,7 +380,7 @@ def backward(self): self.parents[0].accum_grad(summed) -# %% ../../nbs/02_ops.common.ipynb 20 +# %% ../../nbs/02_ops.common.ipynb 21 class Slice(UnaryElementwiseOp): name_template = "slice({})" @@ -392,7 +401,7 @@ def backward(self): p.grad[self.key] += self.out.grad -# %% ../../nbs/02_ops.common.ipynb 22 +# %% ../../nbs/02_ops.common.ipynb 23 class Transpose(UnaryElementwiseOp): """Transpose a tensor""" @@ -400,12 +409,15 @@ class Transpose(UnaryElementwiseOp): def __init__(self, a, dim0, dim1, name=None): super().__init__(a, name=name) + self.dim0 = dim0 + self.dim1 = dim1 self.set_out(np.swapaxes(self.args[0].data, dim0, dim1)) def backward(self): - pass + self.check_backward() + self.parents[0].accum_grad(np.swapaxes(self.out.grad, self.dim0, self.dim1)) -# %% ../../nbs/02_ops.common.ipynb 23 +# %% ../../nbs/02_ops.common.ipynb 24 class Dropout(UnaryElementwiseOp): """Apply Dropout to a tensor""" @@ -431,9 +443,9 @@ def __init__(self, a, p_drop=0.1, training=True, name=None): def backward(self): self.check_backward() - self.parents[0].grad += self.out.grad * (self.mask if self.training else 1) + self.parents[0].accum_grad(self.out.grad * (self.mask if self.training else 1)) -# %% ../../nbs/02_ops.common.ipynb 24 +# %% ../../nbs/02_ops.common.ipynb 25 class Embedding(UnaryElementwiseOp): """Embedding layer""" @@ -446,4 +458,6 @@ def __init__(self, a, indices, name=None): def backward(self): self.check_backward() + if self.parents[0].grad is None: + self.parents[0].grad = np.zeros_like(self.parents[0].data, dtype=np.float32) self.parents[0].grad[self.indices] += self.out.grad diff --git a/tidygrad/tensor.py b/tidygrad/tensor.py index 647a261..3bd7c3a 100644 --- a/tidygrad/tensor.py +++ b/tidygrad/tensor.py @@ -35,10 +35,10 @@ class Tensor: def __init__(self, data, name=None, op=None, eps=1e-8, requires_grad=False): global _num_tensors _num_tensors += 1 - self.data = np.asarray(data) + self.data = np.asarray(data, dtype=np.float64) # , dtype=np.float32 self.grad = ( - np.zeros_like(self.data, dtype=np.float32) if requires_grad else None + np.zeros_like(self.data, dtype=np.float64) if requires_grad else None ) self.eps = eps self.op = op or ops.Load(name=name) @@ -53,8 +53,8 @@ def __repr__(self): if self.op.parents else "" ) - # name="{self.name} - return f'Tensor{list(self.data.shape)}(" op={type(self.op).__name__}{parents}):\n {value_str}\n {grad_str}' + + return f'Tensor{list(self.data.shape)}(name="{self.name}" op={type(self.op).__name__}{parents}):\n {value_str}\n {grad_str}' def accum_grad(self, grad): if not self.requires_grad: @@ -95,8 +95,19 @@ def exp(self, name=None): def mmul(self, other, name=None): return ops.Matmul(self, other, name=name).out - def sum(self, name=None, axis=None, keepdims=False): - return ops.Sum(self, name=name, axis=axis, keepdims=keepdims).out + # XXX move name to the end of arg list + def sum( + self, + name=None, + axis=None, + keepdims=False, + ): + return ops.Sum( + self, + axis=axis, + keepdims=keepdims, + name=name, + ).out def transpose( self, diff --git a/tidygrad/tensor_helpers.py b/tidygrad/tensor_helpers.py index acaf26f..32f30da 100644 --- a/tidygrad/tensor_helpers.py +++ b/tidygrad/tensor_helpers.py @@ -25,7 +25,7 @@ def std(input: Tensor, name=None, axis=None, keepdims=False, correction=1) -> Te if isinstance(axis, int): axis = (axis,) v1 = input - input.mean(axis=axis, keepdims=True) - var = (v1) ** 2 + var = v1**2 if axis is None: numel = np.prod(input.data.shape) diff --git a/tidygrad/training.py b/tidygrad/training.py index 38df6af..ec915dd 100644 --- a/tidygrad/training.py +++ b/tidygrad/training.py @@ -156,14 +156,17 @@ def do_batch_backward(self): # %% ../nbs/06_training.ipynb 9 def one_hot_encode_batch(y, n_classes): - batch_size = len(y) + diag = np.eye(n_classes) + return Tensor(diag[y]) + + batch_size = y.shape[0] assert batch_size > 0 assert n_classes > 0 - assert y.shape == (batch_size,) + # assert y.shape[0] == batch_size assert np.min(y) >= 0 # Initialize a zero matrix of shape (batch_size, num_classes) - one_hot_matrix = np.zeros((batch_size, n_classes)) + one_hot_matrix = np.zeros((*y.shape, n_classes)) # Fill in the appropriate elements one_hot_matrix[np.arange(batch_size), y] = 1 diff --git a/tidygrad/utils/grad_check.py b/tidygrad/utils/grad_check.py index c58cb00..210b084 100644 --- a/tidygrad/utils/grad_check.py +++ b/tidygrad/utils/grad_check.py @@ -23,7 +23,8 @@ def grad_check(func, inputs, params: tuple = (), eps=1e-5, n=1000, verbose=False grad_view = p.grad.reshape(-1) slow_grad = np.zeros_like(p.grad) - slow_grad_view = slow_grad.reshape(-1) + + scaled_slow_grad_view = slow_grad.reshape(-1) indices = np.random.choice( np.arange(grad_view.size), size=min(n, grad_view.size), replace=False @@ -43,19 +44,26 @@ def grad_check(func, inputs, params: tuple = (), eps=1e-5, n=1000, verbose=False data_view[idx] = old_val + eps loss_plus_h = func(inputs, params) - slow_grad_view[idx] = (loss_plus_h.data - loss.data) / eps + scaled_slow_grad_view[idx] = (loss_plus_h.data - loss.data) / eps + # slow_grad_view[idx] = + + # (loss_plus_h.data - loss.data) / eps + if verbose: print( - f"{idx}: loss_plus_h: {loss_plus_h.data}, loss: {loss.data}, diff: {loss_plus_h.data - loss.data}, grad: {grad_view[idx]}, slow_grad: {slow_grad_view[idx]}" + f"{idx}: loss_plus_h: {loss_plus_h.data}, loss: {loss.data}, diff: {loss_plus_h.data - loss.data}, grad: {grad_view[idx]}, slow_grad: {scaled_slow_grad_view[idx] / eps}" ) data_view[idx] = old_val - if abs(slow_grad_view[idx]) > eps: + if abs(scaled_slow_grad_view[idx]) > eps: good_indices.append(idx) differences = ( - slow_grad_view[good_indices] - grad_view[good_indices] - ) / slow_grad_view[good_indices] + scaled_slow_grad_view[good_indices] - grad_view[good_indices] + ) / (grad_view[good_indices]) + + # slow_grad /= eps + max_grad_diff = np.max(np.abs(differences)) print( f"Max fractional gradient difference for {p.name}: {max_grad_diff*100:.4f}%"