main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# BART Arabic From Scratch\n",
    "\n",
    "Implement the [BART: Denoising Sequence-to-Sequence Pre-training](https://arxiv.org/abs/1910.13461) paper from scratch using PyTorch for an abstractive summarization task in Arabic."
   ],
   "metadata": {
    "id": "shpDRLJtBmM6"
   },
   "id": "shpDRLJtBmM6"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Install & Import Dependencies\n",
    "Dependency version details are listed in the `requirements.txt` file."
   ],
   "metadata": {
    "id": "aKBt948JB4lS"
   },
   "id": "aKBt948JB4lS"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d43f8305-71ba-427a-88a8-19be792d607e",
   "metadata": {
    "id": "d43f8305-71ba-427a-88a8-19be792d607e"
   },
   "outputs": [],
   "source": [
    "!pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac7806a8-c541-45cc-866d-5a78787c6a82",
   "metadata": {
    "id": "ac7806a8-c541-45cc-866d-5a78787c6a82"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import math\n",
    "import time\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "from torch import nn, Tensor\n",
    "from torch.optim import Adam\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "from torch.cuda.amp import GradScaler, autocast\n",
    "from datasets import Dataset\n",
    "from tokenizers import ByteLevelBPETokenizer, Tokenizer\n",
    "from tokenizers.processors import TemplateProcessing"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Download & Load the Datasets\n",
    "The dataset used is the [XL-Sum(Arabic Subset)](https://github.com/csebuetnlp/xl-sum?tab=readme-ov-file#:~:text=Arabic,Download) dataset. I chose this dataset because it's well-suited for our task. Additionally, it's written in pure Arabic, which makes it the best choice. The original source: [BBC Arabic](https://www.bbc.com/arabic).\n"
   ],
   "metadata": {
    "id": "lg8fCKq0CEjN"
   },
   "id": "lg8fCKq0CEjN"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c204f66-d2a9-4bdb-b09a-83258eca701c",
   "metadata": {
    "id": "3c204f66-d2a9-4bdb-b09a-83258eca701c",
    "outputId": "61a2565d-7415-4a30-91b2-96ebe2522a96"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading...\n",
      "From (original): https://docs.google.com/uc?export=download&id=1lot6kJ6TPCHuBI6Ky_RQPdYpWaQCk-jl\n",
      "From (redirected): https://docs.google.com/uc?export=download&id=1lot6kJ6TPCHuBI6Ky_RQPdYpWaQCk-jl&confirm=t&uuid=881a1b05-ccb8-43b9-9a16-0da145ee300d\n",
      "To: /teamspace/studios/this_studio/arabic_XLSum_v2.0.tar.bz2\n",
      "100%|██████████████████████████████████████| 44.9M/44.9M [00:02<00:00, 20.4MB/s]\n",
      "./\n",
      "./arabic_test.jsonl\n",
      "./arabic_val.jsonl\n",
      "./arabic_train.jsonl\n",
      "arabic_test.jsonl  arabic_train.jsonl  arabic_val.jsonl\n"
     ]
    }
   ],
   "source": [
    "!gdown \"https://docs.google.com/uc?export=download&id=1lot6kJ6TPCHuBI6Ky_RQPdYpWaQCk-jl\"\n",
    "!mkdir -p data\n",
    "!tar -xvjf arabic_XLSum_v2.0.tar.bz2 -C data/\n",
    "!ls data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "111347e7-1ff7-4cb4-8e11-9a080312fa04",
   "metadata": {
    "id": "111347e7-1ff7-4cb4-8e11-9a080312fa04"
   },
   "outputs": [],
   "source": [
    "data = \"data/\"\n",
    "\n",
    "train_file = os.path.join(data, \"arabic_train.jsonl\")\n",
    "validation_file = os.path.join(data, \"arabic_val.jsonl\")\n",
    "test_file = os.path.join(data, \"arabic_test.jsonl\")\n",
    "\n",
    "train_set = pd.read_json(train_file, lines=True)\n",
    "validation_set = pd.read_json(validation_file, lines=True)\n",
    "test_set = pd.read_json(test_file, lines=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Exploratory Data Analysis (EDA)\n",
    "\n",
    "Analyze the data characteristics and features."
   ],
   "metadata": {
    "id": "mOfpilDeCWCt"
   },
   "id": "mOfpilDeCWCt"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ab6c980-3df5-4dd1-b266-870924b32893",
   "metadata": {
    "id": "4ab6c980-3df5-4dd1-b266-870924b32893",
    "outputId": "cb30f4e7-4670-48ef-d27d-f1bf626e0acf"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((37519, 5), (4689, 5), (4689, 5))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.shape, validation_set.shape, test_set.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbd8cdaf-718d-4dbd-8731-06bb4e64d3f0",
   "metadata": {
    "id": "cbd8cdaf-718d-4dbd-8731-06bb4e64d3f0",
    "outputId": "c25263bb-304d-4e72-bf85-c3d3d9350d04"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>title</th>\n",
       "      <th>summary</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>140323_russian_troops_crimea_naval_base</td>\n",
       "      <td>https://www.bbc.com/arabic/worldnews/2014/03/1...</td>\n",
       "      <td>القوات الأوكرانية تبدأ الانسحاب من القرم</td>\n",
       "      <td>بدأت القوات الأوكرانية الانسحاب من شبه جزيرة ا...</td>\n",
       "      <td>وكان الرئيس الأوكراني المؤقت، الكسندر تورتشينو...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>130528_egypt_nile_dam</td>\n",
       "      <td>https://www.bbc.com/arabic/middleeast/2013/05/...</td>\n",
       "      <td>هل يفرض سد النهضة الإثيوبي واقعا جديدا على مصر؟</td>\n",
       "      <td>\"هل سيتم تغيير العبارة الشهيرة للمؤرخ اليوناني...</td>\n",
       "      <td>بحلول عام 2050 ستحتاج مصر إلى 21 مليار متر مكع...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>world-47242349</td>\n",
       "      <td>https://www.bbc.com/arabic/world-47242349</td>\n",
       "      <td>تعرف على منطقة كشمير التي تسببت بحربين بين اله...</td>\n",
       "      <td>قالت الشرطة في القطاع الهندي من إقليم كشمير إن...</td>\n",
       "      <td>وذكرت وكالة الأنباء المحلية (جي.إن.إس) أن جماع...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>vert-cul-55078328</td>\n",
       "      <td>https://www.bbc.com/arabic/vert-cul-55078328</td>\n",
       "      <td>ماذا تعرف عن العالم الخفي للمعابد اليابانية ال...</td>\n",
       "      <td>في عام 816، تجول راهب يدعى كوكاي، في المنحدرات...</td>\n",
       "      <td>ووقع اختياره على واد عمقه 800 متر محاط بثماني ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>141023_yemen_hodeida</td>\n",
       "      <td>https://www.bbc.com/arabic/middleeast/2014/10/...</td>\n",
       "      <td>اشتباك بين الحوثيين و\"الحراك التهامي\" في الحدي...</td>\n",
       "      <td>أكد مصدر في \"الحراك التهامي\" لأبناء محافظة الح...</td>\n",
       "      <td>مسلح حوثي في إب وقال المصدر إن المسلحين الحوثي...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        id  \\\n",
       "0  140323_russian_troops_crimea_naval_base   \n",
       "1                    130528_egypt_nile_dam   \n",
       "2                           world-47242349   \n",
       "3                        vert-cul-55078328   \n",
       "4                     141023_yemen_hodeida   \n",
       "\n",
       "                                                 url  \\\n",
       "0  https://www.bbc.com/arabic/worldnews/2014/03/1...   \n",
       "1  https://www.bbc.com/arabic/middleeast/2013/05/...   \n",
       "2          https://www.bbc.com/arabic/world-47242349   \n",
       "3       https://www.bbc.com/arabic/vert-cul-55078328   \n",
       "4  https://www.bbc.com/arabic/middleeast/2014/10/...   \n",
       "\n",
       "                                               title  \\\n",
       "0           القوات الأوكرانية تبدأ الانسحاب من القرم   \n",
       "1    هل يفرض سد النهضة الإثيوبي واقعا جديدا على مصر؟   \n",
       "2  تعرف على منطقة كشمير التي تسببت بحربين بين اله...   \n",
       "3  ماذا تعرف عن العالم الخفي للمعابد اليابانية ال...   \n",
       "4  اشتباك بين الحوثيين و\"الحراك التهامي\" في الحدي...   \n",
       "\n",
       "                                             summary  \\\n",
       "0  بدأت القوات الأوكرانية الانسحاب من شبه جزيرة ا...   \n",
       "1  \"هل سيتم تغيير العبارة الشهيرة للمؤرخ اليوناني...   \n",
       "2  قالت الشرطة في القطاع الهندي من إقليم كشمير إن...   \n",
       "3  في عام 816، تجول راهب يدعى كوكاي، في المنحدرات...   \n",
       "4  أكد مصدر في \"الحراك التهامي\" لأبناء محافظة الح...   \n",
       "\n",
       "                                                text  \n",
       "0  وكان الرئيس الأوكراني المؤقت، الكسندر تورتشينو...  \n",
       "1  بحلول عام 2050 ستحتاج مصر إلى 21 مليار متر مكع...  \n",
       "2  وذكرت وكالة الأنباء المحلية (جي.إن.إس) أن جماع...  \n",
       "3  ووقع اختياره على واد عمقه 800 متر محاط بثماني ...  \n",
       "4  مسلح حوثي في إب وقال المصدر إن المسلحين الحوثي...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eba6d3b3-4d17-457c-ae36-e431d4178e04",
   "metadata": {
    "id": "eba6d3b3-4d17-457c-ae36-e431d4178e04",
    "outputId": "87426de4-6cf1-44a1-bcd2-95f50ed156e4"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>title</th>\n",
       "      <th>summary</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>140921_yemen_pm_resign</td>\n",
       "      <td>https://www.bbc.com/arabic/middleeast/2014/09/...</td>\n",
       "      <td>اتفاق لتشكيل حكومة جديدة بين الحوثيين وعدة احز...</td>\n",
       "      <td>وقع الحوثيون اتفاقا مع عدد من الاحزاب اليمنية ...</td>\n",
       "      <td>انضم بعض عناصر القوات المسلحة الى الحوثيين في ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>inthepress-56197077</td>\n",
       "      <td>https://www.bbc.com/arabic/inthepress-56197077</td>\n",
       "      <td>مظاهرات تونس: إلى أي مدى سيصل الصراع بين أطراف...</td>\n",
       "      <td>ناقش معلقون في صحف عربية تطور الأزمة السياسية ...</td>\n",
       "      <td>الرئيس التونسي قيس سعيد ورئيس الحكومة هشام الم...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>151114_gatwick_airport_frenchman_charged</td>\n",
       "      <td>https://www.bbc.com/arabic/worldnews/2015/11/1...</td>\n",
       "      <td>اتهام رجل بحيازة أسلحة بعد إعلان حالة تأهب في ...</td>\n",
       "      <td>اتهمت الشرطة البريطانية رسميا رجلا فرنسيا بحيا...</td>\n",
       "      <td>المسافرون انتظروا ست ساعات على أرض مطار غاتويك...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>media-48880286</td>\n",
       "      <td>https://www.bbc.com/arabic/media-48880286</td>\n",
       "      <td>عالم سري للسوريّات على فيسبوك محظور على الرجال...</td>\n",
       "      <td>تطلب شابة على إحدى صفحات فيسبوك المغلقة نصيحة ...</td>\n",
       "      <td>وانهالت تعليقات النساء على هذه المداخلة بين مؤ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>130311_us_nkorea</td>\n",
       "      <td>https://www.bbc.com/arabic/middleeast/2013/03/...</td>\n",
       "      <td>الولايات المتحدة توسع العقوبات ضد كوريا الشمالية</td>\n",
       "      <td>فرضت الولايات المتحدة عقوبات على بنك العملة ال...</td>\n",
       "      <td>قطعت كوريا الشمالية الخط الهاتفي الساخن بين ال...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         id  \\\n",
       "0                    140921_yemen_pm_resign   \n",
       "1                       inthepress-56197077   \n",
       "2  151114_gatwick_airport_frenchman_charged   \n",
       "3                            media-48880286   \n",
       "4                          130311_us_nkorea   \n",
       "\n",
       "                                                 url  \\\n",
       "0  https://www.bbc.com/arabic/middleeast/2014/09/...   \n",
       "1     https://www.bbc.com/arabic/inthepress-56197077   \n",
       "2  https://www.bbc.com/arabic/worldnews/2015/11/1...   \n",
       "3          https://www.bbc.com/arabic/media-48880286   \n",
       "4  https://www.bbc.com/arabic/middleeast/2013/03/...   \n",
       "\n",
       "                                               title  \\\n",
       "0  اتفاق لتشكيل حكومة جديدة بين الحوثيين وعدة احز...   \n",
       "1  مظاهرات تونس: إلى أي مدى سيصل الصراع بين أطراف...   \n",
       "2  اتهام رجل بحيازة أسلحة بعد إعلان حالة تأهب في ...   \n",
       "3  عالم سري للسوريّات على فيسبوك محظور على الرجال...   \n",
       "4   الولايات المتحدة توسع العقوبات ضد كوريا الشمالية   \n",
       "\n",
       "                                             summary  \\\n",
       "0  وقع الحوثيون اتفاقا مع عدد من الاحزاب اليمنية ...   \n",
       "1  ناقش معلقون في صحف عربية تطور الأزمة السياسية ...   \n",
       "2  اتهمت الشرطة البريطانية رسميا رجلا فرنسيا بحيا...   \n",
       "3  تطلب شابة على إحدى صفحات فيسبوك المغلقة نصيحة ...   \n",
       "4  فرضت الولايات المتحدة عقوبات على بنك العملة ال...   \n",
       "\n",
       "                                                text  \n",
       "0  انضم بعض عناصر القوات المسلحة الى الحوثيين في ...  \n",
       "1  الرئيس التونسي قيس سعيد ورئيس الحكومة هشام الم...  \n",
       "2  المسافرون انتظروا ست ساعات على أرض مطار غاتويك...  \n",
       "3  وانهالت تعليقات النساء على هذه المداخلة بين مؤ...  \n",
       "4  قطعت كوريا الشمالية الخط الهاتفي الساخن بين ال...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36131f5d-8c0b-4f3b-8a93-6c674e18bde9",
   "metadata": {
    "id": "36131f5d-8c0b-4f3b-8a93-6c674e18bde9",
    "outputId": "cc290176-d969-44c5-c7e1-a101fe01e4c4"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>title</th>\n",
       "      <th>summary</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>130806_nidhal_hassan_trial</td>\n",
       "      <td>https://www.bbc.com/arabic/worldnews/2013/08/1...</td>\n",
       "      <td>نضال حسن يمثل أمام محكمة عسكرية أمريكية</td>\n",
       "      <td>تنظر محكمة عسكرية أمريكية في وقت لاحق من اليوم...</td>\n",
       "      <td>نضال حسن واعترف نضال حسن، الذي يدافع عن نفسه، ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>160129_germany_asylum_seekers</td>\n",
       "      <td>https://www.bbc.com/arabic/worldnews/2016/01/1...</td>\n",
       "      <td>ألمانيا تسعى للحد من الهجرة بإعلان الجزائر وتو...</td>\n",
       "      <td>كشفت ألمانيا النقاب عن خطط لإضافة الجزائر والم...</td>\n",
       "      <td>ألمانيا تواجه مصاعب في التعامل مع الأعداد المت...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>130729_syria_homs_area_retaken</td>\n",
       "      <td>https://www.bbc.com/arabic/middleeast/2013/07/...</td>\n",
       "      <td>الخالدية : التليفزيون السوري يعلن استعادة الجي...</td>\n",
       "      <td>قال التليفزيون السوري إن قوات الحكومة استعادت ...</td>\n",
       "      <td>وأكدت وسائل الإعلام السورية أن الجيش \"استعاد ا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>140517_arsenal_fa_cup_winners</td>\n",
       "      <td>https://www.bbc.com/arabic/sports/2014/05/1405...</td>\n",
       "      <td>الارسنال يتوج بلقب كأس انكلترا</td>\n",
       "      <td>توج فريق الارسنال ببطولة كأس انجلترا لكرة القد...</td>\n",
       "      <td>وفاجأ هال سيتي الحضور بمباغتة الارسنال بهدفين ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>140722_iraq_minorities</td>\n",
       "      <td>https://www.bbc.com/arabic/middleeast/2014/07/...</td>\n",
       "      <td>العراق: الأقليات في سهل نينوى</td>\n",
       "      <td>يوضع الصراع في العراق غالبا في إطار صراع بين ا...</td>\n",
       "      <td>معبد يزيدي في سهول محافظة نينوى يعيش المسيحيون...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               id  \\\n",
       "0      130806_nidhal_hassan_trial   \n",
       "1   160129_germany_asylum_seekers   \n",
       "2  130729_syria_homs_area_retaken   \n",
       "3   140517_arsenal_fa_cup_winners   \n",
       "4          140722_iraq_minorities   \n",
       "\n",
       "                                                 url  \\\n",
       "0  https://www.bbc.com/arabic/worldnews/2013/08/1...   \n",
       "1  https://www.bbc.com/arabic/worldnews/2016/01/1...   \n",
       "2  https://www.bbc.com/arabic/middleeast/2013/07/...   \n",
       "3  https://www.bbc.com/arabic/sports/2014/05/1405...   \n",
       "4  https://www.bbc.com/arabic/middleeast/2014/07/...   \n",
       "\n",
       "                                               title  \\\n",
       "0            نضال حسن يمثل أمام محكمة عسكرية أمريكية   \n",
       "1  ألمانيا تسعى للحد من الهجرة بإعلان الجزائر وتو...   \n",
       "2  الخالدية : التليفزيون السوري يعلن استعادة الجي...   \n",
       "3                     الارسنال يتوج بلقب كأس انكلترا   \n",
       "4                      العراق: الأقليات في سهل نينوى   \n",
       "\n",
       "                                             summary  \\\n",
       "0  تنظر محكمة عسكرية أمريكية في وقت لاحق من اليوم...   \n",
       "1  كشفت ألمانيا النقاب عن خطط لإضافة الجزائر والم...   \n",
       "2  قال التليفزيون السوري إن قوات الحكومة استعادت ...   \n",
       "3  توج فريق الارسنال ببطولة كأس انجلترا لكرة القد...   \n",
       "4  يوضع الصراع في العراق غالبا في إطار صراع بين ا...   \n",
       "\n",
       "                                                text  \n",
       "0  نضال حسن واعترف نضال حسن، الذي يدافع عن نفسه، ...  \n",
       "1  ألمانيا تواجه مصاعب في التعامل مع الأعداد المت...  \n",
       "2  وأكدت وسائل الإعلام السورية أن الجيش \"استعاد ا...  \n",
       "3  وفاجأ هال سيتي الحضور بمباغتة الارسنال بهدفين ...  \n",
       "4  معبد يزيدي في سهول محافظة نينوى يعيش المسيحيون...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_set.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Visualize sequence length distribution to decide the maximum length for source and target sequences"
   ],
   "metadata": {
    "id": "_Cav7x0CC2Zh"
   },
   "id": "_Cav7x0CC2Zh"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4da5075a-3d7e-4702-b57b-ce1b22051ba4",
   "metadata": {
    "id": "4da5075a-3d7e-4702-b57b-ce1b22051ba4",
    "outputId": "b6e0a071-722e-4245-de4b-95ec72adc643"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAAJOCAYAAABm7rQwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAADBN0lEQVR4nOzdeVhV5f738c9GBJwAUQEpRNISVJzLMHNIEpVKyyxN0xKlwSGlo0ZHzaEOZWmaWv44ZVhpgx3zlJpKmuGAY5JDZGYYlm48psDBAVDW80cP67jbqGyDjeD7dV3relz3/V1rfxen68f9fFh7LYthGIYAAAAAAAAAJ3Ip7wYAAAAAAABw/SGUAgAAAAAAgNMRSgEAAAAAAMDpCKUAAAAAAADgdIRSAAAAAAAAcDpCKQAAAAAAADgdoRQAAAAAAACcjlAKAAAAAAAATkcoBQAAAAAAAKcjlAKAUrRhwwZZLBZt2LChvFsBAACApMcee0wNGzYs7zYAFINQCkCxLBZLibbSCl+OHj2qKVOmKDU19Yq1n3zyiSwWiz777DO7uZYtW8pisejrr7+2m2vQoIE6dOhQGu2WmkOHDumJJ57QTTfdJA8PD3l6euqOO+7QnDlzdPbs2fJuT5L05ptvKjExsbzbAACgVO3du1cPPviggoKC5OHhoRtuuEF333235s6dW96tVUi9evVS7dq1ZRiGzfju3btlsVgUFBRkd8z69etlsViUkJDgrDZL5LPPPlPPnj1Vt25dubm5KSAgQA899JDWr19f3q1JcmzdDFzrXMu7AQDXpvfff99m/7333lNSUpLdeGhoaKl83tGjRzV16lQ1bNhQrVq1umxtx44dJUmbNm3S/fffb47n5ORo3759cnV11ebNm9W1a1dz7siRIzpy5Ij69+9fKv2WhpUrV6pfv35yd3fX4MGD1bx5c+Xn52vTpk0aN26c9u/ff00s0t58803VrVtXjz32WHm3AgBAqdiyZYu6du2qBg0aaPjw4fL399eRI0e0detWzZkzR6NGjSrvFiucjh076ssvv9S+ffsUFhZmjm/evFmurq7KyMjQr7/+qhtvvNFmrujYa4FhGBo6dKgSExPVunVrxcbGyt/fX8eOHdNnn32mbt26afPmzeX+R05H1s3AtY5QCkCxBg0aZLO/detWJSUl2Y2Xh4CAAAUHB2vTpk024ykpKTIMQ/369bObK9r/q4sewzB07tw5VatW7S+dJz09Xf3791dQUJDWr1+v+vXrm3MjRozQTz/9pJUrV/6lzwAAAMV76aWX5OXlpR07dsjb29tm7vjx4+XTVDkqjfXNxX80/HMo1atXL61fv16bNm2y+QPhpk2bVKdOnb/8R85z587Jzc1NLi5/7YtAM2fOVGJiosaMGaNZs2bJYrGYc3//+9/1/vvvy9WV/y80UJr4+h6Aq1ZYWKjZs2erWbNm8vDwkJ+fn5544gmdOnXKrHnhhRfk4uKidevW2RwbExMjNzc3fffdd9qwYYNuvfVWSdLjjz9ufjXwcl8Z69ixo3bv3m3zFbfNmzerWbNm6tmzp7Zu3arCwkKbOYvFojvuuEOSdP78eU2fPl2NGjWSu7u7GjZsqOeff155eXk2n9OwYUPdc889WrNmjdq1a6dq1arp//7v/yRJv/76q/r06aMaNWrI19dXY8eOtTv+UmbMmKHc3Fy98847NoFUkcaNG+uZZ54x90var8Vi0ZQpU+zO17BhQ5s7nRITE2WxWLR582bFxsaqXr16qlGjhu6//3795z//sTlu//79+uabb8z/Xbp06SJJKigo0NSpU3XzzTfLw8NDderUUceOHZWUlFSinwEAAOXl0KFDatasmV0gJUm+vr7mvw8fPnzJNcmff+dOmTJFFotFP/74owYNGiQvLy/Vq1dPkyZNkmEYOnLkiHr37i1PT0/5+/tr5syZNucrei7lJ598oqlTp+qGG25QrVq19OCDDyo7O1t5eXkaM2aMfH19VbNmTT3++ON264B3331Xd911l3x9feXu7q6mTZvqrbfesuv9Uuubzp07q2XLlsX+zJo0aaLIyMhL/kxvu+02ubm5mXc/Fdm8ebM6deqk2267zWausLBQW7duVYcOHczw5+eff1a/fv3k4+Oj6tWr6/bbb7f7I13Rz+mjjz7SxIkTdcMNN6h69erKycmRJC1fvlzNmzeXh4eHmjdvXuzjHopz9uxZxcfHKyQkRK+99ppNIFXk0Ucf1W233Wbul6TfojXX4cOHi72Oix+F0aVLFzVv3lzff/+9unbtqurVq+uGG27QjBkzbI673Lr54MGD6tu3r/z9/eXh4aEbb7xR/fv3V3Z2dol+DoCzEfMCuGpPPPGEEhMT9fjjj2v06NFKT0/XvHnztHv3bm3evFlVq1bVxIkT9cUXXyg6Olp79+5VrVq1tGbNGv3zn//U9OnT1bJlS2VmZmratGmaPHmyYmJidOedd0rSZW+N7tixo95//31t27bNDEmKbqfu0KGDsrOztW/fPrVo0cKcCwkJUZ06dSRJw4YN06JFi/Tggw/q2Wef1bZt2xQfH6+0tDS7xcuBAwc0YMAAPfHEExo+fLiaNGmis2fPqlu3bsrIyNDo0aMVEBCg999/v8TPGvjiiy900003lfj2b0f6dcSoUaNUu3ZtvfDCCzp8+LBmz56tkSNH6uOPP5YkzZ49W6NGjVLNmjX197//XZLk5+cn6Y/Fd3x8vIYNG6bbbrtNOTk52rlzp7799lvdfffdV90TAABlLSgoSCkpKdq3b5+aN29equd++OGHFRoaqpdfflkrV67Uiy++KB8fH/3f//2f7rrrLr3yyitavHix/va3v+nWW29Vp06dbI6Pj49XtWrV9Nxzz+mnn37S3LlzVbVqVbm4uOjUqVOaMmWKtm7dqsTERAUHB2vy5MnmsW+99ZaaNWum++67T66urvriiy/09NNPq7CwUCNGjLD5nOLWNzVr1tTw4cPtfi47duzQjz/+qIkTJ17yuj08PNS2bVubu9WLHp/QoUMHZWVl2QQ2e/fuVU5OjnmHVWZmpjp06KAzZ85o9OjRqlOnjhYtWqT77rtPn376qc0jGyRp+vTpcnNz09/+9jfl5eXJzc1Na9euVd++fdW0aVPFx8fr999/1+OPP27zlcFL2bRpk06ePKkxY8aoSpUqV6x3tN+SOnXqlHr06KEHHnhADz30kD799FNNmDBBYWFh6tmzp0JDQy+5bs7Pz1dkZKTy8vI0atQo+fv767ffftOKFSuUlZUlLy+vq+oJKFMGAJTAiBEjjIv/T8bGjRsNScbixYtt6lavXm03vnfvXsPNzc0YNmyYcerUKeOGG24w2rVrZxQUFJg1O3bsMCQZ7777bon62b9/vyHJmD59umEYhlFQUGDUqFHDWLRokWEYhuHn52fMnz/fMAzDyMnJMapUqWIMHz7cMAzDSE1NNSQZw4YNsznn3/72N0OSsX79enMsKCjIkGSsXr3apnb27NmGJOOTTz4xx06fPm00btzYkGR8/fXXl+w9OzvbkGT07t27RNfqSL+SjBdeeMHuHEFBQcaQIUPM/XfffdeQZERERBiFhYXm+NixY40qVaoYWVlZ5lizZs2Mzp07252zZcuWRlRUVImuAQCAa8natWuNKlWqGFWqVDHCw8ON8ePHG2vWrDHy8/Nt6tLT0y+5Pvnz79wXXnjBkGTExMSYY+fPnzduvPFGw2KxGC+//LI5furUKaNatWo2v5u//vprQ5LRvHlzmz4GDBhgWCwWo2fPnjafHx4ebgQFBdmMnTlzxq7PyMhI46abbrIZu9T6Jisry/Dw8DAmTJhgMz569GijRo0aRm5urt35LzZu3DhDkvHrr78ahmEYH374oeHh4WHk5eUZq1atMqpUqWLk5OQYhmEY8+bNMyQZmzdvNgzDMMaMGWNIMjZu3Gie77///a8RHBxsNGzY0Lhw4YLNz+mmm26yu95WrVoZ9evXt1nHrF271pBk97P6szlz5hiSjM8+++yydUVK2m/Rmis9Pd3m+KLruHjN2LlzZ0OS8d5775ljeXl5hr+/v9G3b19z7FLr5t27dxuSjKVLl5boGoBrAV/fA3BVli5dKi8vL9199906ceKEubVt21Y1a9a0eftd8+bNNXXqVL399tuKjIzUiRMntGjRor/0nfzQ0FDVqVPH/Gvcd999p9OnT5t3HnXo0MG8RTwlJUUXLlww/xK3atUqSVJsbKzNOZ999llJsrvtOjg42O529VWrVql+/fp68MEHzbHq1asrJibmir0X3V5eq1atEl2ro/06IiYmxub29DvvvFMXLlzQL7/8csVjvb29tX//fh08ePCqPx8AgPJw9913KyUlRffdd5++++47zZgxQ5GRkbrhhhv0+eef/6VzDxs2zPx3lSpV1K5dOxmGoejoaHPc29tbTZo00c8//2x3/ODBg1W1alVzv3379uYDuC/Wvn17HTlyROfPnzfHLn4mVHZ2tk6cOKHOnTvr559/tvv6VnHrGy8vL/Xu3Vsffvih+Ra9Cxcu6OOPPzYfWXA5RWutjRs3SvrjTvW2bdvKzc1N4eHh5lf2iuY8PDzUrl07SX+sd2677Tab53/WrFlTMTExOnz4sL7//nubzxoyZIjN9R47dkypqakaMmSIzR1Bd999t5o2bXrZvqWrW5850m9J1axZ0+YZrm5ubrrtttuK/W/lz4que82aNTpz5sxVfT7gbIRSAK7KwYMHlZ2dLV9fX9WrV89my83NtXtI6Lhx49SyZUtt375dL7zwQokWB5djsVjUoUMH89lRmzdvlq+vrxo3bizJNpT685tdfvnlF7m4uJi1Rfz9/eXt7W0XyAQHB9t9/i+//KLGjRvbPW+gSZMmV+zd09NTkvTf//63JJfqcL+OaNCggc1+7dq1JcnmuWCXMm3aNGVlZemWW25RWFiYxo0bpz179lx1LwAAONOtt96qZcuW6dSpU9q+fbvi4uL03//+Vw8++OBVBwqS/e9WLy8veXh4qG7dunbjxf2+Le54SQoMDLQbLywstAmbNm/erIiICNWoUUPe3t6qV6+enn/+eUkqNpQqzuDBg5WRkWEGS1999ZUyMzP16KOPXvKai9xxxx3mMyuL+il6nqe3t7eaNm1qM3frrbfKzc1N0h/rneLWUUUPQb/S+qxo/uabb7Y7R1mtzxzpt6RuvPFGu/Vl7dq1S7Q2Cw4OVmxsrN5++23VrVtXkZGRmj9/Ps+TwjWNUArAVSksLJSvr6+SkpKK3aZNm2ZT//PPP5t31Ozdu7dUeujYsaOys7O1d+9eu9fzdujQQb/88ot+++03bdq0SQEBAbrppptsji/uAZbF+atv2vszT09PBQQEaN++fQ4dV9J+i3PhwoVixy/1zISiv45eTqdOnXTo0CEtXLhQzZs319tvv602bdro7bffvuo+AQBwNjc3N9166636xz/+obfeeksFBQVaunSppEv/7r3U71Wp+N+tjvy+vVTtlc5x6NAhdevWTSdOnNCsWbO0cuVKJSUlaezYsZJk8wIY6dLrm8jISPn5+emDDz6QJH3wwQfy9/dXREREsfUXq1OnjkJCQrRp0ybl5uZqz549duuzTZs26ddff1VGRsZfeityaa/PQkJCJJXeOrWIo/8N/ZW1mfTHGwT37Nmj559/XmfPntXo0aPVrFkz/frrryVrGHAyQikAV6VRo0b6/fffdccddygiIsJuu/jNLYWFhXrsscfk6emp559/Xh9++KGWLVtmc76rCVwufvXwxX+Jk6S2bdvK3d1dGzZs0LZt22zmgoKCVFhYaPe1s8zMTGVlZSkoKOiKnx0UFKRDhw7ZLRAOHDhQot7vueceHTp0SCkpKSX6rJL2W7t2bWVlZdnU5efn69ixYyXqqziX+9/Gx8dHjz/+uD788EMdOXJELVq0KPbtfwAAVARFXyUr+r1ZdAfxn3+3/pW7lMvKF198oby8PH3++ed64okn1KtXL0VERDgc3lSpUkWPPPKIPv30U506dUrLly/XgAEDSvTwb+mP9dnevXu1du1aXbhwwS6U2rZtm/nGuYtDqaCgoGLXUT/88IM5fzlF88U9VqAk67OOHTuqdu3a+vDDDy8bOjrab1n8N3SldXNYWJgmTpyo5ORkbdy4Ub/99psWLFhw1Z8HlCVCKQBX5aGHHtKFCxc0ffp0u7nz58/b/OKdNWuWtmzZooSEBE2fPl0dOnTQU089pRMnTpg1Rc8o+PMv7Mtp166dPDw8tHjxYv322282ix53d3e1adNG8+fP1+nTp20WPb169ZL0x5vlLjZr1ixJUlRU1BU/u1evXjp69Kg+/fRTc+zMmTNKSEgoUe/jx49XjRo1NGzYMGVmZtrNHzp0SHPmzHG430aNGik5OdmmLiEhoUSLq0upUaNGsf+7/P777zb7NWvWVOPGje1eTw0AwLXm66+/LvbOk6LnOBZ9LcvT01N169a1+9365ptvln2TDioKjS6+ruzsbL377rsOn+vRRx/VqVOn9MQTTyg3N9fmGUdX0rFjR124cEGvvfaabr75ZtWrV8+c69Chg3Jzc/Xmm2/KxcXFZu3Wq1cvbd++3eYPdqdPn1ZCQoIaNmx4xUc/1K9fX61atdKiRYtsvq6WlJRUoq9jVq9eXRMmTFBaWpomTJhQ7H8fH3zwgbZv3+5Qv40aNZIkm/+GLly4UOI1Y3EutW7OycmxecaY9EdA5eLiwvoM16yrf8owgOta586d9cQTTyg+Pl6pqanq3r27qlatqoMHD2rp0qWaM2eOHnzwQaWlpWnSpEl67LHHdO+990qSEhMT1apVKz399NP65JNPJP3xC9vb21sLFixQrVq1VKNGDbVv3/6SzzuQ/ne7/caNG+Xu7q62bdvazHfo0EEzZ86UZPuXuJYtW2rIkCFKSEhQVlaWOnfurO3bt2vRokXq06ePunbtesXrHz58uObNm6fBgwdr165dql+/vt5//31Vr169RD+/Ro0aacmSJeZrowcPHqzmzZsrPz9fW7Zs0dKlS/XYY4853O+wYcP05JNPqm/fvrr77rv13Xffac2aNXbPsXBE27Zt9dZbb+nFF19U48aN5evrq7vuuktNmzZVly5d1LZtW/n4+Gjnzp369NNPNXLkyKv+LAAAnGHUqFE6c+aM7r//foWEhJi/fz/++GM1bNhQjz/+uFk7bNgwvfzyyxo2bJjatWun5ORk/fjjj+XYffG6d+8uNzc33XvvvWaY9M9//lO+vr4O3zHdunVrNW/eXEuXLlVoaKjatGlT4mOL1lwpKSnmWqbILbfcorp16yolJUVhYWHy9vY255577jl9+OGH6tmzp0aPHi0fHx8tWrRI6enp+te//iUXlyvfTxEfH6+oqCh17NhRQ4cO1cmTJzV37lw1a9ZMubm5Vzx+3Lhx2r9/v2bOnKmvv/5aDz74oPz9/WW1WrV8+XJt375dW7ZscajfZs2a6fbbb1dcXJxOnjwpHx8fffTRR3bhkSMutW7+7rvvNHLkSPXr10+33HKLzp8/r/fff19VqlRR3759r/rzgDJVTm/9A1DBjBgxwiju/2QkJCQYbdu2NapVq2bUqlXLCAsLM8aPH28cPXrUOH/+vHHrrbcaN954o82reQ3jf6/d/fjjj82xf//730bTpk0NV1fXS75++c/i4uIMSUaHDh3s5pYtW2ZIMmrVqmWcP3/eZq6goMCYOnWqERwcbFStWtUIDAw04uLijHPnztnUBQUFGVFRUcV+9i+//GLcd999RvXq1Y26desazzzzjLF69Wq71/tezo8//mgMHz7caNiwoeHm5mbUqlXLuOOOO4y5c+fa9FLSfi9cuGBMmDDBqFu3rlG9enUjMjLS+Omnn4ygoCCb104XvZ54x44dNscX93piq9VqREVFGbVq1TIkGZ07dzYMwzBefPFF47bbbjO8vb2NatWqGSEhIcZLL71k9zptAACuNV9++aUxdOhQIyQkxKhZs6bh5uZmNG7c2Bg1apSRmZlpU3vmzBkjOjra8PLyMmrVqmU89NBDxvHjxw1JxgsvvGDWvfDCC4Yk4z//+Y/N8UOGDDFq1Khh10Pnzp2NZs2amftFv4OXLl1qU3ep39nFfd7nn39utGjRwvDw8DAaNmxovPLKK8bChQsNSUZ6erpZd7n1TZEZM2YYkox//OMfl60rTkBAgCHJSEhIsJu77777DEnGU089ZTd36NAh48EHHzS8vb0NDw8P47bbbjNWrFhhU3Opn1ORf/3rX0ZoaKjh7u5uNG3a1Fi2bJkxZMgQIygoqMT9f/rpp0b37t0NHx8fw9XV1ahfv77x8MMPGxs2bHC436K6iIgIw93d3fDz8zOef/55IykpyW7N9ef/JooU139x6+aff/7ZGDp0qNGoUSPDw8PD8PHxMbp27Wp89dVXJb52wNkshlHCJ6YBAAAAAK4Lc+bM0dixY3X48GG7NwICQGkhlAIAAAAAmAzDUMuWLVWnTh19/fXX5d0OgEqMZ0oBAAAAAHT69Gl9/vnn+vrrr7V37179+9//Lu+WAFRy3CkFAAAAANDhw4cVHBwsb29vPf3003rppZfKuyUAlRyhFAAAAAAAAJzuyu/VBAAAwDUlOTlZ9957rwICAmSxWLR8+XK7mrS0NN13333y8vJSjRo1dOuttyojI8OcP3funEaMGKE6deqoZs2a6tu3rzIzM23OkZGRoaioKFWvXl2+vr4aN26c3WvMN2zYoDZt2sjd3V2NGzdWYmJiWVwyAACohAilAAAAKpjTp0+rZcuWmj9/frHzhw4dUseOHRUSEqINGzZoz549mjRpkjw8PMyasWPH6osvvtDSpUv1zTff6OjRo3rggQfM+QsXLigqKkr5+fnasmWLFi1apMTERE2ePNmsSU9PV1RUlLp27arU1FSNGTNGw4YN05o1a8ru4gEAQKXB1/dKSWFhoY4ePapatWrJYrGUdzsAAKAUGIah//73vwoICJCLy7X5tzyLxaLPPvtMffr0Mcf69++vqlWr6v333y/2mOzsbNWrV09LlizRgw8+KEn64YcfFBoaqpSUFN1+++368ssvdc899+jo0aPy8/OTJC1YsEATJkzQf/7zH7m5uWnChAlauXKl9u3bZ/PZWVlZWr16dYn6Zw0FAEDlU9I1FG/fKyVHjx5VYGBgebcBAADKwJEjR3TjjTeWdxslUlhYqJUrV2r8+PGKjIzU7t27FRwcrLi4ODO42rVrlwoKChQREWEeFxISogYNGpihVEpKisLCwsxASpIiIyP11FNPaf/+/WrdurVSUlJszlFUM2bMmBL3yxoKAIDK60prKEKpUlKrVi1Jf/zAPT09y7kbAABQGnJychQYGGj+nq8Ijh8/rtzcXL388st68cUX9corr2j16tV64IEH9PXXX6tz586yWq1yc3OTt7e3zbF+fn6yWq2SJKvVahNIFc0XzV2uJicnR2fPnlW1atXs+svLy1NeXp65X3TTPmsoAAAqj5KuoQilSknR7eaenp4sqAAAqGQq0tfKCgsLJUm9e/fW2LFjJUmtWrXSli1btGDBAnXu3Lk821N8fLymTp1qN84aCgCAyudKa6hr8+EIAAAAuCp169aVq6urmjZtajMeGhpqvn3P399f+fn5ysrKsqnJzMyUv7+/WfPnt/EV7V+pxtPTs9i7pCQpLi5O2dnZ5nbkyJGru1AAAFDhEUoBAABUIm5ubrr11lt14MABm/Eff/xRQUFBkqS2bduqatWqWrdunTl/4MABZWRkKDw8XJIUHh6uvXv36vjx42ZNUlKSPD09zcArPDzc5hxFNUXnKI67u7t5VxR3RwEAcH3j63sAAAAVTG5urn766SdzPz09XampqfLx8VGDBg00btw4Pfzww+rUqZO6du2q1atX64svvtCGDRskSV5eXoqOjlZsbKx8fHzk6empUaNGKTw8XLfffrskqXv37mratKkeffRRzZgxQ1arVRMnTtSIESPk7u4uSXryySc1b948jR8/XkOHDtX69ev1ySefaOXKlU7/mQAAgIrHYhQ9XRJ/SU5Ojry8vJSdnc1f/AAAqCSu1d/vGzZsUNeuXe3GhwwZosTEREnSwoULFR8fr19//VVNmjTR1KlT1bt3b7P23LlzevbZZ/Xhhx8qLy9PkZGRevPNN82v5knSL7/8oqeeekobNmxQjRo1NGTIEL388stydf3f3zU3bNigsWPH6vvvv9eNN96oSZMm6bHHHivxtVyrP2MAAHD1Svr7nVCqlLCgAgCg8uH3e9njZwwAQOVT0t/vPFMKAAAAAAAATkcoBQAAAAAAAKcr11AqOTlZ9957rwICAmSxWLR8+XKbeYvFUuz26quvmjUNGza0m3/55ZdtzrNnzx7deeed8vDwUGBgoGbMmGHXy9KlSxUSEiIPDw+FhYVp1apVZXLNAAAAAAAAKOdQ6vTp02rZsqXmz59f7PyxY8dstoULF8pisahv3742ddOmTbOpGzVqlDmXk5Oj7t27KygoSLt27dKrr76qKVOmKCEhwazZsmWLBgwYoOjoaO3evVt9+vRRnz59tG/fvrK5cAAAAAAAgOuc65VLyk7Pnj3Vs2fPS85f/PYXSfr3v/+trl276qabbrIZr1Wrll1tkcWLFys/P18LFy6Um5ubmjVrptTUVM2aNUsxMTGSpDlz5qhHjx4aN26cJGn69OlKSkrSvHnztGDBgr9yiQAAAAAAAChGhXmmVGZmplauXKno6Gi7uZdffll16tRR69at9eqrr+r8+fPmXEpKijp16iQ3NzdzLDIyUgcOHNCpU6fMmoiICJtzRkZGKiUlpYyuBgAAAAAA4PpWrndKOWLRokWqVauWHnjgAZvx0aNHq02bNvLx8dGWLVsUFxenY8eOadasWZIkq9Wq4OBgm2P8/PzMudq1a8tqtZpjF9dYrdZL9pOXl6e8vDxzPycn5y9dHwAAAAAAwPWkwoRSCxcu1MCBA+Xh4WEzHhsba/67RYsWcnNz0xNPPKH4+Hi5u7uXWT/x8fGaOnVqmZ0fAAAAAACgMqsQX9/buHGjDhw4oGHDhl2xtn379jp//rwOHz4s6Y/nUmVmZtrUFO0XPYfqUjWXek6VJMXFxSk7O9vcjhw54sglAQAAAAAAXNcqRCj1zjvvqG3btmrZsuUVa1NTU+Xi4iJfX19JUnh4uJKTk1VQUGDWJCUlqUmTJqpdu7ZZs27dOpvzJCUlKTw8/JKf4+7uLk9PT5sNAAAAAAAAJVOuoVRubq5SU1OVmpoqSUpPT1dqaqoyMjLMmpycHC1durTYu6RSUlI0e/Zsfffdd/r555+1ePFijR07VoMGDTIDp0ceeURubm6Kjo7W/v379fHHH2vOnDk2X/t75plntHr1as2cOVM//PCDpkyZop07d2rkyJFl+wMAAAAAAAC4TpXrM6V27typrl27mvtFQdGQIUOUmJgoSfroo49kGIYGDBhgd7y7u7s++ugjTZkyRXl5eQoODtbYsWNtAicvLy+tXbtWI0aMUNu2bVW3bl1NnjxZMTExZk2HDh20ZMkSTZw4Uc8//7xuvvlmLV++XM2bNy+jKwcAAAAAALi+WQzDMMq7icogJydHXl5eys7O5qt8AABUEvx+L3v8jAEAqHxK+vu9QjxTCgAAAAAAAJVLuX59DyWXkZGhEydOXNWxdevWVYMGDUq5IwAAAFwKazcAAK6MUKoCyMjIUGhIE505e+6qjq9ezUNpPxxgcQMAAOAEGRkZatIkVOfOnbmq4z08quvAgTTWbgCASo9QqgI4ceKEzpw9pw+elkIDHDs27ag06M1zOnHiBAsbAAAAJzhx4sT/D6Q+kBTq4NFpOnduEGs3AMB1gVCqAgkNkNoEl3cXAAAAKJlQSW3KuwkAAK5ZPOgcAAAAAAAATkcoBQAAAAAAAKcjlAIAAAAAAIDTEUoBAAAAAADA6QilAAAAAAAA4HSEUgAAAAAAAHA6QikAAAAAAAA4HaEUAAAAAAAAnI5QCgAAAAAAAE5HKAUAAAAAAACnI5QCAAAAAACA0xFKAQAAAAAAwOkIpQAAAAAAAOB0hFIAAAAAAABwOkIpAAAAAAAAOB2hFAAAAAAAAJyOUAoAAAAAAABORygFAAAAAAAApyOUAgAAAAAAgNMRSgEAAAAAAMDpCKUAAAAAAADgdIRSAAAAAAAAcDpCKQAAAAAAADgdoRQAAAAAAACcjlAKAAAAAAAATkcoBQAAAAAAAKcjlAIAAAAAAIDTEUoBAAAAAADA6QilAAAAAAAA4HSEUgAAAAAAAHA6QikAAAAAAAA4HaEUAABABZOcnKx7771XAQEBslgsWr58+SVrn3zySVksFs2ePdtm/OTJkxo4cKA8PT3l7e2t6Oho5ebm2tTs2bNHd955pzw8PBQYGKgZM2bYnX/p0qUKCQmRh4eHwsLCtGrVqtK4RAAAcB0glAIAAKhgTp8+rZYtW2r+/PmXrfvss8+0detWBQQE2M0NHDhQ+/fvV1JSklasWKHk5GTFxMSY8zk5OerevbuCgoK0a9cuvfrqq5oyZYoSEhLMmi1btmjAgAGKjo7W7t271adPH/Xp00f79u0rvYsFAACVlmt5NwAAAADH9OzZUz179rxszW+//aZRo0ZpzZo1ioqKsplLS0vT6tWrtWPHDrVr106SNHfuXPXq1UuvvfaaAgICtHjxYuXn52vhwoVyc3NTs2bNlJqaqlmzZpnh1Zw5c9SjRw+NGzdOkjR9+nQlJSVp3rx5WrBgQRlcOQAAqEy4UwoAAKCSKSws1KOPPqpx48apWbNmdvMpKSny9vY2AylJioiIkIuLi7Zt22bWdOrUSW5ubmZNZGSkDhw4oFOnTpk1ERERNueOjIxUSkpKWVwWAACoZLhTCgAAoJJ55ZVX5OrqqtGjRxc7b7Va5evrazPm6uoqHx8fWa1WsyY4ONimxs/Pz5yrXbu2rFarOXZxTdE5ipOXl6e8vDxzPycnp+QXBgAAKhXulAIAAKhEdu3apTlz5igxMVEWi6W827ETHx8vLy8vcwsMDCzvlgAAQDkhlAIAAKhENm7cqOPHj6tBgwZydXWVq6urfvnlFz377LNq2LChJMnf31/Hjx+3Oe78+fM6efKk/P39zZrMzEybmqL9K9UUzRcnLi5O2dnZ5nbkyJG/dL0AAKDiIpQCAACoRB599FHt2bNHqamp5hYQEKBx48ZpzZo1kqTw8HBlZWVp165d5nHr169XYWGh2rdvb9YkJyeroKDArElKSlKTJk1Uu3Zts2bdunU2n5+UlKTw8PBL9ufu7i5PT0+bDQAAXJ94phQAAEAFk5ubq59++sncT09PV2pqqnx8fNSgQQPVqVPHpr5q1ary9/dXkyZNJEmhoaHq0aOHhg8frgULFqigoEAjR45U//79FRAQIEl65JFHNHXqVEVHR2vChAnat2+f5syZo9dff9087zPPPKPOnTtr5syZioqK0kcffaSdO3cqISHBCT8FAABQ0XGnFAAAQAWzc+dOtW7dWq1bt5YkxcbGqnXr1po8eXKJz7F48WKFhISoW7du6tWrlzp27GgTJnl5eWnt2rVKT09X27Zt9eyzz2ry5MmKiYkxazp06KAlS5YoISFBLVu21Keffqrly5erefPmpXexAACg0uJOKQAAgAqmS5cuMgyjxPWHDx+2G/Px8dGSJUsue1yLFi20cePGy9b069dP/fr1K3EvAAAARbhTCgAAAAAAAE5HKAUAAAAAAACnI5QCAAAAAACA0xFKAQAAAAAAwOkIpQAAAAAAAOB0hFIAAAAAAABwOkIpAAAAAAAAOB2hFAAAAAAAAJyOUAoAAAAAAABOV66hVHJysu69914FBATIYrFo+fLlNvOPPfaYLBaLzdajRw+bmpMnT2rgwIHy9PSUt7e3oqOjlZuba1OzZ88e3XnnnfLw8FBgYKBmzJhh18vSpUsVEhIiDw8PhYWFadWqVaV+vQAAAAAAAPhDuYZSp0+fVsuWLTV//vxL1vTo0UPHjh0ztw8//NBmfuDAgdq/f7+SkpK0YsUKJScnKyYmxpzPyclR9+7dFRQUpF27dunVV1/VlClTlJCQYNZs2bJFAwYMUHR0tHbv3q0+ffqoT58+2rdvX+lfNAAAAAAAAORanh/es2dP9ezZ87I17u7u8vf3L3YuLS1Nq1ev1o4dO9SuXTtJ0ty5c9WrVy+99tprCggI0OLFi5Wfn6+FCxfKzc1NzZo1U2pqqmbNmmWGV3PmzFGPHj00btw4SdL06dOVlJSkefPmacGCBaV4xQAAAAAAAJAqwDOlNmzYIF9fXzVp0kRPPfWUfv/9d3MuJSVF3t7eZiAlSREREXJxcdG2bdvMmk6dOsnNzc2siYyM1IEDB3Tq1CmzJiIiwuZzIyMjlZKSUpaXBgAAAAAAcN0q1zulrqRHjx564IEHFBwcrEOHDun5559Xz549lZKSoipVqshqtcrX19fmGFdXV/n4+MhqtUqSrFargoODbWr8/PzMudq1a8tqtZpjF9cUnaM4eXl5ysvLM/dzcnL+0rUCAAAAAABcT67pUKp///7mv8PCwtSiRQs1atRIGzZsULdu3cqxMyk+Pl5Tp04t1x4AAAAAAAAqqmv+63sXu+mmm1S3bl399NNPkiR/f38dP37cpub8+fM6efKk+Rwqf39/ZWZm2tQU7V+p5lLPspKkuLg4ZWdnm9uRI0f+2sUBAAAAAABcRypUKPXrr7/q999/V/369SVJ4eHhysrK0q5du8ya9evXq7CwUO3btzdrkpOTVVBQYNYkJSWpSZMmql27tlmzbt06m89KSkpSeHj4JXtxd3eXp6enzQYAAAAAAICSKddQKjc3V6mpqUpNTZUkpaenKzU1VRkZGcrNzdW4ceO0detWHT58WOvWrVPv3r3VuHFjRUZGSpJCQ0PVo0cPDR8+XNu3b9fmzZs1cuRI9e/fXwEBAZKkRx55RG5uboqOjtb+/fv18ccfa86cOYqNjTX7eOaZZ7R69WrNnDlTP/zwg6ZMmaKdO3dq5MiRTv+ZAAAAAAAAXA/KNZTauXOnWrdurdatW0uSYmNj1bp1a02ePFlVqlTRnj17dN999+mWW25RdHS02rZtq40bN8rd3d08x+LFixUSEqJu3bqpV69e6tixoxISEsx5Ly8vrV27Vunp6Wrbtq2effZZTZ48WTExMWZNhw4dtGTJEiUkJKhly5b69NNPtXz5cjVv3tx5PwwAAAAAAIDrSLk+6LxLly4yDOOS82vWrLniOXx8fLRkyZLL1rRo0UIbN268bE2/fv3Ur1+/K34eAAAAAAAA/roK9UwpAAAAAAAAVA6EUgAAAAAAAHA6QikAAAAAAAA4HaEUAAAAAAAAnI5QCgAAAAAAAE5HKAUAAAAAAACnI5QCAAAAAACA0xFKAQAAAAAAwOkIpQAAAAAAAOB0hFIAAAAAAABwOkIpAAAAAAAAOB2hFAAAAAAAAJyOUAoAAAAAAABORygFAAAAAAAApyOUAgAAAAAAgNMRSgEAAAAAAMDpCKUAAAAAAADgdIRSAAAAAAAAcDpCKQAAAAAAADgdoRQAAAAAAACcjlAKAAAAAAAATkcoBQAAAAAAAKcjlAIAAAAAAIDTEUoBAAAAAADA6QilAAAAAAAA4HSEUgAAAAAAAHA6QikAAAAAAAA4HaEUAAAAAAAAnI5QCgAAAAAAAE5HKAUAAAAAAACnI5QCAACoYJKTk3XvvfcqICBAFotFy5cvN+cKCgo0YcIEhYWFqUaNGgoICNDgwYN19OhRm3OcPHlSAwcOlKenp7y9vRUdHa3c3Fybmj179ujOO++Uh4eHAgMDNWPGDLteli5dqpCQEHl4eCgsLEyrVq0qk2sGAACVD6EUAABABXP69Gm1bNlS8+fPt5s7c+aMvv32W02aNEnffvutli1bpgMHDui+++6zqRs4cKD279+vpKQkrVixQsnJyYqJiTHnc3Jy1L17dwUFBWnXrl169dVXNWXKFCUkJJg1W7Zs0YABAxQdHa3du3erT58+6tOnj/bt21d2Fw8AACoN1/JuAAAAAI7p2bOnevbsWeycl5eXkpKSbMbmzZun2267TRkZGWrQoIHS0tK0evVq7dixQ+3atZMkzZ07V7169dJrr72mgIAALV68WPn5+Vq4cKHc3NzUrFkzpaamatasWWZ4NWfOHPXo0UPjxo2TJE2fPl1JSUmaN2+eFixYUIY/AQAAUBlwpxQAAEAll52dLYvFIm9vb0lSSkqKvL29zUBKkiIiIuTi4qJt27aZNZ06dZKbm5tZExkZqQMHDujUqVNmTUREhM1nRUZGKiUl5ZK95OXlKScnx2YDAADXJ0IpAACASuzcuXOaMGGCBgwYIE9PT0mS1WqVr6+vTZ2rq6t8fHxktVrNGj8/P5uaov0r1RTNFyc+Pl5eXl7mFhgY+NcuEAAAVFiEUgAAAJVUQUGBHnroIRmGobfeequ825EkxcXFKTs729yOHDlS3i0BAIBywjOlAAAAKqGiQOqXX37R+vXrzbukJMnf31/Hjx+3qT9//rxOnjwpf39/syYzM9Ompmj/SjVF88Vxd3eXu7v71V8YAACoNLhTCgAAoJIpCqQOHjyor776SnXq1LGZDw8PV1ZWlnbt2mWOrV+/XoWFhWrfvr1Zk5ycrIKCArMmKSlJTZo0Ue3atc2adevW2Zw7KSlJ4eHhZXVpAACgEiGUAgAAqGByc3OVmpqq1NRUSVJ6erpSU1OVkZGhgoICPfjgg9q5c6cWL16sCxcuyGq1ymq1Kj8/X5IUGhqqHj16aPjw4dq+fbs2b96skSNHqn///goICJAkPfLII3Jzc1N0dLT279+vjz/+WHPmzFFsbKzZxzPPPKPVq1dr5syZ+uGHHzRlyhTt3LlTI0eOdPrPBAAAVDyEUgAAABXMzp071bp1a7Vu3VqSFBsbq9atW2vy5Mn67bff9Pnnn+vXX39Vq1atVL9+fXPbsmWLeY7FixcrJCRE3bp1U69evdSxY0clJCSY815eXlq7dq3S09PVtm1bPfvss5o8ebJiYmLMmg4dOmjJkiVKSEhQy5Yt9emnn2r58uVq3ry5834YAACgwuKZUgAAABVMly5dZBjGJecvN1fEx8dHS5YsuWxNixYttHHjxsvW9OvXT/369bvi5wEAAPwZd0oBAAAAAADA6QilAAAAAAAA4HSEUgAAAAAAAHA6QikAAAAAAAA4HaEUAAAAAAAAnI5QCgAAAAAAAE5HKAUAAAAAAACnI5QCAAAAAACA0xFKAQAAAAAAwOkIpQAAAAAAAOB0hFIAAAAAAABwOkIpAAAAAAAAOB2hFAAAAAAAAJyOUAoAAAAAAABORygFAAAAAAAApyvXUCo5OVn33nuvAgICZLFYtHz5cnOuoKBAEyZMUFhYmGrUqKGAgAANHjxYR48etTlHw4YNZbFYbLaXX37ZpmbPnj2688475eHhocDAQM2YMcOul6VLlyokJEQeHh4KCwvTqlWryuSaAQAAAAAAUM6h1OnTp9WyZUvNnz/fbu7MmTP69ttvNWnSJH377bdatmyZDhw4oPvuu8+udtq0aTp27Ji5jRo1ypzLyclR9+7dFRQUpF27dunVV1/VlClTlJCQYNZs2bJFAwYMUHR0tHbv3q0+ffqoT58+2rdvX9lcOAAAAAAAwHXOtTw/vGfPnurZs2exc15eXkpKSrIZmzdvnm677TZlZGSoQYMG5nitWrXk7+9f7HkWL16s/Px8LVy4UG5ubmrWrJlSU1M1a9YsxcTESJLmzJmjHj16aNy4cZKk6dOnKykpSfPmzdOCBQtK41IBAAAAAABwkQr1TKns7GxZLBZ5e3vbjL/88suqU6eOWrdurVdffVXnz58351JSUtSpUye5ubmZY5GRkTpw4IBOnTpl1kRERNicMzIyUikpKWV3MQAAAAAAANexcr1TyhHnzp3ThAkTNGDAAHl6eprjo0ePVps2beTj46MtW7YoLi5Ox44d06xZsyRJVqtVwcHBNufy8/Mz52rXri2r1WqOXVxjtVov2U9eXp7y8vLM/ZycnL98jQAAAAAAANeLChFKFRQU6KGHHpJhGHrrrbds5mJjY81/t2jRQm5ubnriiScUHx8vd3f3MuspPj5eU6dOLbPzAwAAAAAAVGbX/Nf3igKpX375RUlJSTZ3SRWnffv2On/+vA4fPixJ8vf3V2Zmpk1N0X7Rc6guVXOp51RJUlxcnLKzs83tyJEjjl4aAAAAAADAdeuavlOqKJA6ePCgvv76a9WpU+eKx6SmpsrFxUW+vr6SpPDwcP39739XQUGBqlatKklKSkpSkyZNVLt2bbNm3bp1GjNmjHmepKQkhYeHX/Jz3N3dy/ROLAAAAFy/0tLSrvrYunXr2rwUCACAa1W5hlK5ubn66aefzP309HSlpqbKx8dH9evX14MPPqhvv/1WK1as0IULF8xnPPn4+MjNzU0pKSnatm2bunbtqlq1aiklJUVjx47VoEGDzMDpkUce0dSpUxUdHa0JEyZo3759mjNnjl5//XXzc5955hl17txZM2fOVFRUlD766CPt3LlTCQkJzv2BAAAA4Dp3TJKLBg0adNVn8PCorgMH0gimAADXvHINpXbu3KmuXbua+0XPhxoyZIimTJmizz//XJLUqlUrm+O+/vprdenSRe7u7vroo480ZcoU5eXlKTg4WGPHjrV5zpSXl5fWrl2rESNGqG3btqpbt64mT56smJgYs6ZDhw5asmSJJk6cqOeff14333yzli9frubNm5fh1QMAAAB/liWpUNIHkkKv4vg0nTs3SCdOnCCUAgBc88o1lOrSpYsMw7jk/OXmJKlNmzbaunXrFT+nRYsW2rhx42Vr+vXrp379+l3xXAAAAEDZC5XUprybAACgTF3zDzoHAAAAAABA5UMoBQAAAAAAAKcjlAIAAAAAAIDTEUoBAAAAAADA6QilAAAAAAAA4HSEUgAAAAAAAHA6QikAAAAAAAA4HaEUAAAAAAAAnI5QCgAAAAAAAE5HKAUAAAAAAACncy3vBuAcaWlpV31s3bp11aBBg1LsBgAAAAAAXO8IpSq5Y1mSi0UaNGjQVZ+jejUPpf1wgGAKAAAAAACUGodDqW+//VZVq1ZVWFiYJOnf//633n33XTVt2lRTpkyRm5tbqTeJq5d1Rio0pA+elkIDHD8+7ag06M1zOnHiBKEUAABXifUTAACAPYefKfXEE0/oxx9/lCT9/PPP6t+/v6pXr66lS5dq/Pjxpd4gSkdogNQm2PHtaoIsAABgi/UTAACAPYdDqR9//FGtWrWSJC1dulSdOnXSkiVLlJiYqH/961+l3R8AAECFx/oJAADAnsOhlGEYKiwslCR99dVX6tWrlyQpMDBQJ06cKN3uAAAAKgHWTwAAAPYcDqXatWunF198Ue+//76++eYbRUVFSZLS09Pl5+dX6g0CAABUdKyfAAAA7DkcSs2ePVvffvutRo4cqb///e9q3LixJOnTTz9Vhw4dSr1BAACAio71EwAAgD2H377XokUL7d2712781VdfVZUqVUqlKQAAgMqE9RMAAIA9h++UkqSsrCy9/fbbiouL08mTJyVJ33//vY4fP16qzQEAAFQWrJ8AAABsOXyn1J49e9StWzd5e3vr8OHDGj58uHx8fLRs2TJlZGTovffeK4s+AQAAKizWTwAAAPYcvlMqNjZWjz/+uA4ePCgPDw9zvFevXkpOTi7V5gAAACoD1k8AAAD2HA6lduzYoSeeeMJu/IYbbpDVai2VpgAAACoT1k8AAAD2HA6l3N3dlZOTYzf+448/ql69eqXSFAAAQGVS2uun5ORk3XvvvQoICJDFYtHy5ctt5g3D0OTJk1W/fn1Vq1ZNEREROnjwoE3NyZMnNXDgQHl6esrb21vR0dHKzc21qdmzZ4/uvPNOeXh4KDAwUDNmzLDrZenSpQoJCZGHh4fCwsK0atUqh68HAABcnxwOpe677z5NmzZNBQUFkiSLxaKMjAxNmDBBffv2LfUGAQAAKrrSXj+dPn1aLVu21Pz584udnzFjht544w0tWLBA27ZtU40aNRQZGalz586ZNQMHDtT+/fuVlJSkFStWKDk5WTExMeZ8Tk6OunfvrqCgIO3atUuvvvqqpkyZooSEBLNmy5YtGjBggKKjo7V792716dNHffr00b59+xy+JgAAcP1xOJSaOXOmcnNz5evrq7Nnz6pz585q3LixatWqpZdeeqksegQAAKjQSnv91LNnT7344ou6//777eYMw9Ds2bM1ceJE9e7dWy1atNB7772no0ePmndUpaWlafXq1Xr77bfVvn17dezYUXPnztVHH32ko0ePSpIWL16s/Px8LVy4UM2aNVP//v01evRozZo1y/ysOXPmqEePHho3bpxCQ0M1ffp0tWnTRvPmzbu6HxQAALiuOPz2PS8vLyUlJWnz5s367rvvlJubqzZt2igiIqIs+gMAAKjwnLl+Sk9Pl9VqtTm3l5eX2rdvr5SUFPXv318pKSny9vZWu3btzJqIiAi5uLho27Ztuv/++5WSkqJOnTrJzc3NrImMjNQrr7yiU6dOqXbt2kpJSVFsbKzN50dGRtp9nRAAAKA4DodSRe644w7dcccdpdkLAABApeaM9VPRg9P9/Pxsxv38/Mw5q9UqX19fm3lXV1f5+PjY1AQHB9udo2iudu3aslqtl/2c4uTl5SkvL8/cL+5ZWwAA4Prg8Nf3Ro8erTfeeMNufN68eRozZkxp9AQAAFCpsH76n/j4eHl5eZlbYGBgebcEAADKicOh1L/+9a9i/8LXoUMHffrpp6XSFAAAQGXizPWTv7+/JCkzM9NmPDMz05zz9/fX8ePHbebPnz+vkydP2tQUd46LP+NSNUXzxYmLi1N2dra5HTlyxNFLBAAAlYTDodTvv/8uLy8vu3FPT0+dOHGiVJoCAACoTJy5fgoODpa/v7/WrVtnjuXk5Gjbtm0KDw+XJIWHhysrK0u7du0ya9avX6/CwkK1b9/erElOTjbfGChJSUlJatKkiWrXrm3WXPw5RTVFn1Mcd3d3eXp62mwAAOD65HAo1bhxY61evdpu/Msvv9RNN91UKk0BAABUJqW9fsrNzVVqaqpSU1Ml/fFw89TUVGVkZMhisWjMmDF68cUX9fnnn2vv3r0aPHiwAgIC1KdPH0lSaGioevTooeHDh2v79u3avHmzRo4cqf79+ysgIECS9Mgjj8jNzU3R0dHav3+/Pv74Y82ZM8fmwebPPPOMVq9erZkzZ+qHH37QlClTtHPnTo0cOdLxHxIAALjuOPyg89jYWI0cOVL/+c9/dNddd0mS1q1bp5kzZ2r27Nml3R8AAECFV9rrp507d6pr164255ekIUOGKDExUePHj9fp06cVExOjrKwsdezYUatXr5aHh4d5zOLFizVy5Eh169ZNLi4u6tu3r81zr7y8vLR27VqNGDFCbdu2Vd26dTV58mTFxMSYNR06dNCSJUs0ceJEPf/887r55pu1fPlyNW/e3OFrAgAA1x+HQ6mhQ4cqLy9PL730kqZPny5Jatiwod566y0NHjy41BsEAACo6Ep7/dSlSxcZhnHJeYvFomnTpmnatGmXrPHx8dGSJUsu+zktWrTQxo0bL1vTr18/9evX7/INAwAAFMPhUEqSnnrqKT311FP6z3/+o2rVqqlmzZql3RcAAEClwvoJAADA1lWFUkXq1atXWn0AAABcF1g/AQAA/MHhB51nZmbq0UcfVUBAgFxdXVWlShWbDQAAALZYPwEAANhz+E6pxx57TBkZGZo0aZLq168vi8VSFn0BAABUGqyfAAAA7DkcSm3atEkbN25Uq1atyqAdAACAyof1EwAAgD2Hv74XGBh42be9AAAAwBbrJwAAAHsOh1KzZ8/Wc889p8OHD5dBOwAAAJUP6ycAAAB7Dn997+GHH9aZM2fUqFEjVa9eXVWrVrWZP3nyZKk1BwAAUBmwfgIAALDncCg1e/bsMmgDAACg8mL9BAAAYM/hUGrIkCFl0QcAAEClxfoJAADAnsPPlJKkQ4cOaeLEiRowYICOHz8uSfryyy+1f//+Um0OAACgsmD9BAAAYMvhUOqbb75RWFiYtm3bpmXLlik3N1eS9N133+mFF14o9QYBAAAqOtZPAAAA9hwOpZ577jm9+OKLSkpKkpubmzl+1113aevWraXaHAAAQGXA+gkAAMCew6HU3r17df/999uN+/r66sSJE6XSFAAAQGXC+gkAAMCew6GUt7e3jh07Zje+e/du3XDDDaXSFAAAQGXC+gkAAMCew6FU//79NWHCBFmtVlksFhUWFmrz5s3629/+psGDB5dFjwAAABUa6ycAAAB7DodS//jHPxQSEqLAwEDl5uaqadOm6tSpkzp06KCJEyeWRY8AAAAVGusnAAAAe66OHuDm5qZ//vOfmjRpkvbt26fc3Fy1bt1aN998c1n0BwAAUOGxfgIAALDncChVpEGDBmrQoEFp9gIAAFCpsX4CAAD4H4dDqaFDh152fuHChVfdDAAAQGXE+gkAAMCew6HUqVOnbPYLCgq0b98+ZWVl6a677iq1xgAAACoL1k8AAAD2HA6lPvvsM7uxwsJCPfXUU2rUqFGpNAUAAFCZsH4CAACw5/Db94o9iYuLYmNj9frrrzt0XHJysu69914FBATIYrFo+fLlNvOGYWjy5MmqX7++qlWrpoiICB08eNCm5uTJkxo4cKA8PT3l7e2t6Oho5ebm2tTs2bNHd955pzw8PBQYGKgZM2bY9bJ06VKFhITIw8NDYWFhWrVqlUPXAgAA4IirXT8BAABUFqUSSknSoUOHdP78eYeOOX36tFq2bKn58+cXOz9jxgy98cYbWrBggbZt26YaNWooMjJS586dM2sGDhyo/fv3KykpSStWrFBycrJiYmLM+ZycHHXv3l1BQUHatWuXXn31VU2ZMkUJCQlmzZYtWzRgwABFR0dr9+7d6tOnj/r06aN9+/Y5+FMAAAAouatZPwEAAFQWDn99LzY21mbfMAwdO3ZMK1eu1JAhQxw6V8+ePdWzZ89i5wzD0OzZszVx4kT17t1bkvTee+/Jz89Py5cvV//+/ZWWlqbVq1drx44dateunSRp7ty56tWrl1577TUFBARo8eLFys/P18KFC+Xm5qZmzZopNTVVs2bNMsOrOXPmqEePHho3bpwkafr06UpKStK8efO0YMECh64JAADgz0pz/QQAAFBZOBxK7d6922bfxcVF9erV08yZM6/4ZhlHpKeny2q1KiIiwhzz8vJS+/btlZKSov79+yslJUXe3t5mICVJERERcnFx0bZt23T//fcrJSVFnTp1kpubm1kTGRmpV155RadOnVLt2rWVkpJit1iMjIy0+zohAADA1XDW+gkAAKAicTiU+vrrr8uiDztWq1WS5OfnZzPu5+dnzlmtVvn6+trMu7q6ysfHx6YmODjY7hxFc7Vr15bVar3s5xQnLy9PeXl55n5OTo4jlwcAAK4jzlo/AQAAVCSl9kyp6018fLy8vLzMLTAwsLxbAgAAAAAAqDAcvlOqdevWslgsJar99ttvHW6oiL+/vyQpMzNT9evXN8czMzPVqlUrs+b48eM2x50/f14nT540j/f391dmZqZNTdH+lWqK5osTFxdn85W/nJwcgikAAFAsZ62fAAAAKhKH75Tq0aOHDh06JHd3d3Xp0kVdunSRh4eHDh06pO7du6t3797m9lcEBwfL399f69atM8dycnK0bds2hYeHS5LCw8OVlZWlXbt2mTXr169XYWGh2rdvb9YkJyeroKDArElKSlKTJk1Uu3Zts+bizymqKfqc4ri7u8vT09NmAwAAKI6z1k8AAAAVicN3Sv3nP//R6NGjNX36dJvxF154QUeOHNHChQtLfK7c3Fz99NNP5n56erpSU1Pl4+OjBg0aaMyYMXrxxRd18803Kzg4WJMmTVJAQID69OkjSQoNDVWPHj00fPhwLViwQAUFBRo5cqT69++vgIAASdIjjzyiqVOnKjo6WhMmTNC+ffs0Z84cvf766+bnPvPMM+rcubNmzpypqKgoffTRR9q5c6cSEhIc/fEAAADYKc31EwAAQGXh8J1SS5cu1eDBg+3GBw0apH/9618OnWvnzp1q3bq1WrduLemP1yW3bt1akydPliSNHz9eo0aNUkxMjG699Vbl5uZq9erV8vDwMM+xePFihYSEqFu3burVq5c6duxoEyZ5eXlp7dq1Sk9PV9u2bfXss89q8uTJiomJMWs6dOigJUuWKCEhQS1bttSnn36q5cuXq3nz5g5dDwAAQHFKc/0EAABQWTh8p1S1atW0efNm3XzzzTbjmzdvtgmLSqJLly4yDOOS8xaLRdOmTdO0adMuWePj46MlS5Zc9nNatGihjRs3XramX79+6tev3+UbBgAAuAqluX4CAACoLBwOpcaMGaOnnnpK3377rW677TZJ0rZt27Rw4UJNmjSp1BsEAACo6Fg/AQAA2HM4lHruued00003ac6cOfrggw8k/fFsp3fffVcPPfRQqTcIAABQ0bF+AgAAsOdwKCVJDz30EAsoAAAAB7B+AgAAsOXwg84lKSsrS2+//baef/55nTx5UpL07bff6rfffivV5gAAACoL1k8AAAC2HL5Tas+ePYqIiJCXl5cOHz6sYcOGycfHR8uWLVNGRobee++9sugTAACgwmL9BAAAYM/hO6ViY2P12GOP6eDBgzZvi+nVq5eSk5NLtTkAAIDKgPUTAACAPYdDqR07duiJJ56wG7/hhhtktVpLpSkAAIDKhPUTAACAPYdDKXd3d+Xk5NiN//jjj6pXr16pNAUAAFCZsH4CAACw53Aodd9992natGkqKCiQJFksFmVkZGjChAnq27dvqTcIAABQ0bF+AgAAsOdwKDVz5kzl5ubK19dXZ8+eVefOndW4cWPVqlVLL730Uln0CAAAUKGxfgIAALDn8Nv3vLy8lJSUpM2bN+u7775Tbm6u2rRpo4iIiLLoDwAAoMJj/QQAAGDP4VCqyB133KE77rijNHsBAACo1Fg/AQAA/E+Jv76XkpKiFStW2Iy99957Cg4Olq+vr2JiYpSXl1fqDQIAAFRUrJ8AAAAurcSh1LRp07R//35zf+/evYqOjlZERISee+45ffHFF4qPjy+TJgEAACoi1k8AAACXVuJQKjU1Vd26dTP3P/roI7Vv317//Oc/FRsbqzfeeEOffPJJmTQJAABQEbF+AgAAuLQSh1KnTp2Sn5+fuf/NN9+oZ8+e5v6tt96qI0eOlG53AAAAFRjrJwAAgEsrcSjl5+en9PR0SVJ+fr6+/fZb3X777eb8f//7X1WtWrX0OwQAAKigymv9dOHCBU2aNEnBwcGqVq2aGjVqpOnTp8swDLPGMAxNnjxZ9evXV7Vq1RQREaGDBw/anOfkyZMaOHCgPD095e3trejoaOXm5trU7NmzR3feeac8PDwUGBioGTNmlPr1AACAyqnEoVSvXr303HPPaePGjYqLi1P16tV15513mvN79uxRo0aNyqRJAACAiqi81k+vvPKK3nrrLc2bN09paWl65ZVXNGPGDM2dO9esmTFjht544w0tWLBA27ZtU40aNRQZGalz586ZNQMHDtT+/fuVlJSkFStWKDk5WTExMeZ8Tk6OunfvrqCgIO3atUuvvvqqpkyZooSEhFK/JgAAUPm4lrRw+vTpeuCBB9S5c2fVrFlTixYtkpubmzm/cOFCde/evUyaBAAAqIjKa/20ZcsW9e7dW1FRUZKkhg0b6sMPP9T27dsl/XGX1OzZszVx4kT17t1b0h9vBfTz89Py5cvVv39/paWlafXq1dqxY4fatWsnSZo7d6569eql1157TQEBAVq8eLHy8/O1cOFCubm5qVmzZkpNTdWsWbNswisAAIDilDiUqlu3rpKTk5Wdna2aNWuqSpUqNvNLly5VzZo1S71BAACAiqq81k8dOnRQQkKCfvzxR91yyy367rvvtGnTJs2aNUuSlJ6eLqvVqoiICPMYLy8vtW/fXikpKerfv79SUlLk7e1tBlKSFBERIRcXF23btk3333+/UlJS1KlTJ5ugLTIyUq+88opOnTql2rVrl/q1AQCAyqPEoVQRLy+vYsd9fHz+cjMAAACVkbPXT88995xycnIUEhKiKlWq6MKFC3rppZc0cOBASZLVapUkm4ewF+0XzVmtVvn6+trMu7q6ysfHx6YmODjY7hxFc8WFUnl5ecrLyzP3c3Jy/sqlAgCACqzEz5QCAABAxfDJJ59o8eLFWrJkib799lstWrRIr732mhYtWlTerSk+Pl5eXl7mFhgYWN4tAQCAckIoBQAAUMmMGzdOzz33nPr376+wsDA9+uijGjt2rOLj4yVJ/v7+kqTMzEyb4zIzM805f39/HT9+3Gb+/PnzOnnypE1Ncee4+DP+LC4uTtnZ2eZ25MiRv3i1AACgoiKUAgAAqGTOnDkjFxfbZV6VKlVUWFgoSQoODpa/v7/WrVtnzufk5Gjbtm0KDw+XJIWHhysrK0u7du0ya9avX6/CwkK1b9/erElOTlZBQYFZk5SUpCZNmlzyeVLu7u7y9PS02QAAwPWpRKFUmzZtdOrUKUnStGnTdObMmTJtCgAAoKIrz/XTvffeq5deekkrV67U4cOH9dlnn2nWrFm6//77JUkWi0VjxozRiy++qM8//1x79+7V4MGDFRAQoD59+kiSQkND1aNHDw0fPlzbt2/X5s2bNXLkSPXv318BAQGSpEceeURubm6Kjo7W/v379fHHH2vOnDmKjY112rUCAICKq0ShVFpamk6fPi1Jmjp1qnJzc8u0KQAAgIquPNdPc+fO1YMPPqinn35aoaGh+tvf/qYnnnhC06dPN2vGjx+vUaNGKSYmRrfeeqtyc3O1evVqeXh4mDWLFy9WSEiIunXrpl69eqljx45KSEgw5728vLR27Vqlp6erbdu2evbZZzV58mTFxMQ47VoBAEDFVaK377Vq1UqPP/64OnbsKMMw9Nprr13y9cWTJ08u1QYBAAAqovJcP9WqVUuzZ8/W7NmzL1ljsVg0bdo0TZs27ZI1Pj4+WrJkyWU/q0WLFtq4cePVtgoAAK5jJQqlEhMT9cILL2jFihWyWCz68ssv5epqf6jFYiGUAgAAEOsnAACAKylRKNWkSRN99NFHkiQXFxetW7dOvr6+ZdoYAABARcb6CQAA4PJKFEpdrOitLQAAACgZ1k8AAAD2HA6lJOnQoUOaPXu20tLSJElNmzbVM888o0aNGpVqcwAAAJUF6ycAAABbJXr73sXWrFmjpk2bavv27WrRooVatGihbdu2qVmzZkpKSiqLHgEAACo01k8AAAD2HL5T6rnnntPYsWP18ssv241PmDBBd999d6k1BwAAUBmwfgIAALDn8J1SaWlpio6OthsfOnSovv/++1JpCgAAoDJh/QQAAGDP4VCqXr16Sk1NtRtPTU3ljTIAAADFYP0EAABgz+Gv7w0fPlwxMTH6+eef1aFDB0nS5s2b9corryg2NrbUGwQAAKjoWD8BAADYcziUmjRpkmrVqqWZM2cqLi5OkhQQEKApU6Zo9OjRpd4gAABARcf6CQAAwJ7DoZTFYtHYsWM1duxY/fe//5Uk1apVq9QbAwAAqCxYPwEAANhzOJS6GIspAAAAx7B+AgAA+IPDDzoHAAAAAAAA/ipCKQAAAAAAADgdoRQAAAAAAACczqFQqqCgQN26ddPBgwfLqh8AAIBKhfUTAABA8RwKpapWrao9e/aUVS8AAACVDusnAACA4jn89b1BgwbpnXfeKYteAAAAKiXWTwAAAPZcHT3g/PnzWrhwob766iu1bdtWNWrUsJmfNWtWqTUHAABQGbB+AgAAsOdwKLVv3z61adNGkvTjjz/azFksltLpCgAAoBJh/QQAAGDP4VDq66+/Los+AAAAKi3WTwAAAPYcfqZUkZ9++klr1qzR2bNnJUmGYZRaUwAAAJUR6ycAAID/cTiU+v3339WtWzfdcsst6tWrl44dOyZJio6O1rPPPlvqDQIAAFR0rJ8AAADsORxKjR07VlWrVlVGRoaqV69ujj/88MNavXp1qTYHAABQGbB+AgAAsOfwM6XWrl2rNWvW6MYbb7QZv/nmm/XLL7+UWmMAAACVBesnAAAAew7fKXX69Gmbv/AVOXnypNzd3UulKQAAgMqE9RMAAIA9h0OpO++8U++99565b7FYVFhYqBkzZqhr166l2hwAAEBlwPoJAADAnsNf35sxY4a6deumnTt3Kj8/X+PHj9f+/ft18uRJbd68uSx6BAAAqNBYPwEAANhz+E6p5s2b68cff1THjh3Vu3dvnT59Wg888IB2796tRo0alUWPAAAAFRrrJwAAAHsOh1KS5OXlpb///e/65JNPtGrVKr344ouqX79+afcmSWrYsKEsFovdNmLECElSly5d7OaefPJJm3NkZGQoKipK1atXl6+vr8aNG6fz58/b1GzYsEFt2rSRu7u7GjdurMTExDK5HgAAcH1y5voJAACgInD463uSdOrUKb3zzjtKS0uTJDVt2lSPP/64fHx8SrU5SdqxY4cuXLhg7u/bt0933323+vXrZ44NHz5c06ZNM/cvfpDohQsXFBUVJX9/f23ZskXHjh3T4MGDVbVqVf3jH/+QJKWnpysqKkpPPvmkFi9erHXr1mnYsGGqX7++IiMjS/2aAADA9ceZ6ycAAICKwOE7pZKTk9WwYUO98cYbOnXqlE6dOqU33nhDwcHBSk5OLvUG69WrJ39/f3NbsWKFGjVqpM6dO5s11atXt6nx9PQ059auXavvv/9eH3zwgVq1aqWePXtq+vTpmj9/vvLz8yVJCxYsUHBwsGbOnKnQ0FCNHDlSDz74oF5//fVSvx4AAHD9cfb6CQAAoCJwOJQaMWKEHn74YaWnp2vZsmVatmyZfv75Z/Xv39/8Sl1Zyc/P1wcffKChQ4fKYrGY44sXL1bdunXVvHlzxcXF6cyZM+ZcSkqKwsLC5OfnZ45FRkYqJydH+/fvN2siIiJsPisyMlIpKSllej0AAOD6UJ7rJwAAgGuVw1/f++mnn/Tpp5+qSpUq5liVKlUUGxtr86rjsrB8+XJlZWXpscceM8ceeeQRBQUFKSAgQHv27NGECRN04MABLVu2TJJktVptAilJ5r7Var1sTU5Ojs6ePatq1arZ9ZKXl6e8vDxzPycnp1SuEQAAVD7luX4CAAC4VjkcSrVp00ZpaWlq0qSJzXhaWppatmxZao0V55133lHPnj0VEBBgjsXExJj/DgsLU/369dWtWzcdOnSoTN9mEx8fr6lTp5bZ+QEAQOVRnusnAACAa1WJQqk9e/aY/x49erSeeeYZ/fTTT7r99tslSVu3btX8+fP18ssvl02Xkn755Rd99dVX5h1Ql9K+fXtJf/xFslGjRvL399f27dttajIzMyVJ/v7+5v9bNHZxjaenZ7F3SUlSXFycYmNjzf2cnBwFBgY6dlEAAKDSuhbWTwAAANeyEoVSrVq1ksVikWEY5tj48ePt6h555BE9/PDDpdfdRd599135+voqKirqsnWpqamSZL5iOTw8XC+99JKOHz8uX19fSVJSUpI8PT3VtGlTs2bVqlU250lKSlJ4ePglP8fd3V3u7u5XezkAAKCSuxbWTwAAANeyEoVS6enpZd3HZRUWFurdd9/VkCFD5Or6v5YPHTqkJUuWqFevXqpTp4727NmjsWPHqlOnTmrRooUkqXv37mratKkeffRRzZgxQ1arVRMnTtSIESPMUOnJJ5/UvHnzNH78eA0dOlTr16/XJ598opUrV5bL9QIAgIqvvNdPAAAA17oShVJBQUFl3cdlffXVV8rIyNDQoUNtxt3c3PTVV19p9uzZOn36tAIDA9W3b19NnDjRrKlSpYpWrFihp556SuHh4apRo4aGDBmiadOmmTXBwcFauXKlxo4dqzlz5ujGG2/U22+/rcjISKddIwAAqFzKe/0EAABwrXP4QeeSdPToUW3atEnHjx9XYWGhzdzo0aNLpbGLde/e3ebW9yKBgYH65ptvrnh8UFCQ3dfz/qxLly7avXv3VfcIAABwOc5ePwEAAFzrHA6lEhMT9cQTT8jNzU116tSRxWIx5ywWC4sqAACAP2H9BAAAYM/hUGrSpEmaPHmy4uLi5OLiUhY9AQAAVCqsnwAAAOw5vCo6c+aM+vfvz4IKAACghFg/AQAA2HN4ZRQdHa2lS5eWRS8AAACVEusnAAAAew5/fS8+Pl733HOPVq9erbCwMFWtWtVmftasWaXWHAAAQGXA+gkAAMDeVYVSa9asUZMmTSTJ7kGdAAAAsMX6CQAAwJ7DodTMmTO1cOFCPfbYY2XQDgAAQOXD+gkAAMCew8+Ucnd31x133FEWvQAAAFRKrJ8AAADsORxKPfPMM5o7d25Z9AIAAFApsX4CAACw5/DX97Zv367169drxYoVatasmd2DOpctW1ZqzQEAAFQGrJ8AAADsOXynlLe3tx544AF17txZdevWlZeXl80GAAAAW+Wxfvrtt980aNAg1alTR9WqVVNYWJh27txpzhuGocmTJ6t+/fqqVq2aIiIidPDgQZtznDx5UgMHDpSnp6e8vb0VHR2t3Nxcm5o9e/bozjvvlIeHhwIDAzVjxowyuR4AAFD5OHyn1LvvvlsWfQAAAFRazl4/nTp1SnfccYe6du2qL7/8UvXq1dPBgwdVu3Zts2bGjBl64403tGjRIgUHB2vSpEmKjIzU999/Lw8PD0nSwIEDdezYMSUlJamgoECPP/64YmJitGTJEklSTk6OunfvroiICC1YsEB79+7V0KFD5e3trZiYGKdeMwAAqHgcDqUAAABwbXvllVcUGBhoE4YFBweb/zYMQ7Nnz9bEiRPVu3dvSdJ7770nPz8/LV++XP3791daWppWr16tHTt2qF27dpKkuXPnqlevXnrttdcUEBCgxYsXKz8/XwsXLpSbm5uaNWum1NRUzZo1i1AKAABckcOhVHBwsCwWyyXnf/7557/UEAAAQGXj7PXT559/rsjISPXr10/ffPONbrjhBj399NMaPny4JCk9PV1Wq1URERHmMV5eXmrfvr1SUlLUv39/paSkyNvb2wykJCkiIkIuLi7atm2b7r//fqWkpKhTp05yc3MzayIjI/XKK6/o1KlTNndmFcnLy1NeXp65n5OTU6rXDgAAKg6HQ6kxY8bY7BcUFGj37t1avXq1xo0bV1p9AQAAVBrOXj/9/PPPeuuttxQbG6vnn39eO3bs0OjRo+Xm5qYhQ4bIarVKkvz8/GyO8/PzM+esVqt8fX1t5l1dXeXj42NTc/EdWBef02q1FhtKxcfHa+rUqaVzoQAAoEJzOJR65plnih2fP3++zcMzAQAA8Adnr58KCwvVrl07/eMf/5AktW7dWvv27dOCBQs0ZMiQUv88R8TFxSk2Ntbcz8nJUWBgYDl2BAAAyovDb9+7lJ49e+pf//pXaZ0OAACg0iur9VP9+vXVtGlTm7HQ0FBlZGRIkvz9/SVJmZmZNjWZmZnmnL+/v44fP24zf/78eZ08edKmprhzXPwZf+bu7i5PT0+bDQAAXJ9KLZT69NNP5ePjU1qnAwAAqPTKav10xx136MCBAzZjP/74o4KCgiT98Ywrf39/rVu3zpzPycnRtm3bFB4eLkkKDw9XVlaWdu3aZdasX79ehYWFat++vVmTnJysgoICsyYpKUlNmjQp9qt7AAAAF3P463utW7e2eVCnYRiyWq36z3/+ozfffLNUmwMAAKgMnL1+Gjt2rDp06KB//OMfeuihh7R9+3YlJCQoISFBkmSxWDRmzBi9+OKLuvnmmxUcHKxJkyYpICBAffr0kfTHnVU9evTQ8OHDtWDBAhUUFGjkyJHq37+/AgICJEmPPPKIpk6dqujoaE2YMEH79u3TnDlz9Prrr5f6NQEAgMrH4VCqaKFSxMXFRfXq1VOXLl0UEhJSWn0BAABUGs5eP91666367LPPFBcXp2nTpik4OFizZ8/WwIEDzZrx48fr9OnTiomJUVZWljp27KjVq1fLw8PDrFm8eLFGjhypbt26ycXFRX379tUbb7xhznt5eWnt2rUaMWKE2rZtq7p162ry5MmKiYkp9WsCAACVj8Oh1AsvvFAWfQAAAFRa5bF+uueee3TPPfdcct5isWjatGmaNm3aJWt8fHy0ZMmSy35OixYttHHjxqvuEwAAXL9K7ZlSAAAAAAAAQEmV+E4pFxcXm2chFMdisej8+fN/uSkAAIDKgPUTAADApZU4lPrss88uOZeSkqI33nhDhYWFpdIUAABAZcD6CQAA4NJKHEr17t3bbuzAgQN67rnn9MUXX2jgwIGXfSYBAADA9Yb1EwAAwKVd1TOljh49quHDhyssLEznz59XamqqFi1apKCgoNLuDwAAoFJg/QQAAGDLoVAqOztbEyZMUOPGjbV//36tW7dOX3zxhZo3b15W/QEAAFRorJ8AAACKV+Kv782YMUOvvPKK/P399eGHHxZ7OzoAAAD+h/UTAADApZU4lHruuedUrVo1NW7cWIsWLdKiRYuKrVu2bFmpNQcAAFCRsX4CAAC4tBKHUoMHD77iK40BAADwP6yfAAAALq3EoVRiYmIZtgEAAFD5sH4CAAC4tKt6+x4AAAAAAADwVxBKAQAAAAAAwOkIpQAAAAAAAOB0hFIAAAAAAABwOkIpAAAAAAAAOB2hFAAAAAAAAJyOUAoAAAAAAABORygFAAAAAAAApyOUAgAAAAAAgNMRSgEAAAAAAMDpCKUAAAAAAADgdIRSAAAAAAAAcDpCKQAAAAAAADgdoRQAAAAAAACcjlAKAAAAAAAATkcoBQAAAAAAAKcjlAIAAAAAAIDTEUoBAAAAAADA6QilAAAAAAAA4HSEUgAAAAAAAHA6QikAAAAAAAA4HaEUAAAAAAAAnI5QCgAAAAAAAE5HKAUAAAAAAACnI5QCAAAAAACA013TodSUKVNksVhstpCQEHP+3LlzGjFihOrUqaOaNWuqb9++yszMtDlHRkaGoqKiVL16dfn6+mrcuHE6f/68Tc2GDRvUpk0bubu7q3HjxkpMTHTG5QEAAAAAAFy3rulQSpKaNWumY8eOmdumTZvMubFjx+qLL77Q0qVL9c033+jo0aN64IEHzPkLFy4oKipK+fn52rJlixYtWqTExERNnjzZrElPT1dUVJS6du2q1NRUjRkzRsOGDdOaNWucep0AAAAAAADXE9fybuBKXF1d5e/vbzeenZ2td955R0uWLNFdd90lSXr33XcVGhqqrVu36vbbb9fatWv1/fff66uvvpKfn59atWql6dOna8KECZoyZYrc3Ny0YMECBQcHa+bMmZKk0NBQbdq0Sa+//roiIyOdeq0AAAAAAADXi2v+TqmDBw8qICBAN910kwYOHKiMjAxJ0q5du1RQUKCIiAizNiQkRA0aNFBKSookKSUlRWFhYfLz8zNrIiMjlZOTo/3795s1F5+jqKboHAAAAAAAACh91/SdUu3bt1diYqKaNGmiY8eOaerUqbrzzju1b98+Wa1Wubm5ydvb2+YYPz8/Wa1WSZLVarUJpIrmi+YuV5OTk6OzZ8+qWrVqxfaWl5envLw8cz8nJ+cvXSsAAAAAAMD15JoOpXr27Gn+u0WLFmrfvr2CgoL0ySefXDIscpb4+HhNnTq1XHsAAABA2cnIyNCJEyccPi4tLa0MugEAoPK5pkOpP/P29tYtt9yin376SXfffbfy8/OVlZVlc7dUZmam+Qwqf39/bd++3eYcRW/nu7jmz2/sy8zMlKen52WDr7i4OMXGxpr7OTk5CgwM/EvXBwAAgGtDRkaGmjQJ1blzZ8q7FQAAKq1r/plSF8vNzdWhQ4dUv359tW3bVlWrVtW6devM+QMHDigjI0Ph4eGSpPDwcO3du1fHjx83a5KSkuTp6ammTZuaNRefo6im6ByX4u7uLk9PT5sNAAAAlcOJEyf+fyD1gaRdDm7Ty6NlAAAqnGv6Tqm//e1vuvfeexUUFKSjR4/qhRdeUJUqVTRgwAB5eXkpOjpasbGx8vHxkaenp0aNGqXw8HDdfvvtkqTu3buradOmevTRRzVjxgxZrVZNnDhRI0aMkLu7uyTpySef1Lx58zR+/HgNHTpU69ev1yeffKKVK1eW56UDAADgmhAqqY2Dx/D1PQAASuKaDqV+/fVXDRgwQL///rvq1aunjh07auvWrapXr54k6fXXX5eLi4v69u2rvLw8RUZG6s033zSPr1KlilasWKGnnnpK4eHhqlGjhoYMGaJp06aZNcHBwVq5cqXGjh2rOXPm6MYbb9Tbb7+tyMhIp18vAAAAAADA9eKaDqU++uijy857eHho/vz5mj9//iVrgoKCtGrVqsuep0uXLtq9e/dV9QgAAAAAAADHXdOhFAAAAADHXe0bAOvWrasGDRqUcjcAABSPUAoAAACoNI5JctGgQYOu6mgPj+o6cCCNYAoA4BSEUgAAAEClkSWpUH+8NTDUwWPTdO7cIJ04cYJQCgDgFC7l3QAAAADK1ssvvyyLxaIxY8aYY+fOndOIESNUp04d1axZU3379lVmZqbNcRkZGYqKilL16tXl6+urcePG6fz58zY1GzZsUJs2beTu7q7GjRsrMTHRCVeEKyt6a6Ajm6MhFgAAfw2hFAAAQCW2Y8cO/d///Z9atGhhMz527Fh98cUXWrp0qb755hsdPXpUDzzwgDl/4cIFRUVFKT8/X1u2bNGiRYuUmJioyZMnmzXp6emKiopS165dlZqaqjFjxmjYsGFas2aN064PAABUXIRSAAAAlVRubq4GDhyof/7zn6pdu7Y5np2drXfeeUezZs3SXXfdpbZt2+rdd9/Vli1btHXrVknS2rVr9f333+uDDz5Qq1at1LNnT02fPl3z589Xfn6+JGnBggUKDg7WzJkzFRoaqpEjR+rBBx/U66+/Xi7XCwAAKhZCKQAAgEpqxIgRioqKUkREhM34rl27VFBQYDMeEhKiBg0aKCUlRZKUkpKisLAw+fn5mTWRkZHKycnR/v37zZo/nzsyMtI8R3Hy8vKUk5NjswEAgOsTDzoHAACohD766CN9++232rFjh92c1WqVm5ubvL29bcb9/PxktVrNmosDqaL5ornL1eTk5Ojs2bOqVq2a3WfHx8dr6tSpV31dAACg8uBOKQAAgErmyJEjeuaZZ7R48WJ5eHiUdzs24uLilJ2dbW5Hjhwp75YAAEA5IZQCAACoZHbt2qXjx4+rTZs2cnV1laurq7755hu98cYbcnV1lZ+fn/Lz85WVlWVzXGZmpvz9/SVJ/v7+dm/jK9q/Uo2np2exd0lJkru7uzw9PW02AABwfSKUAgAAqGS6deumvXv3KjU11dzatWungQMHmv+uWrWq1q1bZx5z4MABZWRkKDw8XJIUHh6uvXv36vjx42ZNUlKSPD091bRpU7Pm4nMU1RSdAwAA4HJ4phRKJC0t7aqOq1u3rho0aFDK3QAAgMupVauWmjdvbjNWo0YN1alTxxyPjo5WbGysfHx85OnpqVGjRik8PFy33367JKl79+5q2rSpHn30Uc2YMUNWq1UTJ07UiBEj5O7uLkl68sknNW/ePI0fP15Dhw7V+vXr9cknn2jlypXOvWAAAFAhEUrhso5lSS4WadCgQVd1fPVqHkr74QDBFAAA15jXX39dLi4u6tu3r/Ly8hQZGak333zTnK9SpYpWrFihp556SuHh4apRo4aGDBmiadOmmTXBwcFauXKlxo4dqzlz5ujGG2/U22+/rcjIyPK4JAAAUMEQSuGyss5IhYb0wdNSaIBjx6YdlQa9eU4nTpwglAIAoJxt2LDBZt/Dw0Pz58/X/PnzL3lMUFCQVq1addnzdunSRbt37y6NFgEAwHWGUAolEhogtQku7y4AAAAAAEBlwYPOAQAAAAAA4HSEUgAAAAAAAHA6QikAAAAAAAA4HaEUAAAAAAAAnI5QCgAAAAAAAE5HKAUAAAAAAACnI5QCAAAAAACA0xFKAQAAAAAAwOkIpQAAAAAAAOB0hFIAAAAAAABwOkIpAAAAAAAAOB2hFAAAAAAAAJyOUAoAAAAAAABORygFAAAAAAAApyOUAgAAAAAAgNMRSgEAAAAAAMDpCKUAAAAAAADgdIRSAAAAAAAAcDpCKQAAAAAAADgdoRQAAAAAAACcjlAKAAAAAAAATkcoBQAAAAAAAKcjlAIAAAAAAIDTEUoBAAAAAADA6QilAAAAAAAA4HSEUgAAAAAAAHA6QikAAAAAAAA4HaEUAAAAAAAAnI5QCgAAAAAAAE5HKAUAAAAAAACnI5QCAAAAAACA0xFKAQAAAAAAwOkIpQAAAAAAAOB0hFIAAAAAAABwOkIpAAAAAAAAOB2hFAAAAAAAAJyOUAoAAAAAAABORygFAAAAAAAApyOUAgAAAAAAgNMRSgEAAAAAAMDpCKUAAAAAAADgdNd0KBUfH69bb71VtWrVkq+vr/r06aMDBw7Y1HTp0kUWi8Vme/LJJ21qMjIyFBUVperVq8vX11fjxo3T+fPnbWo2bNigNm3ayN3dXY0bN1ZiYmJZXx4AAAAAAMB165oOpb755huNGDFCW7duVVJSkgoKCtS9e3edPn3apm748OE6duyYuc2YMcOcu3DhgqKiopSfn68tW7Zo0aJFSkxM1OTJk82a9PR0RUVFqWvXrkpNTdWYMWM0bNgwrVmzxmnXCgAAAAAAcD1xLe8GLmf16tU2+4mJifL19dWuXbvUqVMnc7x69ery9/cv9hxr167V999/r6+++kp+fn5q1aqVpk+frgkTJmjKlClyc3PTggULFBwcrJkzZ0qSQkNDtWnTJr3++uuKjIwsuwsEAAAAAAC4Tl3Td0r9WXZ2tiTJx8fHZnzx4sWqW7eumjdvrri4OJ05c8acS0lJUVhYmPz8/MyxyMhI5eTkaP/+/WZNRESEzTkjIyOVkpJSVpcCAAAAAABwXbum75S6WGFhocaMGaM77rhDzZs3N8cfeeQRBQUFKSAgQHv27NGECRN04MABLVu2TJJktVptAilJ5r7Var1sTU5Ojs6ePatq1arZ9ZOXl6e8vDxzPycnp3QuFAAAAAAA4DpQYUKpESNGaN++fdq0aZPNeExMjPnvsLAw1a9fX926ddOhQ4fUqFGjMusnPj5eU6dOLbPzAwAAAAAAVGYV4ut7I0eO1IoVK/T111/rxhtvvGxt+/btJUk//fSTJMnf31+ZmZk2NUX7Rc+hulSNp6dnsXdJSVJcXJyys7PN7ciRI45fGAAAAAAAwHXqmg6lDMPQyJEj9dlnn2n9+vUKDg6+4jGpqamSpPr160uSwsPDtXfvXh0/ftysSUpKkqenp5o2bWrWrFu3zuY8SUlJCg8Pv+TnuLu7y9PT02YDAAC4FsTHx+vWW29VrVq15Ovrqz59+ujAgQM2NefOndOIESNUp04d1axZU3379rX7I11GRoaioqJUvXp1+fr6aty4cTp//rxNzYYNG9SmTRu5u7urcePGSkxMLOvLAwAAlcQ1HUqNGDFCH3zwgZYsWaJatWrJarXKarXq7NmzkqRDhw5p+vTp2rVrlw4fPqzPP/9cgwcPVqdOndSiRQtJUvfu3dW0aVM9+uij+u6777RmzRpNnDhRI0aMkLu7uyTpySef1M8//6zx48frhx9+0JtvvqlPPvlEY8eOLbdrBwAAuFrffPONRowYoa1btyopKUkFBQXq3r27Tp8+bdaMHTtWX3zxhZYuXapvvvlGR48e1QMPPGDOX7hwQVFRUcrPz9eWLVu0aNEiJSYmavLkyWZNenq6oqKi1LVrV6WmpmrMmDEaNmyY1qxZ49TrBQAAFdM1/Uypt956S5LUpUsXm/F3331Xjz32mNzc3PTVV19p9uzZOn36tAIDA9W3b19NnDjRrK1SpYpWrFihp556SuHh4apRo4aGDBmiadOmmTXBwcFauXKlxo4dqzlz5ujGG2/U22+/rcjISKdcJwAAQGlavXq1zX5iYqJ8fX21a9cuderUSdnZ2XrnnXe0ZMkS3XXXXZL+WF+FhoZq69atuv3227V27Vp9//33+uqrr+Tn56dWrVpp+vTpmjBhgqZMmSI3NzctWLBAwcHBmjlzpiQpNDRUmzZt0uuvv846CgAAXNE1HUoZhnHZ+cDAQH3zzTdXPE9QUJBWrVp12ZouXbpo9+7dDvUHAABQEWRnZ0uSfHx8JEm7du1SQUGBIiIizJqQkBA1aNBAKSkpuv3225WSkqKwsDCbNxRHRkbqqaee0v79+9W6dWulpKTYnKOoZsyYMWV/UQAAoMK7pkMpAAAA/DWFhYUaM2aM7rjjDjVv3lySZLVa5ebmJm9vb5taPz8/Wa1Ws+biQKpovmjucjU5OTk6e/ZssS+MycvLU15enrmfk5Pz1y4QAABUWNf0M6UAAADw14wYMUL79u3TRx99VN6tSPrjIexeXl7mFhgYWN4tAQCAckIoBQAAUEmNHDlSK1as0Ndff60bb7zRHPf391d+fr6ysrJs6jMzM+Xv72/W/PltfEX7V6rx9PQs9i4pSYqLi1N2dra5HTly5C9dIwAAqLgIpQAAACoZwzA0cuRIffbZZ1q/fr2Cg4Nt5tu2bauqVatq3bp15tiBAweUkZGh8PBwSVJ4eLj27t2r48ePmzVJSUny9PRU06ZNzZqLz1FUU3SO4ri7u8vT09NmAwAA1yeeKQUAAFDJjBgxQkuWLNG///1v1apVy3wGlJeXl6pVqyYvLy9FR0crNjZWPj4+8vT01KhRoxQeHq7bb79dktS9e3c1bdpUjz76qGbMmCGr1aqJEydqxIgRcnd3lyQ9+eSTmjdvnsaPH6+hQ4dq/fr1+uSTT7Ry5cpyu3YAAFBxcKcUAABAJfPWW28pOztbXbp0Uf369c3t448/Nmtef/113XPPPerbt686deokf39/LVu2zJyvUqWKVqxYoSpVqig8PFyDBg3S4MGDNW3aNLMmODhYK1euVFJSklq2bKmZM2fq7bffVmRkpFOvFwAAVEzcKQUAAFDJGIZxxRoPDw/Nnz9f8+fPv2RNUFCQVq1addnzdOnSRbt373a4RwAAAO6UAgAAAAAAgNMRSgEAAAAAAMDp+PoeylxaWtpVHVe3bl01aNCglLsBAAAAAADXAkIplJljWZKLRRo0aNBVHV+9mofSfjhAMAUAAAAAQCVEKIUyk3VGKjSkD56WQgMcOzbtqDTozXM6ceIEoRQAAAAAAJUQoRTKXGiA1Ca4vLsAAAAAAADXEh50DgAAAAAAAKcjlAIAAAAAAIDTEUoBAAAAAADA6QilAAAAAAAA4HSEUgAAAAAAAHA6QikAAAAAAAA4HaEUAAAAAAAAnM61vBsAAAAAcO1IS0u7quPq1q2rBg0alHI3AIDKjFAKAAAAgKRjklw0aNCgqzraw6O6DhxII5gCAJQYoRQAAAAASVmSCiV9ICnUwWPTdO7cIJ04cYJQCgBQYoRSAAAAAC4SKqlNeTcBALgO8KBzAAAAAAAAOB2hFAAAAAAAAJyOr+/hmsbbXwAAAAAAqJwIpXBNOpYluVh01W9/qV7NQ2k/HCCYAgAAAADgGkUohWtS1hmp0JA+eFoKDXDs2LSj0qA3z/H2FwAAAAAArmGEUrimhQZIbYLLuwsAAAAAAFDaeNA5AAAAAAAAnI5QCgAAAAAAAE5HKAUAAAAAAACnI5QCAAAAAACA0xFKAQAAAAAAwOkIpQAAAAAAAOB0hFIAAAAAAABwOkIpAAAAAAAAOB2hFAAAAAAAAJyOUAoAAAAAAABO51reDQAAAACoHNLS0q762Lp166pBgwal2A0A4FpHKAUAAADgLzomyUWDBg266jN4eFTXgQNpBFMAcB0hlAIAAADwF2VJKpT0gaTQqzg+TefODdKJEycIpQDgOkIoBQAAAKCUhEpqU95NAAAqCB50DgAAAAAAAKfjTilUWjxoEwAAAACAaxehFCqdY1mSi0V/6UGb1at5KO2HAwRTAAAAAACUEUIpVDpZZ6RCQ/rgaSk0wPHj045Kg948x4M2AQAAAAAoQ4RSqLRCA6Q2weXdBQAAAAAAKA4POgcAAAAAAIDTEUoBAAAAAADA6QilAAAAAAAA4HQ8UwoAAADANSEtLe2qjqtbty4vqAGACohQCgAAAEA5OybJRYMGDbqqoz08quvAgTSCKQCoYAilgEvgL3UAAADOkiWpUNIHkkIdPDZN584N0okTJ1iDAUAFQygF/MmxLMnFoqv+S131ah5K++EAiyIAAK4BGRkZOnHihMPHXe0fp/BXhUpqU95NAACchFDqT+bPn69XX31VVqtVLVu21Ny5c3XbbbeVd1twoqwzUqEhffC0FBrg2LFpR6VBb57jL3X4f+3deVxN+f8H8PdtRZso0kQiKWRJSZYWJCNbGIydMPa9hEj2db6GGZMxYwyDjHUw9iXDjG1QabMkJmTLktJ27339/uh3z3Tdlltu3Rvv5+PR41Hn3t73fZZ77+d8zud83owx9knR1PbTv//+Sw0bOlBW1jt1p8IYY4yxAnCnVD67du2i6dOnU1hYGLm6utLatWvJx8eHbt26RTVq1FB3eqycOVgSOdmU7n/51j/GGGOfCk1uP7148eL/O6RKc0vYESKap/qkWJnh9hdjjFU83CmVz9dff02jR4+mESNGEBFRWFgY/fHHH7R582YKCgpSc3asIuBb/xhjjH1qKkb7qTS3hPHtexUHT5LOGGMVFXdK/b+cnBy6du0azZ49W1impaVFnTp1oosXL6oxM1aRqOLWv/Pnz5ODQ0mv5ubJzs4mfX39Uv0vXyVkjDFWUtx+YprhNX3oJOmlbX9x+4kxxj4Md0r9vxcvXpBEIqGaNWvKLa9ZsyYlJCQoPD87O5uys7OFv9+8eUNERGlpaSrPLT09nYiIrt0nSs8q2f/GP6ZS/++H/n9F/F9Vvfa7nJL/78NUIhGVfpQVUd4oLSlK97+V9PVo67ZfFd4DSr2ulhZJpdJSve6H/K86X/tTzPtTXGd1vjavc8lZWFiQhYVFqf+/ILLvdaCUH64fuZK2n4jU04YiukZE6UU9tQCykVIV6X/V+dqakPe7UvzvQyISlbr9padXiX79dWup2k9EFffztiLm/Smuszpfm9e54rx2WbSfiErQhgIDADx69AhEhL///ltueUBAAFq1aqXw/JCQEBAR//AP//AP//AP/3wCP8nJyeXVJKlQStp+ArgNxT/8wz/8wz/88yn9FNeG4pFS/8/MzIy0tbXp6dOncsufPn1aYK/h7Nmzafr06cLfUqmUXr58SdWrVyeRSKSyvNLS0qh27dqUnJxMxsbGKourapyn6lWUXDlP1asouXKeqldRcq0oeaoCAHr79i1ZWpbwnuxPREnbT0Tl04ZS5TGq6uOdc1N/PM5N/bE4N82Ix7mpP5am5/YhlG1DcafU/9PT06OWLVvS6dOnqVevXkSU10g6ffo0TZw4UeH5+vr6CnP3VK1atczyMzY2VvtBpQzOU/UqSq6cp+pVlFw5T9WrKLlWlDw/lImJibpT0FglbT8RlW8bSpXHqKqPd85N/fE4N/XHUnU8zk39sVQdj3NTf6wPoUwbijul8pk+fToNGzaMnJ2dqVWrVrR27VrKyMgQqskwxhhjjDF53H5ijDHGWGlxp1Q+/fv3p+fPn9P8+fPpyZMn1Lx5czp27FipJy5kjDHGGPvYcfuJMcYYY6XFnVLvmThxYqHDzdVBX1+fQkJCFIa5axrOU/UqSq6cp+pVlFw5T9WrKLlWlDxZ+fmY20+qPt45N/XH49zUH0vV8Tg39cdSdTzOTf2xyosI4BrHjDHGGGOMMcYYY6x8aak7AcYYY4wxxhhjjDH26eFOKcYYY4wxxhhjjDFW7rhTijHGGGOMMcYYY4yVO+6UYowxxhhjjDHGGGPljjulGGOMffIqSs2PipInY4wxxhhjyuBOKQ0klUpJIpGoO40PVpFPnjQ59/dz0+Rc8+M8Pxzve9V7/7NWKpWqKZOiVZQ889Pk/c4+ThWx/aTJ7xNNzo19GAAq3b+qjqWqeGWxnpqamyymquJoam6qjsU0g466E2Dy4uLiaOnSpfTkyRNq0KABDRkyhNq0aaPutEpELBaTjo4OSaVS0tbWJqlUSlpamt3/mZaWRpmZmaSnp0empqYkEokIAIlEInWnJufWrVu0fft2+vfff6ldu3bUrl07sre318htnJmZSdnZ2WRkZETa2tokEolIIpGQtra2ulOTU1Hy5H2vevHx8bR+/Xp6/PgxOTg4UN++fally5bqTktBRcmzoP2uiccn+ziVd/uptG2ElJQUSk5OplevXlGnTp3K5HPxY8wtOTmZ4uPj6dmzZ+Tr60sGBgakp6f3UeZWluuanZ1N+vr6JBaLSVdX94NiZWZmkr6+PuXm5pK+vr5G5abKWJqemyr3gybnpspY9+/fp5MnT5KWlhbVrl2bOnfu/EHxEhMTac+ePZSbm0t169alwYMHa0xuqo5XZsA0RkJCAkxMTDBgwAAEBQWhWbNmcHZ2xjfffKPu1JQWFxeHkSNHonfv3hgzZgwSEhLUnVKxoqOj0aZNG9SrVw8uLi4YMWIEcnNz1Z2WgtjYWJiYmKBPnz5o06YNXF1dYWVlhVOnTgEApFKpmjP8z82bN9G5c2c4ODiga9euCAkJER4Ti8XqS+w9FSVP3veqFx8fD2NjYwwbNgx9+vSBt7c39PX1sXXrVnWnJqei5FnUfpdIJOpLjH0SyrL9dOvWLQQGBmL48OFYu3Ytbt++LTxW0s/eqKgoWFtbw87ODiYmJrC3t8eOHTuQmppaqtzu3LmDZcuWISgoCDt27MDbt28/2twsLCzg6OgIY2Nj1KlTB4sXL0ZycvJHl5uq4+UXExMDPz8/dOrUCT4+Pjh37hyys7NLFevmzZvo2LEjWrdujcaNG2Pbtm149OiRRuSmylianpsq94Mm56bKWNHR0ahevTpat26N+vXrw9DQEKNGjcLjx49LnZuJiQk8PDzg4uICfX19+Pr64tKlS2rPTdXxyhJ3SmkIqVSKOXPmoF+/fsKytLQ0LF68GM2bN8eKFSvUmJ1yEhISYGRkhGHDhuHLL79Ehw4dUKlSJfz000/IyMhQd3oFun//PszNzTFjxgzs3bsXK1euRIMGDeDo6Ig7d+6oOz2BWCzG4MGDMWjQIGHZjRs34O/vD21tbRw+fBiAZpz8JSYmolq1apgwYQLCwsIwduxY1K9fHx4eHkKnhCZ0TlSUPHnfl43x48ejV69ewt9Pnz5FcHAwtLW1sWHDBgCa0dlXEfJUZr9rwvHJPk5l2X6SXRDo0qUL+vTpAxMTE3Tq1AmbNm2Se31lPHv2DPb29pgzZw4SExPx6NEj9O/fHw4ODggJCcGzZ89KlFtMTAyqVq0KDw8PuLu7Q0dHB3369MGxY8c+qtxevnwJJycnBAYG4unTp5BIJJgxYwZcXV0xdOhQ3L9//6PJTdXx8rt9+zaMjY0xZswYBAQEoG/fvhCJRAgJCcGDBw9KFCsxMRGmpqaYMGEC1q9fj0mTJsHU1BRjxozBP//8o9bcVBlL03NT5X7Q5NxUGevt27dwc3PDpEmTAAApKSk4evQoqlWrhi5duuDu3bslivfu3Tv4+Phg/PjxAIDMzEzExcXB1tYW7u7uOHPmjNpyU3W8ssadUhpk+PDhcHd3l1uWlpaG1atXw9nZGb/++quaMlPOhAkT0KNHD+HvnJwczJ07F1paWli3bh1ycnLUmF3B9u7dC2dnZ7x580ZYlpiYCFdXVzg4OODp06cA1H9ClZOTAw8PDwQFBcktf/bsGcaNG4dKlSrh4sWLaspO3o8//ghPT0/h6kpOTg7OnDkDW1tbtG7dWnieurdpRcmT933Z6N27N/z9/RWWL126FCKRCH/88QcA9Xf4VIQ8ld3v6t6W7ONVFu2n7OxsDB48GKNHjxaW3blzB/3790fr1q1LPAorNjYWdevWVTiJmjVrFhwdHbFy5UqlL+C9e/cO3bp1w4QJE4Rl165dg7OzMzp16oR9+/Z9NLk9ePAA1tbWwshgmfXr18PNzQ3jx4/H8+fPK3xuZREvv+DgYHTu3Flu2bp161C9enXMmjULT548UTrW6tWrFd5v27dvR9OmTTF06FDcvHlTbbmpMpam56bK/aDJuakyVmZmJpycnBAeHi63/NatWzAzM0OvXr1KfPG0bdu2WLlyJQAId9o8evQITZs2hYeHh9KjHFWdW1msa1niiR40AP5/sjYnJyeSSCR069Yt4TEjIyMaOXIktWjRgjZs2EDv3r1TV5rFev36NVWrVo2I8iYb1dXVpcWLF9PChQtpxowZdPLkSeExTZGSkkL3798nY2NjIsrLrV69erR//37S09Oj3r17ExGpfU4UXV1datKkCZ07d45evXolLDc3N6fZs2eTr68vLVq0iNLS0tSYZZ7k5GRKSkoS5j/Q1dUlT09P2rZtGz1//pz69u1LROrfphUlT973ZaNp06Z04sQJevz4MRH99zk8c+ZM+uqrr2jmzJn05MkTtc8rVxHyVHa/q3tbso9PWbaf9PT06OnTp8JxC4BsbW1p5cqVZG9vT3v27KFDhw4pHS83N5fEYrGQR2ZmJhERLV++nLy8vOj777+nu3fvyq1XYSpXrkwvX74kMzMzIspruzg5OdG2bdtILBbTDz/8QFFRUUrnlp2drbG5aWlpUZUqVYTPQLFYTEREEydOpN69e9PZs2fpr7/+Ujq31NRUjcyNKO8zsnLlyiqLl59sn+aPO2nSJFqyZAl9++23tH//fiJSvo2elpZG6enpwvMHDhxI8+bNo4sXL1J4eDhJpVKlc1RlbqpeT03OjUh1+0GTc1NlLIlEQk+fPpX7rsjNzSU7Ozs6ffo0nTx5kpYtW6ZUTgAoKyuLsrOz6d69e0REpKOjQzk5OWRpaUnHjx+nmzdv0ooVK5SKp8rcyiJemVNDRxgrxN27d2FmZoaRI0cK97fLriz/+++/EIlEOHr0qDpTLNLcuXNhYWGB169fA4DcyKivvvoKtWvXxosXL9SVnhzZdn3w4AE+++wzLFu2THhMNorjr7/+gq2trUIPc3l59uyZ3JD5PXv2oEWLFlizZg3S0tLknrtlyxZYWlri33//Le80BbLtdvnyZdja2ipcmc7JycGvv/4KR0dHjRjZU1HyBIBdu3ahefPmGrvvZSrSNr148SLatGmDiRMnKoyIPHXqFCwtLXHjxo1yz+vRo0e4deuW8Pfly5fRtm1bjcszfx4Vab+zj5Oq209isRg5OTkYMWIE+vbti6ysLEilUuGYT0xMhJubG/r371+iPF1cXODl5SX8nZWVJfzu7OyMAQMGKBXn7du38PLywtixY4V8ZVfoY2NjYWVlhSlTphQZ4/Hjx4iNjZV7/Q/JTbZt0tLS4OXlhXHjxpU6t4yMDLm5bHr06IEWLVoI7cv8835+/vnncnkXJDk5GVevXoVYLP7g3N7XrVu3D8pNIpHIjR7+4osv4OjoWOp4hfnmm29gZGQkzMOTf/uGhobC0NBQ6XbErl27ULlyZVy/fl0h1vfffw89PT3ExMQondu6detUlpsq11PVuakyFvDh+yE9Pb3Mctu5c6fKjpHw8HCVHm9r1qyBlZUVDh06JCyTnbMuXrwYrq6uSE1NLXZ0t+zxvXv3Ksz1mZmZCQDYunUr6tatiwcPHig1Wvzrr79WSW6qXtfywJ1SGubMmTPQ19fHhAkT5IbopqSkoFmzZvj777/VmJ28Bw8eID4+Xvj79u3bcHNzw8CBA4Xb4WQH/tWrV2FlZaX2/GWNLFleb968wdSpU9G+fXvs2LFD7rlv3ryBnZ0dlixZUu55Xrt2DTVr1sT58+fllo8fPx729vbYsGGD3OSjsbGxsLW1lWtglhdZg0k2BPThw4fo0aMHfH198eeff8o99+nTpzA1NUVYWFi553nnzh2cOXNGaPw9efIEPXr0QLdu3TQqz6SkJPzwww/48ccf5ea4mDhxIuzs7DRq31eUbXr37l0sX74cixYtkms0rFmzBs2bN0dAQAAePnwoLH/48CEaNGiACxculGue169fR/Xq1eUaDwCwbNkyODk5aUyeFeU9zz4tqmg/vX8rQ0REBLS1teVu1ZM9JyIiAlpaWoWeDKWnpyMtLU1ueoDr16+jRo0a+PLLL4VlsvfT9OnT0b1790JzS01NRXx8vNBpfejQIYhEIuzduxdAXueGrG2zY8cOmJqaFjoXzMOHD1G9enX4+fkJHcY3btyAmZlZqXK7ceMGunXrJpzk7t69u9S53bx5E76+vjh37pwQ7/nz57CxsYG3t7fCxMtr165F+/btC70NJSYmBrVr18a0adMA5J0slza35ORk7Nq1C3v37hVOkD8kt9jYWAwZMgReXl4YMWIEjhw5gmfPnqFZs2bw8vIqcbyiZGdnw93dHa1btxYuEMtOnFNSUlC7du1ib1/Mf+Lq5+eH2rVrCxdL8ndi2traYt26dUrn9u7dO3h6epYqN6lUKrc9srKySh0LyHuf5b8gnJGRUep4d+7cwZUrV1SynrJ4+/btk9vWpd0PCQkJGDx4sHCsp6enf1BuGRkZePnyJd69eycs69mzp8qOkR49epQq1uPHj3H58mUcO3ZMOE6SkpLwxRdfoH379jh+/Ljc88PCwuDg4FDo7coFzZOZmpqKyZMno169egrnkvv27YOdnV2BgzJevHiB+Ph4uYt2N2/eRP/+/UuV28OHD3HixAls2bJF+OyOiYkpdbzyxp1SGujgwYPQ19dH7969ER4ejri4OAQFBaFWrVoqqb6hCtevX4e5uTl2794tLJNIJPj222/h6uqKUaNG4dWrV8JjycnJqF+/vsIJS3l6v6pEREQEgLzONV9fX3h4eGDz5s1y/9OlSxesXr0aQPnNhxIZGQlDQ8NCr9YNGTIEjo6OmDp1Ku7evYvnz58jMDCw0A+9svR+tcW4uDgAedUeGjVqhO7du8t9CEokEnh4eGDbtm3lmmdkZCREIhE2btwI4L99eePGDTg4OKBHjx4akWdBVTKGDx8ujI7y9/dHkyZNNGLfV5RtWlBVlC5duiA6OhpA3pUiFxcXdO/eHZGRkbhz5w6CgoJgbW2NlJSUcsszMjISBgYGmDp1aoGPz507F66urmrPs6K859mn6UPaT7du3cLq1asVqhKtXr0aWlpacpObA3kXjxwcHJCUlKQQKzY2Fp07d0aLFi1gaWkpjCLMzMzEzp07YWZmhr59+yInJ0c4uRk8eDAGDBiA3NxchfbGzZs30aJFCzg6OkJXVxehoaHIysrCpEmToK+vr9CRfeTIETg4OBT6vXD27Fno6OigQ4cOGDp0qNDBEh4eDlNTU/Tq1Uvp3CIjI1G5cmXMmjVLWJabm4uJEydCX18fBw8eVDo32STkX331lcKojIsXL8LS0hIeHh64ffu2cMLs7++Prl27FlglLDIyElWqVIGNjQ1q1qwpfFbKtpusUIgyuUVHR8Pa2hrOzs6oWbMmunfvLnQQXrx4EVZWViXKLT4+HqampvD398eaNWvg4+ODevXqYcqUKfjrr7/QuHFjtG3bVul4+RVWMfL06dNo1aoVOnbsKHdx6+XLl7C3t1c4joC8Cwv52/OyYyImJgZt27aFjY2N3HsrIyMDLVq0KPQz/969e/j6668xffp04U4EqVSKQ4cOwc3NrUS53bp1C1OmTIGvry9CQ0OF/Xb48OESxwLyRkDWq1cP8+bNk6vsdvjwYbRp06ZE8W7cuAFjY2P88MMPcstLm1tUVBTMzc0xevRoPHr0SHgfxsTEwM3NrUT7QfaeFYlE+PnnnwHk7YM//vijVLnFxMSgW7ducHBwQK9evYT3fGmOkYSEBAQFBWHw4MFYtWoVrl27BiDv4qKrq2uJYr1fUbRhw4bYuXMncnJycPXqVXTr1g0uLi7YuXMngLwBC4GBgfDw8FC4KwHI+xz29PQUPpvyd0zFxMRgzJgxsLCwwLp165CZmYn09HTMmTMHTk5OePnypVys6OhotGrVCg0bNkSNGjXk5vM6deoUevTogVatWimdW3R0NOzs7ODk5AQDAwM0b95c6Jg6ffo0evToUaJ1VQfulNJQ165dg4eHB6ytrVG/fn3Y2dkJjQZ1k508ya465ZeTk4MVK1bA1dUV7u7uiI2Nxc2bNxEcHIw6dep8ULnYD1FYVYng4GBkZGQgKSkJ/fr1g6OjIwYPHoxt27Zh7NixMDY2liv/XNaio6NhYmKCgIAAAHkfeDdv3sTFixcRGRkpPG/+/Plo3749RCIRWrZsCQsLi3I/Pgqqtqivry98AUdGRsLFxQXt2rXDrFmzcPz4cUyZMgWmpqZITEwstzyjoqJgYGAgbFMZ2ZfJpUuXNCLPoqpkdOzYUbg6FBoaqvZ9X1G2aVFVUdq0aSOU6926dSs+//xziEQiNGnSBNbW1uW6TWNjY2FkZCRsT7FYjPPnz+PIkSM4ffq08Lyff/5ZrXlWlPc8+7SVpv10584dVKtWDSKRCLNnz5YbaZWRkYHQ0FChzXD9+nWkpqYiKCgItra2CpXpYmNjUb16dUybNg3bt2/H9OnToaurK+SQkZGBgwcPwsrKCvb29ujVqxf69esHAwODAifslcWbOXMmYmNjsXr1aohEIjx69AiPHj3C6NGjoauri++//x4pKSnIzMxEUFAQmjVrpnAiJJOamooePXpg48aNcHJywsCBA4W2zoEDB9CoUSM0bNiw2NwK+y4Qi8V48eIFJkyYoHRu6enp6Ny5s3BrHZDXcXPjxg3hJDQmJgaNGjVCgwYN0KpVK/Ts2ROGhoaIiopSyE124j1nzhw8f/4cjRo1wuLFiwHkdYyMGTMGurq62LhxY7G53b9/H5999hmCgoKQnp6OI0eOwMLCApcvXxaeU5LcsrKyMGjQIEyePFlYlpmZiebNm0MkEuHLL79EdHQ0XF1dUa9evWLj5VdQxcgOHToIo4QPHTqEVq1awcbGBsePH8eZM2cQHBwMCwsLhRFicXFx0NPTQ9++feVG/MlcuXIFnp6eqFq1KjZu3IidO3ciKCgI1atXL/AzPzo6GlZWVujYsSPatGkDLS0tLF++HEDeMfPbb78JHSzF5RYdHY0aNWqgb9+++Oqrr6Cnp4f58+cLsXbt2iV0YhQXSyYsLAwikQgtWrTAkiVLhHMWsViM8PBwpbebrDN0+vTpCq8hFouxe/fuEuX24MED1KlTR+F9ln9btG/fXqn9IHtfBAYGYubMmWjfvr2wnhKJpET7AMg73mSV8cLCwtC2bVthtKVUKsXVq1fh7u6uVG6xsbGoWrUqvvjiC4wdOxa1a9dG8+bNhTbGzZs3lV7PwiqK2tnZCZ36kZGRGDt2LHR0dNCsWTO0bt0apqamBU6JkJSUBFtbW4hEIjRo0ED4TMp/e+2dO3ewePFi6Ovrw9bWFs2aNYO5ubnC909CQgLMzMwQFBSEixcv4vjx46hXr55cx/5ff/2FcePGKZVbfHw8zMzMEBwcjAcPHuDevXswMzOTuyBw8eJFpddVXbhTSoO9efMGSUlJiI6OLnW1DVWLj49HlSpVMGfOHAB5b8aIiAjs378fZ8+eBZD3gfv777+jU6dO0NPTg729PerVqyf0dqtDYVUlqlWrhpkzZyInJwePHz/Gjz/+CCcnJ2Heh/wdQWUtNzcX9vb2MDAwQFZWFiQSCXr06IGWLVvCwMAAderUQXBwsPD8Z8+e4ejRo7hw4YJaRtAVVm1RJBLh66+/BpD3wTt37lzY2dmhSZMmcHZ2LtcPwNjYWFSqVEnoQJVIJAgPD8eSJUvw3XffCVc7YmJi1JonUHyVjG7dugnLnj59qrZ9X5G2KVB0VZS2bdsKJ5RisRiXL19GbGxsuY08kkqlyM3NRadOnWBsbIz4+HhIJBL4+vrCyckJ1apVQ+XKleWqf6kjT5mK8J5nDChZ+yk9PR0jR47E8OHD8d1330EkEiEgIECus0kikeCXX36BhYUFPvvsM9jb28PS0lKhXZOamorOnTvLdTYAgKenp3DBQSYtLQ2BgYEYNWoUJk6cWOAt2M+fP4e7u7vcyGmpVAofHx9cunQJ0dHRuHLlCjZs2AA9PT3Y2NigadOmBZ4IyYjFYjx79gx2dnZ4+PAh9u3bBxcXF/j7+8PDwwP9+vVDWloaZs6cWWRuKSkpsLCwgI+PjxB36tSp+Pzzz9GoUSOsX78eZ8+exbp165TKLSsrC+3atcP169chFovh4+MDFxcXGBoawtXVFT/++KPw3HXr1iEoKAghISFISEhQiBUVFQV9fX2hrSqRSNC3b1+0bNlSeM7jx4+xdOlS6OnpoV69ekXmtnHjRnh6esqNEuvatSs2btyILVu2CO1fZXKT6dixIxYsWADgv9ukAgMD0bt3b7Rs2RLfffcdgLzKe8rEA4quGOni4iKMbI6Li8OXX34Jc3Nz2NnZoXHjxgrH8pMnT9CmTRt06NABZmZm+OKLLwrsmHr58iWmT58OBwcHNGzYEK6urgVuw/v378PW1haBgYHCBayffvoJNWvWFNZLKpUiMjISgwYNKjK3e/fuoW7dupg9e7awbMGCBRg/frzcbV1xcXEYMGBAkbHyi4qKwrBhw7B48WJYWlpi0aJFcqPmbt++jYEDBxYZ7/bt29DX18fcuXMB5H1PHjx4ED/88AP27dsnd1tVcftA5tChQ+jatasQb+7cuejVqxdGjhwp3C6WlZWFyZMnF7kf/vnnHxgbGwvvi507d8LExERuCgBl9wGQd+GvV69ecp9Pv//+O/z8/JCSkiLsC2Vye/v2LXx8fBAYGCgsk91mbG5ujhUrVgjPmzp1arHHW1EVRRs3bozVq1dDKpUiPT0dFy9exKJFixAWFoY7d+4oxMrMzERwcDD8/Pxw+vRpuLu7w9rausCOKSDvXPmnn35CeHi4wkjat2/fol+/fsLFUiDv82nSpEly7SvZc4vL7fXr1+jatavCKHsfHx9s2rQJq1atEqbayc7Oxt9//11kPHXiTimmNLFYDD8/P5iZmQkjDLp3745mzZqhZs2a0NXVxdixY+W+tC5fvoz4+PhyP3l634wZM4ROqfwfHmFhYahSpYrw5S+TmZkpNBLKU2xsLMzMzNC9e3e0a9cOnTt3xp9//okLFy7gm2++gba2tvDBrG6DBg3C8OHDAcgPYV28eDF0dHSEHvrc3Fzk5OTgxYsXwgS05eWbb76BSCTCnj178ObNG3h4eMDNzQ3169eHo6MjqlevLlx1VGeeYrEY6enp+OyzzxAaGiosl81zIbsaLWvAqtP69esrxDaVSqXIzMyEs7OzMBkw8N8EmSkpKahWrZrcVfnyJpub4M6dO2jZsiW8vb3RvHlz+Pj44Nq1a4iPj8fvv/8OAwODAq+4lreK8J5nrKTevXuH7777TrggsGvXrgI7poC8q+Xnzp3D0aNH5eZ3k3ny5AlatWolTFUge5+MGDECgwYNAgC5CdNl3v9b5sWLF1i6dKnciO2FCxdCJBKhadOmqFOnDrp06YK4uDgkJCRg165dCA8Px/379wtdX1nHyqBBg4R5C//44w+YmZnB0NBQrvOnqNxSUlLg5+cHZ2dnHDhwAF26dEHHjh0xY8YMjB8/HvXr18eoUaOQnp6OqKioYnN78uQJzM3NceLECUybNg0+Pj6IiorC0aNHERAQAAsLC4X5Wgpz5coVzJs3Ty7/hIQEmJiY4Ntvv5V7rjK5hYWFoV69esLJ7+LFiyESidCpUyc4OzujRo0aCrdpFUYqlSIjIwPt27fHkCFDhDbpw4cPYW1tjc2bN2Pw4MFo3769UvHe5+3tjTFjxgivBeSNtBk+fDjatm2LI0eOCM+Nj4/Ho0ePCuy4PXr0KAYOHIirV6/i8uXLqFatWqEdU7L8X716JXern4xEIsHy5cvRpUsXYQJ34L+RUwV1thWWm1gsxqpVqzBu3Di5XEaNGgU3Nze4uLhgzJgxSq9nfpGRkWjQoAGkUilCQ0NRu3ZtrF27Fj179hSOp6Li5ebmYvLkyahevbowvUnXrl3RtGlT1K1bF1paWujdu7cwfYCyuYWGhqJ169YAgE6dOsHT0xNTpkyBt7c3mjVrJnQyAYXvh/T09ALvcunYsSM6dOig0LGiTG4SiQTt27eXa7fOnDkTdevWxWeffQZPT0+5jsOijpGMjAy4uLgI73HZPEdffPEFOnbsiNatW8sVq3j06FGhsYC8fWllZSV8Fuef62ry5MmwtrYudtRhfuHh4di1axeAvA7W9u3by3VMFTTXVEHevn0Lf39/hc+LXbt2wdHREdnZ2UJnnrLTxoSFhcnNmbho0SLo6OjAy8sLDg4OMDc3x2+//abciqoRd0oxpSQnJyMpKQlxcXHw8fGBj48P7O3t0aVLF1y/fh0PHjzAH3/8AT09Pbnhh5qiuKoSBgYGhQ7nLW9xcXHC/cBPnjwRlmdlZWHq1Klo27YtXr58qfZKCXPmzCmy2qKVlZXaRvjl71AMDg6GtrY2bG1t4efnh8TERGRkZODOnTvo0aMHbGxs1FYV8v0vU2WrbhT3pVfWNHmbvm/37t3FVkW5f/9+ub+f7t27h7CwMCGXxMRENG7cGG3atFGYS+Xrr79G3bp18e+//6r1fV9chVV1vucZ+xD5q1ABeScgIpEIM2fOFI7p3NxcpdoJ+TuQZO+R4OBgDBkyRO55+U+oi3pf55/vQzZJ965du5CamoqIiAg4OzsLty2VxNChQxEUFAQgb64iU1NTNGrUCCNHjpSbeLeo3B4/foyhQ4eicuXK8Pb2lvvc//XXX2FiYlLoPDTvk0qlGDBgACZOnIhu3brJFfpITk7G4MGDMXbsWOTm5grfgcp+HkqlUrx+/Vq4HVEWQ9nv0nv37qFNmzawtbVFnz59IBKJcODAAUilUjx9+hSTJ0+Gp6cnnj9/rnRuFy5cgJaWFtzd3TFkyBAYGBhg1KhRAPJuUzIyMkJ8fLxwsltcPGUrRvbr109uuxTm2bNnciPALl68KHRM5e9YKqgzoyDnzp0TjjcZiUSCunXryr2OMpKTk+WO0UWLFkFbWxtz587FunXr4OLigg4dOght/pJ8b3bu3FkY3bJy5UoYGBjAxMRE7ngs6ri5ffs2xowZg9atW6N27dro2rUr4uPj8e7dO/zzzz/47LPP5D4LlMnt5MmT6NChA3788Ud4e3sLHeKvX78WOqxkHV1F5ZZ/1I7suNq0aRPs7OyEkVASiUTIqajcJBIJ3rx5Ax8fH/j5+eG7777D7NmzUblyZfz88884evQoQkND4eTkJEyQXlhusveRpaUlVq1aJSxPTk5Go0aN8Msvv6Bp06bC+6O43GRUVe20oHwTExOFEVOy/ZGZmYnr168XOXG4RCKR6/yWrYesUyq/97+XCsrjfX/++Sfq16+PgwcPCnn06NEDzs7Oyq2cGnGnFCtWTEwMrKyshKGBV69eRdu2beHt7a0wLPHbb7+FmZkZkpOT1XrypOqKF2WloMoLQN5VrZ9//lnupA/I6whycXFRuhFQlu7cuQNXV1eNq7aYkJCAGTNmyF0VCQkJgY2NjcLtmMePH4e5ubncvBDl5caNG2jXrp3clZqEhIQKUyVDE7fpv//+i+PHj2Pbtm1ITU1FdnY2srOzMWHCBNjY2JSoKkpZioqKQqVKlWBpaSn3Xn706BHCw8MV3vf/+9//0LRpU7ljujxUxAqrjH0IsVgstF1kHUABAQF49OgRpk2bht69eyM9PV2p9k3+E7C5c+cKt7kBwNKlS7FmzZoSf5ffv39f4TYaX19fudu7iyPLfcuWLQgJCcG4ceNQq1Yt3Lt3D/v27UP9+vUxduxYuRO4ojx69AizZ88W5r/Lv21sbW0xc+ZMpXO7evUqDAwMIBKJFCZInzFjBtzd3T+obbl3716IRKJSVSy9d+8edu3ahZCQEPTt21fuseXLl6NZs2YlHmF/5coVDB48GKNGjZIbsf/777/DwcFBrvOnMKqsGFlYVb/880XmHzGVk5ODDRs24MSJEyWKJ9uHEokENjY2cv9/6tQphRGKRcV68eIFpk6dKjeKJi4uDiKRSG6ZsvE8PT3xyy+/AMjrrDU2NoaFhQVWrlxZ6Ly478e6e/cuhgwZAl9fX4VRYAcPHoRIJBImyVcmXnx8PCwtLdGoUSN06tRJ7rF///0XVapUKXQUYf5YBb133r59i9q1a2PChAmF5lNUbpcuXUKXLl0wcOBANGzYED/99JPw2JMnT1CnTh0sW7ZMqVjffvstRCIRRo4cieDgYBgaGgq3ou7evRt169bFixcvCuzcUnW104LiAfKf63fv3hU6pu7du4cJEybA2dm5wFFqxcXavXs3GjduLPw9ffp0dOvWrcDjtLB4QN5oNNn8WrJ1XbVqFVxdXRXalpqGO6VYkd6vXCKrSpOQkIA9e/YIB7zsg+7bb7+Fo6OjWm59k1F1xYuy8n7lhfwN1sKMGTMG/v7+xVZeUbU7d+5g2bJlCAoKwo4dO4ST402bNsHZ2Vljqi3KTvZFIpFCVZ2rV68Kecu+CC5cuICGDRsWO0+DqkVGRkJXV7fASStPnDihUVUyHjx4gM2bN2PNmjU4efKk3GOatE2joqJgYWEBR0dHGBsbo3bt2liyZAlevHiBR48ewd/fHzVr1lSqKkpZkn2mDh48GA0aNMCSJUsAFH11c9KkSejXr1+5dkpVxAqrjKlC/tEl4eHh0NXVRcOGDaGjo1Pi+dFkbaO5c+fi888/BwDMmzcPIpHog+eslEgkyMzMRP/+/YXPkZI4d+4cRCIRLCws5OZd2b9/P+7du1eiWG/evJFrl0ilUrx48QJubm7Yvn17iWL9+eefEIlE6Natm1znyeTJkzFq1KgPOrHKzs5G586dMWjQoFJ/nm7atAm+vr5y6ztt2jT07Nmz2JENBSmoo2DmzJnw9PQs9FY5GVVWjCws1vtkt/L169cPI0aMgK6uLu7evatUvPzrmpubi/T0dNja2gpTgsyePVuYxL8kucku1sneu9HR0XBycpK7Ta64eLLjatasWdi2bRsmTZoES0tL3Lt3D0uXLkWVKlWwZs0ahU6CwnJ78OABjh49KsSVrfuePXtgb29f6G1nhcU7fPgwdHR0UKNGDbmLP9nZ2ejQoYPcSK7iYsnI1uW7775D/fr1FeZfUjZeeno6xGIx3NzchFvcgLxt6u3tLXS45t//BcWSSCTYsmULXFxc0KVLF7npStavX48WLVoU+H5RdbXTwuIV9NqJiYnw9PSESCSCgYGB3ICIksT6448/0LBhQwAQRpy9P2ChpLnJ+Pv7Y+TIkRoxoKEo3CnFCvV+5ZLGjRtj4cKFRQ4lnjJlCvr06aO20RyqrnhRVgqrvJD/3uv8UlJSMHfuXFSvXr3ACUfLkqxEs4eHB9zd3aGjowM/Pz/h5HPt2rVo1aqV2qstyo7XCRMmYMCAARg0aBAyMjKK/BAOCAgQbocsLzExMahcubJwu4VUKkVqaqpco05TqmTISmC3bdsWjRo1gq6urnAFsTDq2KYvX76Ek5MTAgMD8fTpU0gkEsyYMQMuLi4YNmwYnj17hufPn2PhwoXQ09MrsipKWZJVLpVNgDpgwAB4e3sXeozeu3cPc+fORdWqVQusfFXWeVakCquMqZJUKhXaOB06dEC1atUKPcEtiuwEKCQkBGPGjMGqVaugr6+vssIv8+bNQ506dUpVJTgnJwc//fSTMFpX1aPb58+fjwYNGhQ5v1Vhzp07B0tLS7Rq1Qr+/v4YMmQITExMVPI5uGzZMhgbG5d6rlNZdbuVK1di69atCAwMRNWqVUt1fLwvOjoa48ePh7GxcbGdlqqsGFlUrIJcuHABIpEI1apVK/BYViaerFNV1hmycOHCAk/qi4pV2G1mc+bMgaura4EjrorLbfPmzRCJRKhVqxauXr0qLF+xYoXC+6y4WIV1OPr4+BTY4VhcvJ07d0JLSws+Pj7YuXMn7ty5g6CgIFhaWirc9l+SfSq7rfD9+XWVjSeRSJCeng5XV1fMmzcPr169wtu3bzFv3jxhFGZJcsvMzFQYqTlx4kT07dsXmZmZctu1rKqdvh+vsPZ3dnY2BgwYgGrVqimcn5Uk1u+//47WrVtjzpw50NPTK/B9VdLccnJyEBwcDDMzM7lR75qKO6VYgQqrXOLi4iI8J/+V/cTERMybNw9Vq1YtdFhwWSuLihdloSSVF4C8sqBDhw6FpaVluXdKvHv3Dt26dZMb1nvt2jU4OzvDy8tLuMXs0KFDaq22eO3aNRgZGQn7fu3atTA2NhYqS7w/CiUqKgpTp06FqalpiSY6/FAvXryAra0tWrRoISwbMWIEWrZsiVq1aqFdu3ZCQ1SZqhtl6d69e7C2tsasWbOQlZWF58+fIzQ0FC1atEBKSopCY0td2xTIuyJpbW2NU6dOyS1fv349XF1dMX78eOE2iKKqopSlxMREiEQi4RgFgPPnz0MkEmHv3r0Kz5dVv6lTp065vu8raoVVxlRNLBZj2rRpEIlEH/yZJpsY28TERO5Et7R+++03TJgwAdWrV/+gjvWymJ9w586dGDNmDExNTT8ot4SEBAQHB6NTp04YN27cB3dIyb6zXr58iZYtW37Q5/+ZM2dQv359NGjQAJ6enir5zsvKysK+ffswYMCAYuOpsmJkYbEK68TIzs7G2LFjYWRkVOBF0pLGa9GiBVxcXKCnp6fw3ihprNjYWAQHB8PY2LjAbahMvFu3biE4OFj43i3sPaJMrPztJFlFYmNj4wI7MJVd11OnTsHNzQ01a9aEvb097OzsFN5nJd1uADBs2DA0bNgQOTk5Cu07ZePJikTY2dnB1dUV1tbWpcot/+vHx8dj6tSpMDIyUthuqq52qky8/LlJJBKsX78e2traCutZ0liybWdqalrgiLWSxjt16hT69OkDKyurcr34+iG4U4oVqKjKJRs2bJB7bmxsLHr27Im6deuqrfR3WVW8KAvKVF7IPzz93r172Llzp3CPcHlr06YNQkJCAPx3LMTHx8PT0xPe3t5yve/qqLb46tUrhdFxOTk5cHZ2xtChQxW+XOPj4xEUFIQmTZp88O0TpTF+/Hi0a9cOISEhwhDlH374Afv374ebmxtq166t9jKtOTk5mDdvHnr27Cl3i8OJEydgYWGhMHRb3ds0OTkZDg4OwmTm+UcerVq1Cg0bNsSBAwfKPa/3bd68WfhdNimtn58f+vTpo3BbZmpqKk6dOlWuHWcVucIqY6omFovx448/qqRdc/XqVYhEIpWNdI6JiUG/fv0QFxenkniqFBUVBV9fX5VdoCzJhOTKkJWB/1Cpqal48uRJobdhlUZWVpZSuamyYmRRsQpqH1+5cgWNGzdWGNFU0nhisRipqakwMTGBtrZ2gR01JcntwYMH8PPzg4ODQ6HtEGXj5b/bo7ARhCXJLSkpCV26dEG9evUK/TwpSbwXL17g9u3buHHjRoH7qCSxZOt36dKlQm/bLUm8CxcuYPHixQgLCyuw/VKSWGlpaVi3bh08PDwK3G6qrnaqTLz3HTx4sMCRqiWNdffuXbRv377QEZcliSebiH3BggXlPpXGh+BOKaaU9yuXiMVi4Q2RnZ2Ns2fPluvJU0HKouJFWShJ5QXZF6O6qq29ffsWXl5eGDt2LIC8RoTshD82NhZWVlYKVyPUIf+XlVQqhVgsxpw5c9C4cWOhgZZ/mHd8fLxcZcOy9vjxY7mrdtOmTUPNmjXh6+urkEfjxo0xbNiwcsvtfbLbc3/77TeFeUpev36N2rVrK3xpSqVSxMXFles2zcjIkJvTo0ePHmjRooUwIip/x9Tnn38OT0/Pcsstv4yMjCJPMjZs2AATExOh4aCu93pFr7DKWFlQZTtBFR0h+WnypLXlPe/lp0iVFSOLiiUrBCKRSIRbxIq7PV+ZeLm5uXj+/DmOHTtWZAemMrHEYjGePn2K5ORkJCcnlzo3WXtRIpEoNa+asrk9e/YMSUlJxe4LZbebMudbyu5TZS94K3O85eTkKHWhvyT7NDc3t8jjTdXVTpWNp8z8rsrGevv2LYDivyOUjSeLo+5K3SXFnVKsRD6kcklZUHXFi7JS2soLvr6+hVYIKSupqamIj48XttOhQ4fkbjGSSCTCh+GOHTtgamqKBw8elHsnX2pqKuLi4nD79m2FCVYB4OnTpzAyMsLChQsVHitPDx8+RPXq1eHn5yc3aeHKlSuxd+9eISfZfu7Tp49CZZ/ycuPGDXTr1g3v3r2TK1YgyzE9PR21a9eWG/6vjmprN2/ehK+vL86dOyd8+T5//hw2Njbw9vZWOCFau3Yt2rdvX+7vpfx5vj8/Qv5j0dXVFQMGDFBbR3lFrLDKGGNMtRUji4vVq1evEk0QX1w8Pz8/peegVSa3khRZUma7aXJuqtqnsvVU9vtcmX2qjuNN1dVOVRmvuFirV68uUfu0LCq7agLulGIloorKJaqi6ooXZUXVlRfK0s2bN9GiRQs4OjpCV1cXoaGhyMrKwqRJk6Cvr69QofDIkSNwcHAQrmyoI099fX0sWrRI7gNd9vvMmTPRpk2bcp3E/n1nz56Fjo4OOnTogKFDhwq3RQFQeA9JpVL07dtXbhL08iKbKD4wMFBuuWxb5uTk4MmTJ7C0tBRu2ZRVyiloMtGyIpt4/6uvvlKY2PPixYuwtLSEh4cHbt++LTQC/f390bVr13K9el9UnoD8yL1FixahSZMmBd5aUdYqYoVVxhhj/1F1xcjCYpVmbprC4mlra2tsbpq83VSdW2luT9bU7abqaqeqjKfJuWkK7pRiJfahlUtUQdUVL8qKqisvlEeuM2fORGxsLFavXi2U5n306BFGjx4NXV1dfP/990hJSUFmZiaCgoLQrFmzcq20VlieBZ34nzhxAkZGRti/f3+55fe+1NRU9OjRAxs3boSTkxMGDRok3P6W/2pHbm4ugoODUatWrXKfUyoqKgoGBgYICAiQW/7+CLTnz58LJZIXLlwIQ0PDQueVKAvp6eno3Lkzxo0bJyyLj4/HjRs3hCH7MTExaNSoERo0aIBWrVqhZ8+eMDQ0LNfJ14vKM38HqazD79WrVxCJRFi0aFG55QhUzAqrjDHGFKmqYqSqY3FunFt5xVJ1tVNVxtPk3DQFd0oxpamycsmHUHXFi7Ki6soLZen58+dwd3fHlClThGVSqRQ+Pj64dOkSoqOjceXKFWzYsAF6enqwsbFB06ZNYW5uXq5VHQrLs0uXLvj7779x48YNhc6pzz//HO3bt4dEIin3241kcwnY2dnh4cOH2LdvH1xcXDB69Gi0adMGffr0AQAcO3YM3bt3h4WFRblXyUhJSYGFhYUw/FcsFmPq1Knw9fWFvb09/ve//wm3wmZlZaFJkyZC1bXyPk6zsrLQrl07XL9+HWKxGD4+PnBxcYGhoSFcXV3x448/Cs9dt24dgoKCEBISUu4TPRaWp5GREVq3bi2Xp2xE5/Lly8u1ZG9FrLDKGGOscKqsGKnKWJybZsT7VHJTdbVTVcbT5NzUjTulWImpqnJJaam64kVZUXXlhbL04sULLF26VG4SvYULF0IkEqFp06aoU6cOunTpgri4OCQkJGDXrl0IDw+Xm7Bd3Xk2b94cVlZW8PHxwfnz54XH9+/fj7t375ZrnjKyTrBBgwbh2LFjAPJu0zQzM4ORkRF+/vlnAHkj/wIDA8u1U0ImJSUFfn5+cHZ2xoEDB9ClSxd07NgRM2bMwIQJE2BjYwN/f38kJSUhOTkZIpEI+vr65TrySObJkycwNzfHiRMnMG3aNPj4+CAqKgpHjx5FQEAALCwssGPHjnLPqzR5yqqEypT3/f8VrcIqY4yxoqmyYqQqY6k6Huem/liqjqfJ1U5VGU+Tc1M37pRiFZKqK16UFVVXXihL+StJyCYc3LVrF1JTUxEREQFnZ2dhriN1KirPc+fOwcXFBQsWLFBjhoqGDh2KoKAgAHnzG5mamqJRo0YYOXKkcPtbeU/Cnd/jx48xdOhQVK5cGd7e3nJzhG3fvh1Vq1bF4cOHAeRNGq6uLz+pVIoBAwZg4sSJ6Natm9DRB+RVjxs8eDDGjh2L3NxcoaNFHZNxK5tn/iqm6lYRKqwyxhgrmiq/81T9/cm5qT/ep5Kbqs+nVBlPk3NTJx1irAIyMDAgIiKJREJaWlrUv39/AkADBw4kkUhEU6dOpdWrV1NSUhLt3LmTKlWqpJY8GzRoQEREUqmUdHV1iYgIAD179kx4zrJly0hPT4+mTp0qrJc6GBkZCb+7ubnRP//8Q05OTkRE5OHhQTVr1qTr16+rKz1BUXm6u7tTjRo16Nq1a+pKTw4AEolE1KFDB0pKSqLx48fTkSNH6Nq1axQZGUkBAQGkq6tLjo6OajtGiYhq1apFy5Yto88++4w6depE1atXF3IfOHAghYSE0Llz58jX15cmTpxI2traaslTJBLRjBkzyNPTk969e0djxowRHrOysqKaNWvS1atXSVtbm0QikfA/mpqnlpaWWvIriEgkIhMTExoyZAj17duXJk+eTG3btiUiIj09PfL09FRvgowxxoqlyu8UVX8/cW7qj/ep5Kbq8ylVxtPk3NSJO6VYhaatrU0ASCqV0oABA0gkEtGQIUPo4MGDlJiYSFeuXFHryb6MlpaWcJIv+5uIaP78+bR48WK6ceOG2k70C2JtbU3W1tZElNehlpOTQ4aGhtS0aVM1ZyZP0/OU7W8bGxsaMWIE1axZkw4fPkw2NjZkY2NDIpGImjVrphHHqKWlJQUFBQm5iEQiAkAvX74kc3NzYZuq+zh1dnamo0ePkoeHB/3www9Ur149aty4MRER5ebmkp2dHYnFYqETmPMsmW7dupG3tzd9//335OTkRJUrV1Z3Sowxxhhj7CMmAgB1J8HYh5IdxiKRiDp27EiRkZEUERFBjo6Oas7sP1KplLS0tGjBggWUkpJCDRo0oODgYPr777+FkT6aav78+fTLL7/QqVOnhNFfmkhT88zNzaVt27aRs7MzNW3aVK6DUtOFhITQzp076eTJk0IHoCb4888/6csvvyQrKytydHSknJwcOnjwIF24cIGaNGmi7vQEFSXP/JYvX07Lli2jW7dukYWFhbrTYYwxxhhjHzEeKcU+CiKRiCQSCQUEBNDZs2cpMjJSozqkiP4bHaWrq0ubNm0iY2NjunDhgkZ3SO3evZvOnTtH4eHhdPLkSY3q6MlP0/PU1dWl4cOHC8dAReiQCg8Pp7Nnz9Lu3bvp9OnTGtUhRZR3q+aZM2fo119/pUuXLlGDBg00sqOnouRJ9N/tpl999RXt2bOHsrKy1J0SY4wxxhj7yPFIKfbRkEgktGXLFmrZsiU1b95c3ekU6p9//qFWrVpRTEwMNWrUSN3pFCk2NpYWLlxICxYsIAcHB3WnU6iKkmdFEh0dTXPmzKEVK1YIt51pKqlUSkT/dfxqqoqSJwB69+7dRzNPAWOMMcYY01zcKcU+KhXltqiMjIwKc8KXm5urcfPeFKSi5FmR5OTkkJ6enrrTYIwxxhhjjH2kuFOKMcYYY4wxxhhjjJU7zb6HgDHGGGOMMcZYiXh6etLUqVPVnQZjjBWLO6UYY4wxxhhjTEXCwsLIyMiIxGKxsCw9PZ10dXXJ09NT7rkREREkEokoMTGxnLPUHElJSTRw4ECytLSkSpUqkZWVFfXs2ZMSEhLUnRpjrBxwpxRjjDHGGGOMqYiXlxelp6fTP//8Iyw7f/48WVhY0OXLl+Wqm549e5bq1KlD9evXL/HrAJDr+NJ0ubm5BS7z9vamN2/e0L59++jWrVu0a9cucnR0pNevX5d/koyxcsedUowxxhhjjDGmIg0bNqRatWpRRESEsCwiIoJ69uxJNjY2dOnSJbnlXl5eRESUnZ1NkydPpho1alClSpWoXbt2dPXqVbnnikQiOnr0KLVs2ZL09fXpwoULlJGRQUOHDiVDQ0OqVasWrVmzRiGnDRs2UIMGDahSpUpUs2ZN6tu3b6H5b9myhapWrUoHDhwQ/sfHx4eSk5Plnvf777+Tk5MTVapUierVq0ehoaFynWQikYi+//576tGjBxkYGNCSJUsUXis2NpYSExNpw4YN1Lp1a7K2tqa2bdvS4sWLqXXr1sLzkpOTqV+/flS1alWqVq0a9ezZk+7fvy88LpFIaPr06VS1alWqXr06BQYG0rBhw6hXr17Cc+rWrUtr166Ve/3mzZvTggULhL9fv35No0aNInNzczI2NqYOHTpQVFSU8PiCBQuoefPmtG3bNqpbty6ZmJjQgAED6O3bt8JzpFIprVy5kmxtbUlfX5/q1Kkjt+7FrQtjnxrulGKMMcYYY4wxFfLy8qKzZ88Kf589e5Y8PT3Jw8NDWJ6ZmUmXL18WOqUCAwNp79699Msvv9D169fJ1taWfHx86OXLl3Kxg4KCaPny5RQfH09NmzalgIAAOnfuHP3+++904sQJioiIoOvXrwvP/+eff2jy5Mm0cOFCunXrFh07dozc3d2LzP/du3e0ZMkS2rp1K/3111/0+vVrGjBggPD4+fPnaejQoTRlyhSKi4ujjRs30pYtWxQ6nhYsWEB+fn508+ZNGjlypMLrmJubk5aWFu3Zs4ckEkmBueTm5pKPjw8ZGRnR+fPn6a+//iJDQ0Pq0qUL5eTkEBHRmjVraMuWLbR582a6cOECvXz5kvbv31/kOhbkiy++oGfPntHRo0fp2rVr5OTkRB07dpTbB4mJiXTgwAE6fPgwHT58mM6dO0fLly8XHp89ezYtX76c5s2bR3FxcbRjxw6qWbOm0uvC2CcHjDHGGGOMMcZUZtOmTTAwMEBubi7S0tKgo6ODZ8+eYceOHXB3dwcAnD59GkSEBw8eID09Hbq6uti+fbsQIycnB5aWlli5ciUA4OzZsyAiHDhwQHjO27dvoaenh99++01YlpqaisqVK2PKlCkAgL1798LY2BhpaWlK5f7zzz+DiHDp0iVhWXx8PIgIly9fBgB07NgRS5culfu/bdu2oVatWsLfRISpU6cW+3rffvstqlSpAiMjI3h5eWHhwoVITEyUi9uwYUNIpVJhWXZ2NipXrozjx48DAGrVqiVsJwDIzc2FlZUVevbsKSyztrbG//73P7nXbtasGUJCQgAA58+fh7GxMbKysuSeU79+fWzcuBEAEBISgipVqshty4CAALi6ugIA0tLSoK+vj02bNhW4rsqsC2OfGh4pxRjTaFw9hjHGGGMVjaenJ2VkZNDVq1fp/PnzZGdnR+bm5uTh4SHMKxUREUH16tWjOnXqUGJiIuXm5lLbtm2FGLq6utSqVSuKj4+Xi+3s7Cz8npiYSDk5OeTq6iosq1atGjVs2FD429vbm6ytralevXo0ZMgQ2r59O717967I/HV0dMjFxUX4297enqpWrSrkEhUVRQsXLiRDQ0PhZ/To0ZSSkiIXO3+uhZkwYQI9efKEtm/fTm5ubrR7925q3LgxnTx5Unitu3fvkpGRkfBa1apVo6ysLEpMTKQ3b95QSkqK3DbQ0dFR6rXzi4qKovT0dKpevbrceiUlJclNRF+3bl0yMjIS/q5VqxY9e/aMiIji4+MpOzubOnbsWOhrFLUujH2KdNSdAGNMc4WFhVFAQAC9evWKdHTyPi7S09PJ1NSU2rZtqzBXgpeXF929e7dUk3V+DJKSkmju3LkUERFBL1++JDMzM2rZsiWtWLGC7O3t1Z0eY4wxxsqJra0tWVlZ0dmzZ+nVq1fk4eFBRESWlpZUu3Zt+vvvv+ns2bPUoUOHEsc2MDAo0fONjIzo+vXrFBERQSdOnKD58+fTggUL6OrVq1S1atUSvz5RXnswNDSUevfurfBYpUqVSpyrkZERde/enbp3706LFy8mHx8fWrx4MXl7e1N6ejq1bNmStm/frvB/5ubmSuespaVFAOSW5Z98PT09XWEuMJn820lXV1fuMZFIRFKplIiIKleuXGQOqloXxj4mPFKKMVYorh5TMK4ewxhjjLHieHl5UUREBEVERJCnp6ew3N3dnY4ePUpXrlwR5pOqX78+6enp0V9//SU8Lzc3l65evUqNGjUq9DXq169Purq6dPnyZWHZq1ev6Pbt23LP09HRoU6dOtHKlSspOjqa7t+/T2fOnCk0rlgslmv/3bp1i16/fk0ODg5EROTk5ES3bt0iW1tbhR8trQ87xRSJRGRvb08ZGRnCa925c4dq1Kih8FomJiZkYmJCtWrVktsGYrGYrl27JhfX3NycUlJShL/T0tIoKSlJ+NvJyYmePHlCOjo6Cq9jZmamVO4NGjSgypUr0+nTpwt8vLh1YexTxJ1SjLFCcfWYPFw9hqvHMMYYYyXl5eVFFy5coMjISGGkFBGRh4cHbdy4kXJycoS2k4GBAY0bN44CAgLo2LFjFBcXR6NHj6Z3796Rv79/oa9haGhI/v7+FBAQQGfOnKGYmBgaPny4XMfQ4cOHad26dRQZGUkPHjygrVu3klQqlbvF7326uro0adIkunz5Ml27do2GDx9OrVu3platWhER0fz582nr1q0UGhpKsbGxFB8fT+Hh4RQcHFyibRQZGUk9e/akPXv2UFxcHN29e5d++ukn2rx5M/Xs2ZOIiAYNGkRmZmbUs2dPOn/+PCUlJVFERARNnjyZHj58SEREU6ZMoeXLl9OBAwcoISGBxo8fr3BRsEOHDrRt2zY6f/483bx5k4YNG0ba2trC4506dSI3Nzfq1asXnThxgu7fv09///03zZ07V66DriiVKlWiWbNmUWBgIG3dupUSExPp0qVL9NNPPym9Lox9arhTijFWJK4ek4erx3D1GMYYY6wkvLy8KDMzk2xtbYXvT6K8Tqm3b98KF/9kli9fTn369KEhQ4aQk5MT3b17l44fP06mpqZFvs6qVauoffv21L17d+rUqRO1a9eOWrZsKTxetWpV2rdvH3Xo0IEcHBwoLCyMdu7cSY0bNy40ZpUqVWjWrFk0cOBAatu2LRkaGtKuXbuEx318fOjw4cN04sQJcnFxodatW9P//vc/sra2LtE2srKyorp161JoaCi5urqSk5MTffPNNxQaGkpz584Vcvnzzz+pTp061Lt3b3JwcCB/f3/KysoiY2NjIiKaMWMGDRkyhIYNG0Zubm5kZGREfn5+cq81e/Zs8vDwoG7dupGvry/16tVLboS/SCSiI0eOkLu7O40YMYLs7OxowIAB9ODBA7n9V5x58+bRjBkzaP78+eTg4ED9+/cX5pxSZl0Y++Soe6Z1xphm4+oxXD2mIFw9hjHGGPs4/fzzzzAxMVF3Gh9s2LBhcu0nxphm4pFSjLEicfUYxVwLw9VjuHoMY4wxxhhjTHlcfY8xViSuHlOyXLl6DFePYYwxxhhjjCmHR0oxxorF1WNKh6vHMMYYY6yiGT58+EdROXjLli104MABdafBGCsGd0oxxorF1WOKx9VjuHoMY4wxxhhjrGT49j3GWLFk1WPs7e2Vrh4jlUppyJAh9PbtW3J2dla6ekx6ejp1796djIyMaMaMGfTmzRvhcVn1mAULFlBWVhY1aNCgRNVjHj16RO3btxc6Voj+qx6zcOFCWrFiBenq6pK9vT2NGjWqRNsof/WY+/fvk0gkEv6eNm2akMuff/5Js2bNot69e9Pbt2/ps88+o44dO8pVj0lJSaFhw4aRlpYWjRw5kvz8/OS2w+zZsykpKYm6detGJiYmtGjRIrmRUrLqMXPnzqURI0bQ8+fPycLCgtzd3UtcPUZHR4fmz59Pjx8/plq1atHYsWOVXhfGGGOMMcYYK4oI709MwhhjH4ktW7bQ1KlTK/wQdNkweh6CzhhjjDHGGPuY8O17jDHGGGOMMcYYY6zccacUY4wxxhhjjDHGGCt3fPseY4wxxhhjjDHGGCt3PFKKMcYYY4wxxhhjjJU77pRijDHGGGOMMcYYY+WOO6UYY4wxxhhjjDHGWLnjTinGGGOMMcYYY4wxVu64U4oxxhhjjDHGGGOMlTvulGKMMcYYY4wxxhhj5Y47pRhjjDHGGGOMMcZYueNOKcYYY4wxxhhjjDFW7rhTijHGGGOMMcYYY4yVu/8DsSXsPxnun+YAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1200x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_set[\"text_length\"] = train_set[\"text\"].apply(lambda x: len(x.split()))\n",
    "train_set[\"summary_length\"] = train_set[\"summary\"].apply(lambda x: len(x.split()))\n",
    "\n",
    "text_length_min, text_length_max = train_set[\"text_length\"].min(), train_set[\"text_length\"].max()\n",
    "summary_length_min, summary_length_max = train_set[\"summary_length\"].min(), train_set[\"summary_length\"].max()\n",
    "\n",
    "text_ticks = range(0, text_length_max + 1, 750)\n",
    "summary_ticks = range(0, summary_length_max + 1, 10)\n",
    "\n",
    "plt.figure(figsize=(12, 6))\n",
    "\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.hist(train_set[\"text_length\"], bins=30, color=\"orange\", edgecolor=\"black\", range=(text_length_min, text_length_max))\n",
    "plt.title(\"Text Word Counts\")\n",
    "plt.xlabel(\"Words per Sequence\")\n",
    "plt.ylabel(\"Number of Sequences\")\n",
    "plt.xticks(ticks=text_ticks, rotation=45)\n",
    "\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.hist(train_set[\"summary_length\"], bins=30, color=\"blue\", edgecolor=\"black\", range=(summary_length_min, summary_length_max))\n",
    "plt.title(\"Summary Word Counts\")\n",
    "plt.xlabel(\"Words per Sequence\")\n",
    "plt.ylabel(\"Number of Sequences\")\n",
    "plt.xticks(ticks=summary_ticks, rotation=45)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Clean the Datasets\n",
    "Remove unnecessary features and clean the datasets of duplicates and non-representative samples."
   ],
   "metadata": {
    "id": "xVwzWZ0lCg4i"
   },
   "id": "xVwzWZ0lCg4i"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e471ca56-34b3-429b-a5c5-06858eb7edf9",
   "metadata": {
    "id": "e471ca56-34b3-429b-a5c5-06858eb7edf9",
    "outputId": "7b78ed82-b7f1-4a00-d111-07d98d918ca3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((37519, 4), (4689, 2), (4689, 2))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns = [\"id\", \"url\", \"title\"]\n",
    "\n",
    "train_set.drop(columns=columns, inplace=True)\n",
    "validation_set.drop(columns=columns, inplace=True)\n",
    "test_set.drop(columns=columns, inplace=True)\n",
    "\n",
    "train_set.shape, validation_set.shape, test_set.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5aac9a1-93a0-4541-9d30-5d6e21fc8250",
   "metadata": {
    "id": "f5aac9a1-93a0-4541-9d30-5d6e21fc8250",
    "outputId": "a6a2ee65-f1a0-46e5-b49d-caf536b78642"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((37425, 4), (4689, 2), (4689, 2))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.drop_duplicates(subset=[\"text\"], inplace=True)\n",
    "validation_set.drop_duplicates(subset=[\"text\"], inplace=True)\n",
    "test_set.drop_duplicates(subset=[\"text\"], inplace=True)\n",
    "\n",
    "train_set.shape, validation_set.shape, test_set.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "748f74ee-6097-42b6-9104-8ba09cf96556",
   "metadata": {
    "id": "748f74ee-6097-42b6-9104-8ba09cf96556",
    "outputId": "3781ecc2-0bb8-42b2-e2d5-5eb49a445050"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>summary</th>\n",
       "      <th>text</th>\n",
       "      <th>text_length</th>\n",
       "      <th>summary_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>بدأت القوات الأوكرانية الانسحاب من شبه جزيرة ا...</td>\n",
       "      <td>وكان الرئيس الأوكراني المؤقت، الكسندر تورتشينو...</td>\n",
       "      <td>525</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"هل سيتم تغيير العبارة الشهيرة للمؤرخ اليوناني...</td>\n",
       "      <td>بحلول عام 2050 ستحتاج مصر إلى 21 مليار متر مكع...</td>\n",
       "      <td>462</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>قالت الشرطة في القطاع الهندي من إقليم كشمير إن...</td>\n",
       "      <td>وذكرت وكالة الأنباء المحلية (جي.إن.إس) أن جماع...</td>\n",
       "      <td>106</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>في عام 816، تجول راهب يدعى كوكاي، في المنحدرات...</td>\n",
       "      <td>ووقع اختياره على واد عمقه 800 متر محاط بثماني ...</td>\n",
       "      <td>1343</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>أكد مصدر في \"الحراك التهامي\" لأبناء محافظة الح...</td>\n",
       "      <td>مسلح حوثي في إب وقال المصدر إن المسلحين الحوثي...</td>\n",
       "      <td>185</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             summary  \\\n",
       "0  بدأت القوات الأوكرانية الانسحاب من شبه جزيرة ا...   \n",
       "1  \"هل سيتم تغيير العبارة الشهيرة للمؤرخ اليوناني...   \n",
       "2  قالت الشرطة في القطاع الهندي من إقليم كشمير إن...   \n",
       "3  في عام 816، تجول راهب يدعى كوكاي، في المنحدرات...   \n",
       "4  أكد مصدر في \"الحراك التهامي\" لأبناء محافظة الح...   \n",
       "\n",
       "                                                text  text_length  \\\n",
       "0  وكان الرئيس الأوكراني المؤقت، الكسندر تورتشينو...          525   \n",
       "1  بحلول عام 2050 ستحتاج مصر إلى 21 مليار متر مكع...          462   \n",
       "2  وذكرت وكالة الأنباء المحلية (جي.إن.إس) أن جماع...          106   \n",
       "3  ووقع اختياره على واد عمقه 800 متر محاط بثماني ...         1343   \n",
       "4  مسلح حوثي في إب وقال المصدر إن المسلحين الحوثي...          185   \n",
       "\n",
       "   summary_length  \n",
       "0               8  \n",
       "1              40  \n",
       "2              32  \n",
       "3              32  \n",
       "4              55  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2b63ab4-485d-4147-9ecf-484bfdaf7adb",
   "metadata": {
    "id": "a2b63ab4-485d-4147-9ecf-484bfdaf7adb",
    "outputId": "44a5ee0e-3b0d-4e6c-9c33-0a6c0cf2100b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>summary</th>\n",
       "      <th>text</th>\n",
       "      <th>text_length</th>\n",
       "      <th>summary_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>37514</th>\n",
       "      <td>*هذا الفيديو بدون تعليق*</td>\n",
       "      <td>يثور بركان بولسكي تولباشيك شرقي روسيا منذ نوفم...</td>\n",
       "      <td>47</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37515</th>\n",
       "      <td>بي بي سي ترند تقرير مي نعمان</td>\n",
       "      <td>بينما احتدمت أزمة احتجاز الرهائن على يد لاجئ إ...</td>\n",
       "      <td>302</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37516</th>\n",
       "      <td>تداعيات متواصلة</td>\n",
       "      <td>رمضان آخر يستقبله المسلمون في ظل كورونا للعام ...</td>\n",
       "      <td>712</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37517</th>\n",
       "      <td>حملة شامتة</td>\n",
       "      <td>بعض الأصوات المؤيدة للنظم العربية استغلت اقتحا...</td>\n",
       "      <td>736</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37518</th>\n",
       "      <td>*هذا الفيديو بدون تعليق*</td>\n",
       "      <td>تزامنا مع الذكرى التسعين لاكتشاف مقبرة توت عنخ...</td>\n",
       "      <td>65</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            summary  \\\n",
       "37514      *هذا الفيديو بدون تعليق*   \n",
       "37515  بي بي سي ترند تقرير مي نعمان   \n",
       "37516               تداعيات متواصلة   \n",
       "37517                    حملة شامتة   \n",
       "37518      *هذا الفيديو بدون تعليق*   \n",
       "\n",
       "                                                    text  text_length  \\\n",
       "37514  يثور بركان بولسكي تولباشيك شرقي روسيا منذ نوفم...           47   \n",
       "37515  بينما احتدمت أزمة احتجاز الرهائن على يد لاجئ إ...          302   \n",
       "37516  رمضان آخر يستقبله المسلمون في ظل كورونا للعام ...          712   \n",
       "37517  بعض الأصوات المؤيدة للنظم العربية استغلت اقتحا...          736   \n",
       "37518  تزامنا مع الذكرى التسعين لاكتشاف مقبرة توت عنخ...           65   \n",
       "\n",
       "       summary_length  \n",
       "37514               4  \n",
       "37515               7  \n",
       "37516               2  \n",
       "37517               2  \n",
       "37518               4  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08735442-71e5-44e2-b42b-b4307f5a9716",
   "metadata": {
    "id": "08735442-71e5-44e2-b42b-b4307f5a9716",
    "outputId": "a0808f3e-f05c-4310-f78d-39fef82b97af"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "summary                   text                                                                                                                                                                                                                                                                                                                                                                                   text_length  summary_length\n",
       "*هذا الفيديو بدون تعليق*  اجتاح إعصاران منطقة غربي اليابان لكن بدون احداث اصابات بشرية او مادية. واظهرت لقطات بثها تلفزيون فوجي الياباني إعصاراً فوق مياه المحيط في منطقة واكاياما. وحسب خبراء الارصاد الجوية فان الإعصار استمر حوالي عشر دقائق وتلاشى قبل ان يصل الى البر. اما الاعصار الثاني فقد تشكل في واكاياما وامتد الى منطقة إشيكاوا إثر عاصفة محملة بالبرد. وتعتبر الاعاصير ظاهرة نادرة في اليابان.      62           4                 1\n",
       "                          تزامنا مع الذكرى التسعين لاكتشاف مقبرة توت عنخ أمون تسلمت مصر نسخة طبق الأصل من مقبرته. النسخة صممتها ورشة فكتوم الفنية بمدريد، وسيتم حفظها بشكل مؤقت فى القاهرة، لحين انعقاد مؤتمر فريق العمل الخاص بالاتحاد الأوروبى عن السياحة والاستثمار فى 13 و14 نوفمبر. ومن المقرر أن يتم نقلها إلى وادى الملوك أو إلى أحد المتاحف الأثرية الجديدة، حيث يبدأ عرضها على الجمهور قبل إبريل 2013.  65           4                 1\n",
       "                          يثور بركان بولسكي تولباشيك شرقي روسيا منذ نوفمبر/ تشرين ثاني الماضي. وقد طالب المسؤولون سكان المنطقة بتوخي الحذر. ويقذف البركان بأنهار من الحمم البركانية عبر الثلوج في منظر خلاب. ومن غير المعروف إلى متى ستستمر ثور البركان الذي يعتقد أنه ظل خاملا على مذى الثلاثين عاما الماضية.                                                                                                   47           4                 1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.loc[train_set[\"summary\"].str.contains(\"هذا الفيديو بدون تعليق\", na=False)].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44ee3bee-d427-487b-97b4-e128e8a17abb",
   "metadata": {
    "id": "44ee3bee-d427-487b-97b4-e128e8a17abb",
    "outputId": "a48b0af0-43b5-4102-b055-cc93646f70b2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((37422, 4), (4689, 2), (4689, 2))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.drop(train_set[train_set[\"summary\"].str.contains(\"هذا الفيديو بدون تعليق\", na=False)].index, inplace=True)\n",
    "validation_set.drop(validation_set[validation_set[\"summary\"].str.contains(\"هذا الفيديو بدون تعليق\", na=False)].index, inplace=True)\n",
    "test_set.drop(test_set[test_set[\"summary\"].str.contains(\"هذا الفيديو بدون تعليق\", na=False)].index, inplace=True)\n",
    "\n",
    "train_set.shape, validation_set.shape, test_set.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4958f56c-839e-4ae2-a87f-0d6a34b8ed8b",
   "metadata": {
    "id": "4958f56c-839e-4ae2-a87f-0d6a34b8ed8b",
    "outputId": "f1350325-ec60-491b-cc65-3e60fa07351d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['test.json',\n",
       " 'arabic_test.jsonl',\n",
       " 'arabic_val.jsonl',\n",
       " 'validation.json',\n",
       " 'arabic_train.jsonl',\n",
       " 'train.json']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_file = os.path.join(data, \"train.json\")\n",
    "validation_file = os.path.join(data, \"validation.json\")\n",
    "test_file = os.path.join(data, \"test.json\")\n",
    "\n",
    "train_set.to_json(train_file, orient=\"records\")\n",
    "validation_set.to_json(validation_file, orient=\"records\")\n",
    "test_set.to_json(test_file, orient=\"records\")\n",
    "\n",
    "os.listdir(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Build & Train the Tokenizer\n",
    "Combined the data for training and build a Byte Pair Encoding `BPE` tokenizer, as described in the BART paper.\n",
    "\n",
    "Details:\n",
    "- vocabulary size: `49,152`\n",
    "- type: `Byte Level`\n",
    "- language: `Arabic`\n"
   ],
   "metadata": {
    "id": "A8CqgflVDp2Z"
   },
   "id": "A8CqgflVDp2Z"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8041b915-25ed-4156-bb0e-ab4d1e106055",
   "metadata": {
    "id": "8041b915-25ed-4156-bb0e-ab4d1e106055",
    "outputId": "e05c144f-08d5-4209-f009-2677f2eac81b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['test.json',\n",
       " 'texts.txt',\n",
       " 'arabic_test.jsonl',\n",
       " 'arabic_val.jsonl',\n",
       " 'validation.json',\n",
       " 'arabic_train.jsonl',\n",
       " 'train.json']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def extract_text(file_path):\n",
    "    with open(file_path, \"r\", encoding=\"utf-8\") as file:\n",
    "        data = json.load(file)\n",
    "\n",
    "    return [record[\"text\"] for record in data]\n",
    "\n",
    "\n",
    "train_texts = extract_text(train_file)\n",
    "validation_texts = extract_text(validation_file)\n",
    "test_texts = extract_text(test_file)\n",
    "\n",
    "all_texts = train_texts + validation_texts + test_texts\n",
    "texts_file = os.path.join(data, \"texts.txt\")\n",
    "\n",
    "with open(texts_file, \"w\", encoding=\"utf-8\") as file:\n",
    "    for text in all_texts:\n",
    "        file.write(text + \"\\n\")\n",
    "\n",
    "os.listdir(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d00c1a5d-5afd-432a-a042-589a9ea2e02b",
   "metadata": {
    "id": "d00c1a5d-5afd-432a-a042-589a9ea2e02b"
   },
   "outputs": [],
   "source": [
    "files=[texts_file]\n",
    "vocab_size = 768 * 64\n",
    "special_tokens = [\"<s>\", \"<pad>\", \"</s>\", \"<unk>\", \"<mask>\"]\n",
    "\n",
    "tokenizer = ByteLevelBPETokenizer()\n",
    "tokenizer.train(files=files, vocab_size=vocab_size, special_tokens=special_tokens)\n",
    "\n",
    "single = \"<s> $A </s>\"\n",
    "pair = \"<s> $A </s> <s> $B </s>\"\n",
    "special_tokens = [(\"<s>\", tokenizer.token_to_id(\"<s>\")), (\"</s>\", tokenizer.token_to_id(\"</s>\"))]\n",
    "\n",
    "tokenizer.post_processor = TemplateProcessing(single=single, pair=pair, special_tokens=special_tokens)\n",
    "tokenizer.enable_padding(pad_id=tokenizer.token_to_id(\"<pad>\"), pad_token=\"<pad>\")\n",
    "tokenizer.enable_truncation(max_length=768)\n",
    "\n",
    "tokenizer_file = \"tokenizer/\"\n",
    "os.makedirs(tokenizer_file, exist_ok=True)\n",
    "tokenizer_file = os.path.join(tokenizer_file, \"tokenizer.json\")\n",
    "\n",
    "tokenizer.save(tokenizer_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Test the Tokenizer\n",
    "Ensure that our tokenizer works as expected."
   ],
   "metadata": {
    "id": "JhrNhFCqGOmM"
   },
   "id": "JhrNhFCqGOmM"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ac961fd-4507-4aed-8929-dec0ddcd44e9",
   "metadata": {
    "id": "6ac961fd-4507-4aed-8929-dec0ddcd44e9",
    "outputId": "d113aa51-4410-45cc-eec5-f4b50c4919ec"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoded Tokens: ['<s>', 'Ø¨ÙĬ', 'Ø·ÙĦØ¹', 'ĠØ¹Ø¨', 'Ø§ÙĦÙĬ', 'ĠØ£Ø±Ø¬Ø¹', 'ĠØ£ÙĨØ§', 'ĠÙĪ', 'ÙĬØ§Ùĥ', 'ĠØ¥ÙĨØª', 'ĠØŃÙĦ', 'Ø§ÙĦÙĬ', 'ĠØ£Ø±Ø¬Ø¹', 'ĠØ£ÙĨØ§', 'ĠÙĪ', 'ÙĬØ§Ùĥ', 'ĠØ£ÙĨØ§', 'ĠÙĪØ¥', 'ÙĨØª', 'ØĮ', 'ĠÙħÙĦØ§', 'ĠØ¥ÙĨØª', '</s>']\n",
      "\n",
      "Token IDs: [0, 538, 2788, 1684, 567, 40222, 4337, 280, 14999, 16726, 1801, 567, 40222, 4337, 280, 14999, 4337, 877, 564, 295, 22782, 16726, 2]\n",
      "\n",
      "Decoded Text: بيطلع عبالي أرجع أنا وياك إنت حلالي أرجع أنا وياك أنا وإنت، ملا إنت\n",
      "\n"
     ]
    }
   ],
   "source": [
    "text = [\"بيطلع عبالي أرجع أنا وياك إنت حلالي أرجع أنا وياك أنا وإنت، ملا إنت\"]\n",
    "\n",
    "encoded = tokenizer.encode_batch(text)\n",
    "decoded = [tokenizer.decode(encoding.ids) for encoding in encoded]\n",
    "\n",
    "for index, encoding in enumerate(encoded):\n",
    "    print(f\"Encoded Tokens: {encoding.tokens}\\n\")\n",
    "    print(f\"Token IDs: {encoding.ids}\\n\")\n",
    "    print(f\"Decoded Text: {decoded[index]}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Reduce the Sequence Length\n",
    "\n",
    "Minimize the number of words in the source sequences for computational efficiency. Since most sequences have 750 words or fewer, I will set the maximum length to 768 (inclusive)."
   ],
   "metadata": {
    "id": "CfvkyToDEBOY"
   },
   "id": "CfvkyToDEBOY"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d62f5dc3-90df-4cf5-9681-7f55f0cbace3",
   "metadata": {
    "id": "d62f5dc3-90df-4cf5-9681-7f55f0cbace3",
    "outputId": "39da4102-c9c1-4cf5-e46a-ab5dabeed7c6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32473, 2)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set = train_set[train_set[\"text_length\"] <= 768]\n",
    "\n",
    "train_set.drop(columns=[\"text_length\", \"summary_length\"], inplace=True)\n",
    "\n",
    "train_set.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ad14db7-4d3c-4c40-9e71-adaa1d19ea1a",
   "metadata": {
    "id": "4ad14db7-4d3c-4c40-9e71-adaa1d19ea1a",
    "outputId": "99c5326b-987a-4a7f-f05d-79b37c47c55a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>summary</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>بدأت القوات الأوكرانية الانسحاب من شبه جزيرة ا...</td>\n",
       "      <td>وكان الرئيس الأوكراني المؤقت، الكسندر تورتشينو...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"هل سيتم تغيير العبارة الشهيرة للمؤرخ اليوناني...</td>\n",
       "      <td>بحلول عام 2050 ستحتاج مصر إلى 21 مليار متر مكع...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>قالت الشرطة في القطاع الهندي من إقليم كشمير إن...</td>\n",
       "      <td>وذكرت وكالة الأنباء المحلية (جي.إن.إس) أن جماع...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>أكد مصدر في \"الحراك التهامي\" لأبناء محافظة الح...</td>\n",
       "      <td>مسلح حوثي في إب وقال المصدر إن المسلحين الحوثي...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>افادت مصادر حركة حماس أن كتائب القسام، الجناح ...</td>\n",
       "      <td>وكانت اذاعة الجيش الاسرائيلي افادت بمقتل ثلاثة...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             summary  \\\n",
       "0  بدأت القوات الأوكرانية الانسحاب من شبه جزيرة ا...   \n",
       "1  \"هل سيتم تغيير العبارة الشهيرة للمؤرخ اليوناني...   \n",
       "2  قالت الشرطة في القطاع الهندي من إقليم كشمير إن...   \n",
       "4  أكد مصدر في \"الحراك التهامي\" لأبناء محافظة الح...   \n",
       "8  افادت مصادر حركة حماس أن كتائب القسام، الجناح ...   \n",
       "\n",
       "                                                text  \n",
       "0  وكان الرئيس الأوكراني المؤقت، الكسندر تورتشينو...  \n",
       "1  بحلول عام 2050 ستحتاج مصر إلى 21 مليار متر مكع...  \n",
       "2  وذكرت وكالة الأنباء المحلية (جي.إن.إس) أن جماع...  \n",
       "4  مسلح حوثي في إب وقال المصدر إن المسلحين الحوثي...  \n",
       "8  وكانت اذاعة الجيش الاسرائيلي افادت بمقتل ثلاثة...  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8f8bcda-b3f6-4da0-b206-735f84d3d986",
   "metadata": {
    "id": "e8f8bcda-b3f6-4da0-b206-735f84d3d986",
    "outputId": "22ca364e-33fc-458f-959d-d96bc20b5410"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>summary</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>37512</th>\n",
       "      <td>\"ثلاثون عاما من الألم\"</td>\n",
       "      <td>أظهر العديد من مشجعي ليفربول خلال مباراة فريقه...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37513</th>\n",
       "      <td>لماذا التشكيك؟</td>\n",
       "      <td>مع بدء عمليات التلقيح البعض يتخوفون من بعض الل...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37515</th>\n",
       "      <td>بي بي سي ترند تقرير مي نعمان</td>\n",
       "      <td>بينما احتدمت أزمة احتجاز الرهائن على يد لاجئ إ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37516</th>\n",
       "      <td>تداعيات متواصلة</td>\n",
       "      <td>رمضان آخر يستقبله المسلمون في ظل كورونا للعام ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37517</th>\n",
       "      <td>حملة شامتة</td>\n",
       "      <td>بعض الأصوات المؤيدة للنظم العربية استغلت اقتحا...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            summary  \\\n",
       "37512        \"ثلاثون عاما من الألم\"   \n",
       "37513                لماذا التشكيك؟   \n",
       "37515  بي بي سي ترند تقرير مي نعمان   \n",
       "37516               تداعيات متواصلة   \n",
       "37517                    حملة شامتة   \n",
       "\n",
       "                                                    text  \n",
       "37512  أظهر العديد من مشجعي ليفربول خلال مباراة فريقه...  \n",
       "37513  مع بدء عمليات التلقيح البعض يتخوفون من بعض الل...  \n",
       "37515  بينما احتدمت أزمة احتجاز الرهائن على يد لاجئ إ...  \n",
       "37516  رمضان آخر يستقبله المسلمون في ظل كورونا للعام ...  \n",
       "37517  بعض الأصوات المؤيدة للنظم العربية استغلت اقتحا...  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6c1df1a-7199-4b11-abd3-98bd70f244d6",
   "metadata": {
    "id": "c6c1df1a-7199-4b11-abd3-98bd70f244d6",
    "outputId": "e839a12b-72c9-42c1-b6a1-3c93a909ec5f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['test.json', 'validation.json', 'train.json']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_data = \"cleaned_data/\"\n",
    "os.makedirs(cleaned_data, exist_ok=True)\n",
    "\n",
    "train_file = os.path.join(cleaned_data, \"train.json\")\n",
    "validation_file = os.path.join(cleaned_data, \"validation.json\")\n",
    "test_file = os.path.join(cleaned_data, \"test.json\")\n",
    "\n",
    "train_set.to_json(train_file, orient=\"records\")\n",
    "validation_set.to_json(validation_file, orient=\"records\")\n",
    "test_set.to_json(test_file, orient=\"records\")\n",
    "\n",
    "os.listdir(cleaned_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Tokenize & Dataload\n",
    "Tokenize the sequences into tensors of ids for the model (model inputs), which will use them to map to embeddings."
   ],
   "metadata": {
    "id": "KtuIfioxEMmu"
   },
   "id": "KtuIfioxEMmu"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f92f0fa-513d-45b7-b61a-9b9ad5bbd303",
   "metadata": {
    "id": "4f92f0fa-513d-45b7-b61a-9b9ad5bbd303"
   },
   "outputs": [],
   "source": [
    "tokenizer_file = \"tokenizer/tokenizer.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "243bba22-727d-4b54-b5f0-b3a78cbb0ef3",
   "metadata": {
    "id": "243bba22-727d-4b54-b5f0-b3a78cbb0ef3"
   },
   "outputs": [],
   "source": [
    "train_file = os.path.join(\"cleaned_data/\", \"train.json\")\n",
    "validation_file = os.path.join(\"cleaned_data/\", \"validation.json\")\n",
    "test_file = os.path.join(\"cleaned_data/\", \"test.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d8c04f4-4626-4483-a776-74797facc887",
   "metadata": {
    "id": "5d8c04f4-4626-4483-a776-74797facc887"
   },
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer.from_file(tokenizer_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2ae0c42-c6a5-4492-9c11-0b37d8b78175",
   "metadata": {
    "id": "c2ae0c42-c6a5-4492-9c11-0b37d8b78175",
    "outputId": "87919a6d-d63f-45c5-d33f-29a1e040a6ee"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Dataset({\n",
       "     features: ['summary', 'text'],\n",
       "     num_rows: 32473\n",
       " }),\n",
       " Dataset({\n",
       "     features: ['summary', 'text'],\n",
       "     num_rows: 4689\n",
       " }),\n",
       " Dataset({\n",
       "     features: ['summary', 'text'],\n",
       "     num_rows: 4689\n",
       " }))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set = pd.read_json(train_file)\n",
    "validation_set = pd.read_json(validation_file)\n",
    "test_set = pd.read_json(test_file)\n",
    "\n",
    "train_set = train_set.reset_index(drop=True)\n",
    "validation_set = validation_set.reset_index(drop=True)\n",
    "test_set = test_set.reset_index(drop=True)\n",
    "\n",
    "train = Dataset.from_pandas(train_set)\n",
    "validation = Dataset.from_pandas(validation_set)\n",
    "test = Dataset.from_pandas(test_set)\n",
    "\n",
    "train, validation, test"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Batch the Datasets\n",
    "\n",
    "Batch the dataset into batches of tensors."
   ],
   "metadata": {
    "id": "8bHS8KQIEuBj"
   },
   "id": "8bHS8KQIEuBj"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "178f605f-f77f-47d7-92e7-94d36e868cc7",
   "metadata": {
    "id": "178f605f-f77f-47d7-92e7-94d36e868cc7"
   },
   "outputs": [],
   "source": [
    "def collator(batch):\n",
    "    source = [sample[\"text\"] for sample in batch]\n",
    "    target = [sample[\"summary\"] for sample in batch]\n",
    "\n",
    "    source_encodings = tokenizer.encode_batch(source)\n",
    "    target_encodings = tokenizer.encode_batch(target)\n",
    "\n",
    "    source_input_ids = [torch.tensor(encoding.ids) for encoding in source_encodings]\n",
    "    target_input_ids = [torch.tensor(encoding.ids) for encoding in target_encodings]\n",
    "\n",
    "    max_length = max(max(len(ids) for ids in source_input_ids), max(len(ids) for ids in target_input_ids))\n",
    "\n",
    "    source_input_ids_padded = [torch.nn.functional.pad(ids, (0, max_length - len(ids)), value=tokenizer.token_to_id(\"<pad>\")) for ids in source_input_ids]\n",
    "    target_input_ids_padded = [torch.nn.functional.pad(ids, (0, max_length - len(ids)), value=tokenizer.token_to_id(\"<pad>\")) for ids in target_input_ids]\n",
    "\n",
    "    source_input_ids_tensor = torch.stack(source_input_ids_padded)\n",
    "    target_input_ids_tensor = torch.stack(target_input_ids_padded)\n",
    "\n",
    "    return source_input_ids_tensor, target_input_ids_tensor\n",
    "\n",
    "\n",
    "train_loader = DataLoader(dataset=train, batch_size=16, collate_fn=collator, shuffle=True, pin_memory=True)\n",
    "validation_loader = DataLoader(dataset=validation, batch_size=16, collate_fn=collator, pin_memory=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Test the Dataloader\n",
    "\n",
    "Ensure that our dataloaders works as expected."
   ],
   "metadata": {
    "id": "tPJjQvALGIPm"
   },
   "id": "tPJjQvALGIPm"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8c8457a-2cc9-41a5-8b4d-816f6906eea6",
   "metadata": {
    "id": "b8c8457a-2cc9-41a5-8b4d-816f6906eea6",
    "outputId": "333ec044-015e-408f-c84b-cb52de7ade63"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Largest length in the batch: 582\n",
      "Source batch length: [582, 582, 582, 582, 582, 582, 582, 582, 582, 582, 582, 582, 582, 582, 582, 582]\n",
      "Target batch length: [582, 582, 582, 582, 582, 582, 582, 582, 582, 582, 582, 582, 582, 582, 582, 582]\n",
      "Padding token counts in source batch: [176, 122, 372, 357, 324, 179, 345, 0, 272, 460, 264, 379, 13, 5, 198, 342]\n",
      "Padding token counts in target batch: [561, 547, 556, 542, 548, 565, 562, 539, 553, 534, 550, 551, 546, 566, 561, 554]\n"
     ]
    }
   ],
   "source": [
    "for batch in train_loader:\n",
    "    source_batch, target_batch = batch\n",
    "\n",
    "    source_pad_counts = [(ids == tokenizer.token_to_id(\"<pad>\")).sum().item() for ids in source_batch]\n",
    "    target_pad_counts = [(ids == tokenizer.token_to_id(\"<pad>\")).sum().item() for ids in target_batch]\n",
    "\n",
    "    source_lengths = [len(ids) for ids in source_batch]\n",
    "    target_lengths = [len(ids) for ids in target_batch]\n",
    "\n",
    "    print(\"Largest length in the batch:\", max(source_lengths))\n",
    "    print(\"Source batch length:\", source_lengths)\n",
    "    print(\"Target batch length:\", target_lengths)\n",
    "    print(\"Padding token counts in source batch:\", source_pad_counts)\n",
    "    print(\"Padding token counts in target batch:\", target_pad_counts)\n",
    "\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## BART Architecture\n",
    "\n",
    "Input Embedding `->` Positional Encoding `->` Dropout `->`\n",
    "\n",
    "**Encoder**:\n",
    "\n",
    "Multi Head Attention `->` Dropout `->` Add & Norm `->`\n",
    "\n",
    "Feed Forward `->` Dropout `->` Add & Norm\n",
    "\n",
    "<br>\n",
    "\n",
    "Output Embedding `->` Positional Encoding `->` Dropout `->`\n",
    "\n",
    "**Decoder**:\n",
    "\n",
    "Masked Multi Head Attention `->` Dropout `->` Add & Norm `->`\n",
    "\n",
    "Cross Multi Head Attention `->` Dropout `->` Add & Norm `->`\n",
    "\n",
    "Feed Forward `->` Dropout `->` Add & Norm\n"
   ],
   "metadata": {
    "id": "bu32Zmh3GSy_"
   },
   "id": "bu32Zmh3GSy_"
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Emdedding Layer\n",
    "Transform each token into a numerical representation that captures its semantic meaning (token to vector)."
   ],
   "metadata": {
    "id": "nULc3kyhGZJf"
   },
   "id": "nULc3kyhGZJf"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f0a1119-c599-4105-b09f-2f078ef39346",
   "metadata": {
    "id": "4f0a1119-c599-4105-b09f-2f078ef39346"
   },
   "outputs": [],
   "source": [
    "class Embedding(nn.Module):\n",
    "  def __init__(self, vocab_size: int, d_model: int):\n",
    "    super().__init__()\n",
    "\n",
    "    self.embedding = nn.Embedding(vocab_size, d_model)\n",
    "\n",
    "\n",
    "  def forward(self, sequence: Tensor):\n",
    "    return self.embedding(sequence)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Positional Encoding Layer\n",
    "Transform token embeddings into position aware representations by adding positional encodings that indicate each token's position in the sequence."
   ],
   "metadata": {
    "id": "4JF1dIh1GelX"
   },
   "id": "4JF1dIh1GelX"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "855eb083-3f3c-4b1c-b863-83466a57843f",
   "metadata": {
    "id": "855eb083-3f3c-4b1c-b863-83466a57843f"
   },
   "outputs": [],
   "source": [
    "class PositionalEncoding(nn.Module):\n",
    "  def __init__(self, max_sequence_length: int, d_model: int):\n",
    "    super().__init__()\n",
    "\n",
    "    pe = torch.zeros(max_sequence_length, d_model)\n",
    "    pos = torch.arange(0, max_sequence_length, dtype=torch.float).unsqueeze(1)\n",
    "\n",
    "    _2i = torch.arange(0, d_model, 2, dtype=torch.float)\n",
    "\n",
    "    pe[:, 0::2] = torch.sin(pos / 10000**(_2i / d_model))\n",
    "    pe[:, 1::2] = torch.cos(pos / 10000**(_2i / d_model))\n",
    "\n",
    "    self.register_buffer(\"pe\", pe.unsqueeze(0))\n",
    "\n",
    "\n",
    "  def forward(self, embedding: Tensor):\n",
    "    return embedding + self.pe[:, :embedding.shape[1]]"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Multi Head Self-Attention Layer\n",
    "Transform token embeddings (semantic meaning) into contextual representations (contextualizing the embeddings) so that each token will acquire meaning according to its role in the sequence."
   ],
   "metadata": {
    "id": "UeyTSVN8GoJK"
   },
   "id": "UeyTSVN8GoJK"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb4f12cb-e105-4d77-be59-7ffd37e982ad",
   "metadata": {
    "id": "cb4f12cb-e105-4d77-be59-7ffd37e982ad"
   },
   "outputs": [],
   "source": [
    "class MultiHeadAttention(nn.Module):\n",
    "  def __init__(self, d_model: int, h: int):\n",
    "    super().__init__()\n",
    "\n",
    "    self.d_model = d_model\n",
    "    self.h = h\n",
    "    self.d_k = d_model // h\n",
    "\n",
    "    self.W_q = nn.Linear(d_model, d_model)\n",
    "    self.W_k = nn.Linear(d_model, d_model)\n",
    "    self.W_v = nn.Linear(d_model, d_model)\n",
    "    self.W_o = nn.Linear(d_model, d_model)\n",
    "\n",
    "\n",
    "  def split(self, matrix: Tensor):\n",
    "    batch_size, sequence_length, d_model = matrix.shape\n",
    "\n",
    "    return matrix.view(batch_size, sequence_length, self.h, self.d_k).transpose(1, 2)\n",
    "\n",
    "\n",
    "  def concat(self, matrix: Tensor):\n",
    "    batch_size, h, sequence_length, d_k = matrix.shape\n",
    "\n",
    "    return matrix.transpose(1, 2).contiguous().view(batch_size, sequence_length, h * d_k)\n",
    "\n",
    "\n",
    "  def attention(self, Q: Tensor, K: Tensor, V: Tensor, mask):\n",
    "    scaled_scores = torch.matmul(Q, K.transpose(-1, -2)) / math.sqrt(self.d_k)\n",
    "\n",
    "    if mask is not None:\n",
    "        scaled_scores = scaled_scores.masked_fill(mask == 0, 1e-9)\n",
    "\n",
    "    weights = torch.softmax(scaled_scores, dim=-1)\n",
    "\n",
    "    return torch.matmul(weights, V)\n",
    "\n",
    "\n",
    "  def forward(self, Q: Tensor, K: Tensor, V: Tensor, mask=None):\n",
    "    Q = self.split(self.W_q(Q))\n",
    "    K = self.split(self.W_k(K))\n",
    "    V = self.split(self.W_v(V))\n",
    "\n",
    "    head = self.attention(Q, K, V, mask)\n",
    "\n",
    "    multi_head = self.concat(head)\n",
    "\n",
    "    return self.W_o(multi_head)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Position-Wise Feed Forward Networks\n",
    "\n",
    "Apply feed forward to each token's contextual representation, enabling independent transformations that enhance feature learning."
   ],
   "metadata": {
    "id": "KwA22JIXGvfC"
   },
   "id": "KwA22JIXGvfC"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83db03d0-74d5-4f29-b18a-78171e8ba391",
   "metadata": {
    "id": "83db03d0-74d5-4f29-b18a-78171e8ba391"
   },
   "outputs": [],
   "source": [
    "class PositionWiseFeedForward(nn.Module):\n",
    "  def __init__(self, d_model: int, d_ff: int):\n",
    "    super().__init__()\n",
    "\n",
    "    self.fc1 = nn.Linear(d_model, d_ff)\n",
    "    self.fc2 = nn.Linear(d_ff, d_model)\n",
    "    self.act = nn.GELU()\n",
    "\n",
    "\n",
    "  def forward(self, contextual_embedding: Tensor):\n",
    "    return self.fc2(self.act(self.fc1(contextual_embedding)))"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Dropout Layer\n",
    "Prevent overfitting and enhance model generalization."
   ],
   "metadata": {
    "id": "59_bZIsNG1G7"
   },
   "id": "59_bZIsNG1G7"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c19ce614-0161-40ab-bf1c-67c70455bf13",
   "metadata": {
    "id": "c19ce614-0161-40ab-bf1c-67c70455bf13"
   },
   "outputs": [],
   "source": [
    "class Dropout(nn.Module):\n",
    "  def __init__(self, dropout: float):\n",
    "    super().__init__()\n",
    "\n",
    "    self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "\n",
    "  def forward(self, sublayer: Tensor):\n",
    "    return self.dropout(sublayer)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Normalization Layer\n",
    "Standardize the inputs by scaling and shifting to improve training stability and convergence speed."
   ],
   "metadata": {
    "id": "jZvPBBYjG36Q"
   },
   "id": "jZvPBBYjG36Q"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d86a356-ac90-4878-abc3-86ae2c04530c",
   "metadata": {
    "id": "7d86a356-ac90-4878-abc3-86ae2c04530c"
   },
   "outputs": [],
   "source": [
    "class Normalization(nn.Module):\n",
    "  def __init__(self, d_model: int):\n",
    "    super().__init__()\n",
    "\n",
    "    self.norm = nn.LayerNorm(d_model)\n",
    "\n",
    "\n",
    "  def forward(self, residual_sum: Tensor):\n",
    "    return self.norm(residual_sum)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Encoder Block\n",
    "Consists of multi-head attention, feed forward networks, dropout, skip connections (ResNet), and normalization layers."
   ],
   "metadata": {
    "id": "ZmtENgaoG8ia"
   },
   "id": "ZmtENgaoG8ia"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d2880ed-19e5-46cd-a85b-a49e29add323",
   "metadata": {
    "id": "6d2880ed-19e5-46cd-a85b-a49e29add323"
   },
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "  def __init__(self, d_model: int, h: int, d_ff: int, dropout: float):\n",
    "    super().__init__()\n",
    "\n",
    "    self.self_attention = MultiHeadAttention(d_model, h)\n",
    "    self.feed_forward = PositionWiseFeedForward(d_model, d_ff)\n",
    "    self.dropout = Dropout(dropout)\n",
    "    self.norm = Normalization(d_model)\n",
    "\n",
    "\n",
    "  def forward(self, embedding):\n",
    "    contextual_embedding = self.self_attention(embedding, embedding, embedding)\n",
    "    contextual_embedding = self.dropout(contextual_embedding)\n",
    "    contextual_embedding = self.norm(embedding + contextual_embedding)\n",
    "\n",
    "    return self.norm(contextual_embedding + self.dropout(self.feed_forward(contextual_embedding)))"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Decoder Block\n",
    "Consists of masked multi-head attention, multi-head attention over encoder outputs (cross attention), feed forward networks, dropout, skip connections (ResNet), and normalization layers."
   ],
   "metadata": {
    "id": "2QsTZKSYG_Ue"
   },
   "id": "2QsTZKSYG_Ue"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b7f1019-7e65-42c8-8a72-e5a70cbf5e9e",
   "metadata": {
    "id": "2b7f1019-7e65-42c8-8a72-e5a70cbf5e9e"
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "  def __init__(self, d_model: int, h: int, d_ff: int, dropout: float):\n",
    "    super().__init__()\n",
    "\n",
    "    self.self_attention = MultiHeadAttention(d_model, h)\n",
    "    self.cross_attention = MultiHeadAttention(d_model, h)\n",
    "    self.feed_forward = PositionWiseFeedForward(d_model, d_ff)\n",
    "    self.dropout = Dropout(dropout)\n",
    "    self.norm = Normalization(d_model)\n",
    "\n",
    "\n",
    "  def forward(self, embedding, encoder_output, encoder_mask, decoder_mask):\n",
    "    contextual_embedding = self.self_attention(embedding, embedding, embedding, decoder_mask)\n",
    "    contextual_embedding = self.dropout(contextual_embedding)\n",
    "    contextual_embedding = self.norm(embedding + contextual_embedding)\n",
    "\n",
    "    cross_contextual_embedding = self.cross_attention(contextual_embedding, encoder_output, encoder_output, encoder_mask)\n",
    "    cross_contextual_embedding = self.dropout(cross_contextual_embedding)\n",
    "    cross_contextual_embedding = self.norm(contextual_embedding + cross_contextual_embedding)\n",
    "\n",
    "    return self.norm(cross_contextual_embedding + self.dropout(self.feed_forward(cross_contextual_embedding)))"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Transformer (All Parts Together)\n",
    "Integrates embeddings, positional encoding, and dropout with stacked encoder and decoder blocks."
   ],
   "metadata": {
    "id": "D8nPNEpCHDMi"
   },
   "id": "D8nPNEpCHDMi"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0d034be-0f3f-479f-af96-f540abf97078",
   "metadata": {
    "id": "b0d034be-0f3f-479f-af96-f540abf97078"
   },
   "outputs": [],
   "source": [
    "class ArabBart(nn.Module):\n",
    "  def __init__(self, pad_idx: int, vocab_size: int, d_model: int, max_sequence_length: int, dropout: float, d_ff: int, h: int, n: int):\n",
    "    super().__init__()\n",
    "\n",
    "    self.pad_idx = pad_idx\n",
    "\n",
    "    self.embedding = Embedding(vocab_size, d_model)\n",
    "    self.pe = PositionalEncoding(max_sequence_length, d_model)\n",
    "    self.dropout = Dropout(dropout)\n",
    "\n",
    "    self.encoders = nn.ModuleList([Encoder(d_model, h, d_ff, dropout) for _ in range(n)])\n",
    "    self.decoders = nn.ModuleList([Decoder(d_model, h, d_ff, dropout) for _ in range(n)])\n",
    "\n",
    "    self.pre_softmax = nn.Linear(d_model, vocab_size)\n",
    "\n",
    "\n",
    "  def masking(self, source_sequence: Tensor, target_sequence: Tensor):\n",
    "    device = source_sequence.device\n",
    "\n",
    "    source_mask = (source_sequence != self.pad_idx).unsqueeze(1).unsqueeze(2).to(device)\n",
    "    target_mask = (target_sequence != self.pad_idx).unsqueeze(1).unsqueeze(3).to(device)\n",
    "\n",
    "    sequence_length = target_sequence.shape[1]\n",
    "    ones = torch.ones(sequence_length, sequence_length, device=device)\n",
    "\n",
    "    future_mask = torch.tril(ones, diagonal=0).bool()\n",
    "    target_mask = target_mask & future_mask\n",
    "\n",
    "    return source_mask, target_mask\n",
    "\n",
    "\n",
    "  def forward(self, source_sequence: Tensor, target_sequence: Tensor):\n",
    "    input_embedding = self.dropout(self.pe(self.embedding(source_sequence)))\n",
    "    output_embedding = self.dropout(self.pe(self.embedding(target_sequence)))\n",
    "\n",
    "    encoder_mask, decoder_mask = self.masking(source_sequence, target_sequence)\n",
    "\n",
    "    encoder_embedding = input_embedding\n",
    "    decoder_embedding = output_embedding\n",
    "\n",
    "    for encoder in self.encoders:\n",
    "       encoder_embedding = encoder(encoder_embedding)\n",
    "\n",
    "    for decoder in self.decoders:\n",
    "       decoder_embedding = decoder(decoder_embedding, encoder_embedding, encoder_mask, decoder_mask)\n",
    "\n",
    "    return self.pre_softmax(decoder_embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Configurations and Hyperparameters\n",
    "\n",
    "**Model**:\n",
    "\n",
    "d_model (embedding size): `768`\n",
    "\n",
    "d_ff (hidden size): `3072`\n",
    "\n",
    "number of heads (h): `12`\n",
    "\n",
    "number of layers (n): `6`\n",
    "\n",
    "dropout (regularization): `0.1`\n",
    "\n",
    "max sequence length (positional encoding): `768`\n",
    "\n",
    "pad token: `1`\n",
    "\n",
    "vocab size: `49,152`\n",
    "\n",
    "device: `cuda (1 x L4OS)`\n",
    "\n",
    "**Loss**:\n",
    "\n",
    "type: `label smoothed cross entropy loss`\n",
    "\n",
    "**Optimizer**:\n",
    "\n",
    "learning rate: `0.000001 (1e-6)`\n",
    "\n",
    "beta 0: `0.9`\n",
    "\n",
    "beta 1: `0.98`\n",
    "\n",
    "epsilon: `0.000000001 (1e-9)`\n"
   ],
   "metadata": {
    "id": "u4ElXlZiHevl"
   },
   "id": "u4ElXlZiHevl"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60fe2e3f-a835-4a80-84d4-fc0dca10d951",
   "metadata": {
    "id": "60fe2e3f-a835-4a80-84d4-fc0dca10d951",
    "outputId": "2bbd162e-5c3e-467c-f687-3f226709f1bb"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ArabBart(\n",
       "  (embedding): Embedding(\n",
       "    (embedding): Embedding(49152, 768)\n",
       "  )\n",
       "  (pe): PositionalEncoding()\n",
       "  (dropout): Dropout(\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (encoders): ModuleList(\n",
       "    (0-5): 6 x Encoder(\n",
       "      (self_attention): MultiHeadAttention(\n",
       "        (W_q): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (W_k): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (W_v): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (W_o): Linear(in_features=768, out_features=768, bias=True)\n",
       "      )\n",
       "      (feed_forward): PositionWiseFeedForward(\n",
       "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        (act): GELU(approximate='none')\n",
       "      )\n",
       "      (dropout): Dropout(\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "      (norm): Normalization(\n",
       "        (norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (decoders): ModuleList(\n",
       "    (0-5): 6 x Decoder(\n",
       "      (self_attention): MultiHeadAttention(\n",
       "        (W_q): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (W_k): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (W_v): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (W_o): Linear(in_features=768, out_features=768, bias=True)\n",
       "      )\n",
       "      (cross_attention): MultiHeadAttention(\n",
       "        (W_q): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (W_k): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (W_v): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (W_o): Linear(in_features=768, out_features=768, bias=True)\n",
       "      )\n",
       "      (feed_forward): PositionWiseFeedForward(\n",
       "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        (act): GELU(approximate='none')\n",
       "      )\n",
       "      (dropout): Dropout(\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "      (norm): Normalization(\n",
       "        (norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (pre_softmax): Linear(in_features=768, out_features=49152, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pad_idx = tokenizer.token_to_id(\"<pad>\")\n",
    "vocab_size = tokenizer.get_vocab_size()\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "bart = ArabBart(pad_idx=pad_idx, vocab_size=vocab_size, d_model=768, max_sequence_length=768, dropout=0.1, d_ff=3072, h=12, n=6).to(device)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=pad_idx, label_smoothing=0.1)\n",
    "optimizer = Adam(bart.parameters(), lr=1e-6, betas=(0.9, 0.98), eps=1e-9)\n",
    "\n",
    "bart"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Setup Trainer\n",
    "Prepare the train function for training stage."
   ],
   "metadata": {
    "id": "V07NWMIFH1pa"
   },
   "id": "V07NWMIFH1pa"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42007523-4bea-4769-98f9-a6aecb00bd39",
   "metadata": {
    "id": "42007523-4bea-4769-98f9-a6aecb00bd39"
   },
   "outputs": [],
   "source": [
    "def train(model, criterion, optimizer, train_loader, device, clip_value):\n",
    "    model.train()\n",
    "\n",
    "    total_loss = 0\n",
    "    scaler = GradScaler()\n",
    "\n",
    "    for source_input_ids, target_input_ids in train_loader:\n",
    "        source_input_ids, target_input_ids = source_input_ids.to(device), target_input_ids.to(device)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        with autocast():\n",
    "            output = model(source_input_ids, target_input_ids[:, :-1])\n",
    "\n",
    "            loss = criterion(output.view(-1, output.size(-1)), target_input_ids[:, 1:].reshape(-1))\n",
    "\n",
    "        scaler.scale(loss).backward()\n",
    "        clip_grad_norm_(model.parameters(), clip_value)\n",
    "        scaler.step(optimizer)\n",
    "        scaler.update()\n",
    "\n",
    "        total_loss += loss.item()\n",
    "\n",
    "    return total_loss / len(train_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Setup Evaluator\n",
    "Prepare the evaluation function for the evaluation stage, similar to the training function but with some important adjustments."
   ],
   "metadata": {
    "id": "ZvWsCCtcH48S"
   },
   "id": "ZvWsCCtcH48S"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39be17cc-bbc5-43da-84f8-79b3a60ca104",
   "metadata": {
    "id": "39be17cc-bbc5-43da-84f8-79b3a60ca104"
   },
   "outputs": [],
   "source": [
    "def evaluate(model, criterion, validation_loader, device):\n",
    "    model.eval()\n",
    "\n",
    "    total_loss = 0\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for source_input_ids, target_input_ids in validation_loader:\n",
    "            source_input_ids, target_input_ids = source_input_ids.to(device), target_input_ids.to(device)\n",
    "\n",
    "            output = model(source_input_ids, target_input_ids[:, :-1])\n",
    "\n",
    "            loss = criterion(output.view(-1, output.size(-1)), target_input_ids[:, 1:].reshape(-1))\n",
    "\n",
    "            total_loss += loss.item()\n",
    "\n",
    "    return total_loss / len(validation_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Start Training & Evaluating\n",
    "Execute the training and evaluation loops, recording the results for each epoch.\n"
   ],
   "metadata": {
    "id": "HFT5WqEdICK2"
   },
   "id": "HFT5WqEdICK2"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d7d38c2-11ab-4798-8ef3-3da24a6ff918",
   "metadata": {
    "id": "4d7d38c2-11ab-4798-8ef3-3da24a6ff918",
    "outputId": "4be79536-b13d-4448-ab7c-a657a7312be2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Epoch': 1, 'Loss(train)': 10.003258495612684, 'Loss(validation)': 9.725820372704746, 'Epoch Time (hours)': 0.23}\n",
      "{'Epoch': 2, 'Loss(train)': 9.616397437203695, 'Loss(validation)': 9.44553768715891, 'Epoch Time (hours)': 0.22}\n",
      "{'Epoch': 3, 'Loss(train)': 9.362677955627442, 'Loss(validation)': 9.222394797266746, 'Epoch Time (hours)': 0.22}\n",
      "{'Epoch': 4, 'Loss(train)': 9.164095487265751, 'Loss(validation)': 9.054170621495668, 'Epoch Time (hours)': 0.22}\n",
      "{'Epoch': 5, 'Loss(train)': 9.013892841808902, 'Loss(validation)': 8.928206524881375, 'Epoch Time (hours)': 0.22}\n",
      "Total training time: 1.1 hours\n"
     ]
    }
   ],
   "source": [
    "epochs = 5\n",
    "results = []\n",
    "\n",
    "start_time = time.time()\n",
    "for epoch in range(epochs):\n",
    "    epoch_start_time = time.time()\n",
    "\n",
    "    train_loss = train(bart, criterion, optimizer, train_loader, device, clip_value=1.0)\n",
    "    validation_loss = evaluate(bart, criterion, validation_loader, device)\n",
    "\n",
    "    epoch_time = (time.time() - epoch_start_time) / 3600\n",
    "\n",
    "    results.append({\"Epoch\": epoch + 1, \"Loss(train)\": train_loss, \"Loss(validation)\": validation_loss, \"Epoch Time (hours)\": round(epoch_time, 2)})\n",
    "\n",
    "    print(results[epoch])\n",
    "total_time = (time.time() - start_time) / 3600\n",
    "\n",
    "print(f\"Total training time: {round(total_time, 2)} hours\")\n",
    "\n",
    "results_df = pd.DataFrame(results)\n",
    "results_df[\"Training Time (hours)\"] = round(total_time, 2)\n",
    "results_df[\"Device\"] = \"1xL4OS\"\n",
    "\n",
    "results_df.to_csv(\"results.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f27725b2-514c-4e50-ae30-6ecbcd6d3254",
   "metadata": {
    "id": "f27725b2-514c-4e50-ae30-6ecbcd6d3254",
    "outputId": "d6004641-79b5-4c78-f6c4-78b513e7459a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "</style>\n",
       "<table id=\"T_b509e\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th id=\"T_b509e_level0_col0\" class=\"col_heading level0 col0\" >Epoch</th>\n",
       "      <th id=\"T_b509e_level0_col1\" class=\"col_heading level0 col1\" >Loss(train)</th>\n",
       "      <th id=\"T_b509e_level0_col2\" class=\"col_heading level0 col2\" >Loss(validation)</th>\n",
       "      <th id=\"T_b509e_level0_col3\" class=\"col_heading level0 col3\" >Epoch Time (hours)</th>\n",
       "      <th id=\"T_b509e_level0_col4\" class=\"col_heading level0 col4\" >Training Time (hours)</th>\n",
       "      <th id=\"T_b509e_level0_col5\" class=\"col_heading level0 col5\" >Device</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td id=\"T_b509e_row0_col0\" class=\"data row0 col0\" >1</td>\n",
       "      <td id=\"T_b509e_row0_col1\" class=\"data row0 col1\" >10.003258</td>\n",
       "      <td id=\"T_b509e_row0_col2\" class=\"data row0 col2\" >9.725820</td>\n",
       "      <td id=\"T_b509e_row0_col3\" class=\"data row0 col3\" >0.230000</td>\n",
       "      <td id=\"T_b509e_row0_col4\" class=\"data row0 col4\" >1.100000</td>\n",
       "      <td id=\"T_b509e_row0_col5\" class=\"data row0 col5\" >1xL4OS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td id=\"T_b509e_row1_col0\" class=\"data row1 col0\" >2</td>\n",
       "      <td id=\"T_b509e_row1_col1\" class=\"data row1 col1\" >9.616397</td>\n",
       "      <td id=\"T_b509e_row1_col2\" class=\"data row1 col2\" >9.445538</td>\n",
       "      <td id=\"T_b509e_row1_col3\" class=\"data row1 col3\" >0.220000</td>\n",
       "      <td id=\"T_b509e_row1_col4\" class=\"data row1 col4\" >1.100000</td>\n",
       "      <td id=\"T_b509e_row1_col5\" class=\"data row1 col5\" >1xL4OS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td id=\"T_b509e_row2_col0\" class=\"data row2 col0\" >3</td>\n",
       "      <td id=\"T_b509e_row2_col1\" class=\"data row2 col1\" >9.362678</td>\n",
       "      <td id=\"T_b509e_row2_col2\" class=\"data row2 col2\" >9.222395</td>\n",
       "      <td id=\"T_b509e_row2_col3\" class=\"data row2 col3\" >0.220000</td>\n",
       "      <td id=\"T_b509e_row2_col4\" class=\"data row2 col4\" >1.100000</td>\n",
       "      <td id=\"T_b509e_row2_col5\" class=\"data row2 col5\" >1xL4OS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td id=\"T_b509e_row3_col0\" class=\"data row3 col0\" >4</td>\n",
       "      <td id=\"T_b509e_row3_col1\" class=\"data row3 col1\" >9.164095</td>\n",
       "      <td id=\"T_b509e_row3_col2\" class=\"data row3 col2\" >9.054171</td>\n",
       "      <td id=\"T_b509e_row3_col3\" class=\"data row3 col3\" >0.220000</td>\n",
       "      <td id=\"T_b509e_row3_col4\" class=\"data row3 col4\" >1.100000</td>\n",
       "      <td id=\"T_b509e_row3_col5\" class=\"data row3 col5\" >1xL4OS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td id=\"T_b509e_row4_col0\" class=\"data row4 col0\" >5</td>\n",
       "      <td id=\"T_b509e_row4_col1\" class=\"data row4 col1\" >9.013893</td>\n",
       "      <td id=\"T_b509e_row4_col2\" class=\"data row4 col2\" >8.928207</td>\n",
       "      <td id=\"T_b509e_row4_col3\" class=\"data row4 col3\" >0.220000</td>\n",
       "      <td id=\"T_b509e_row4_col4\" class=\"data row4 col4\" >1.100000</td>\n",
       "      <td id=\"T_b509e_row4_col5\" class=\"data row4 col5\" >1xL4OS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f9f9926f250>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df.style.hide(axis=\"index\")"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Save & Load the Model\n",
    "Save the model as a `.pt` file with a filename that reflects the parameter count in millions."
   ],
   "metadata": {
    "id": "ZqquVVMRIPHS"
   },
   "id": "ZqquVVMRIPHS"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30fe65f6-fbcd-4ed0-9290-705c8a81daa9",
   "metadata": {
    "id": "30fe65f6-fbcd-4ed0-9290-705c8a81daa9",
    "outputId": "f4e8c5c3-a601-4b08-ef5c-ffdf9fdfd8f6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model saved to models/ArabBart_174M.pt\n"
     ]
    }
   ],
   "source": [
    "def save_model(model, path):\n",
    "    num_params = sum(p.numel() for p in model.parameters()) / 1e6\n",
    "    model_name = f\"ArabBart_{int(num_params)}M.pt\"\n",
    "    model_path = os.path.join(path, model_name)\n",
    "\n",
    "    torch.save(model.state_dict(), model_path)\n",
    "    print(f\"Model saved to {model_path}\")\n",
    "\n",
    "path = \"models/\"\n",
    "os.makedirs(path, exist_ok=True)\n",
    "save_model(bart, path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53c59547-e152-4f45-9e66-013e0d7633fb",
   "metadata": {
    "id": "53c59547-e152-4f45-9e66-013e0d7633fb"
   },
   "outputs": [],
   "source": [
    "def load_model(model, model_path):\n",
    "    model.load_state_dict(torch.load(model_path))\n",
    "    model.eval()\n",
    "\n",
    "    return model\n",
    "\n",
    "pad_idx = tokenizer.token_to_id(\"<pad>\")\n",
    "vocab_size = tokenizer.get_vocab_size()\n",
    "\n",
    "model_path = \"models/ArabBart_174M.pt\"\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "bart = ArabBart(pad_idx=pad_idx, vocab_size=vocab_size, d_model=768, max_sequence_length=768, dropout=0.1, d_ff=3072, h=12, n=6).to(device)\n",
    "model = load_model(bart, model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Inference the Model\n",
    "Test the model by generating a summary from the input text."
   ],
   "metadata": {
    "id": "1hXK8YsRIVQi"
   },
   "id": "1hXK8YsRIVQi"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2536e219-868e-4cb3-9eab-fa2ce06ab790",
   "metadata": {
    "id": "2536e219-868e-4cb3-9eab-fa2ce06ab790",
    "outputId": "c05d6c8f-39df-4d2c-8b37-dcefe513b275"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "طول النص المدخل: 206\n",
      "\n",
      "طول الملخص المتوقع: 12\n",
      "\n",
      "طول الملخص الحقيقي: 36\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def summarize(model, tokenizer, text, device, max_summarization_length):\n",
    "    model.eval()\n",
    "\n",
    "    encoded = tokenizer.encode(text)\n",
    "    input_ids = torch.tensor([encoded.ids]).to(device)\n",
    "\n",
    "    target_ids = torch.tensor([[tokenizer.token_to_id(\"<s>\")]]).to(device)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for _ in range(max_summarization_length):\n",
    "            outputs = model(input_ids, target_ids)\n",
    "            prediction_ids = outputs.argmax(dim=-1).cpu().numpy()[0]\n",
    "            next_token = prediction_ids[-1].item()\n",
    "\n",
    "            target_ids = torch.cat([target_ids, torch.tensor([[next_token]]).to(device)], dim=-1)\n",
    "\n",
    "            if next_token == tokenizer.token_to_id(\"</s>\"):\n",
    "                break\n",
    "\n",
    "    summary = tokenizer.decode(target_ids[0].tolist(), skip_special_tokens=True)\n",
    "    return summary\n",
    "\n",
    "text = test[\"text\"][0]\n",
    "max_summarization_length = 60\n",
    "\n",
    "summary = summarize(model=bart, tokenizer=tokenizer, text=text, device=device, max_summarization_length=max_summarization_length)\n",
    "\n",
    "print(f\"طول النص المدخل: {len(text.split())}\\n\")\n",
    "print(f\"طول الملخص المتوقع: {len(summary.split())}\\n\")\n",
    "print(f\"طول الملخص الحقيقي: {len(test['summary'][0].split())}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3ea8ba9-20e0-402f-8dc1-ee731cb6483d",
   "metadata": {
    "id": "e3ea8ba9-20e0-402f-8dc1-ee731cb6483d"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "colab": {
   "provenance": []
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}