Q1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MOfsnCp5x_Qv"
   },
   "source": [
    "##  CS6910: Fundamentals of Deep Learning\n",
    "#### Assignment 2 - Team 3\n",
    "\n",
    "S Renganathan, CH16B058\t     \n",
    "S Nithya, CH16B113\t\t         \n",
    "Vasistha Singhal, CH16B119"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "id": "BWS-AEODiATe",
    "outputId": "95bb5952-ddfa-47e6-cbc3-2cc5c2e8a8f0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import tarfile\n",
    "import pandas as pd\n",
    "import os\n",
    "import re\n",
    "from torch.utils.data import Dataset, DataLoader, ConcatDataset, random_split\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch.optim as optim\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from io import StringIO\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "njt7AfSTiATp"
   },
   "outputs": [],
   "source": [
    "class DatasetClass(Dataset):\n",
    "    \n",
    "    def __init__(self, folder, filename, label_dict):\n",
    "        \n",
    "        self.data = []\n",
    "        self.filename = filename\n",
    "        tar = tarfile.open(folder + '/' + filename)\n",
    "        for file in tar.getmembers():\n",
    "            f = tar.extractfile(file)\n",
    "            if f != None:\n",
    "                content = pd.read_csv(StringIO(f.read().decode()), sep=' ', header=None).values.ravel()\n",
    "                self.data.append(content)\n",
    "            \n",
    "        self.y = torch.tensor(label_dict[self.filename[:-7]], dtype=torch.long)\n",
    "    \n",
    "    def __getitem__(self, idx):     \n",
    "        \n",
    "        return torch.tensor(self.data[idx], dtype=torch.float), self.y\n",
    "      \n",
    "    def __len__(self):\n",
    "        \n",
    "        return len(self.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JWUBsi9EiATu"
   },
   "outputs": [],
   "source": [
    "def train_test_loader(directory, label_dict, train_fraction=0.8, num_workers=2, batch_size=32):\n",
    "\n",
    "    all_files = list(filter(lambda x: x.endswith('.tar.gz'), os.listdir(directory)))\n",
    "    files = [file for file in all_files if file[:-7] in label_dict.keys()]\n",
    "    \n",
    "    datasets = list(map(lambda x : DatasetClass(directory, x, label_dict), files))\n",
    "    dataset = ConcatDataset(datasets)\n",
    "    N = dataset.cumulative_sizes[-1]\n",
    "    \n",
    "    train_size = int(N*train_fraction)\n",
    "    test_size = N - train_size\n",
    "\n",
    "    train_data, test_data = torch.utils.data.random_split(dataset, [train_size, test_size])\n",
    "\n",
    "    trainloader = DataLoader(train_data, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n",
    "    testloader = DataLoader(test_data, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n",
    "    \n",
    "    return trainloader, testloader, train_size, test_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "n9FTc7eJiATx"
   },
   "outputs": [],
   "source": [
    "label_dict = {'tallbuilding': 0, 'opencountry':1, 'mountain': 2, 'highway': 3, 'coast': 4}\n",
    "# trainloader, testloader = train_test_loader('Data_Set_1(Colored_Images)', label_dict, train_fraction=0.8, num_workers=0)\n",
    "trainloader, testloader, train_size, test_size = train_test_loader('/content/drive/My Drive/Data_Set_1(Colored_Images)', label_dict, train_fraction=0.8, num_workers=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HwjmJE-siAT4"
   },
   "outputs": [],
   "source": [
    "class AutoEncoder(nn.Module):\n",
    "    \n",
    "    def __init__(self, n_features, h_layer_sizes):\n",
    "        super(AutoEncoder, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(n_features, h_layer_sizes[0])\n",
    "        self.fc2 = nn.Linear(h_layer_sizes[0], h_layer_sizes[1])\n",
    "        self.fc3 = nn.Linear(h_layer_sizes[1], h_layer_sizes[2])\n",
    "        self.out = nn.Linear(h_layer_sizes[2], n_features)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        \n",
    "        x = torch.tanh(self.fc1(x)) # Hidden Layer 1 (Tanh)\n",
    "        x = self.fc2(x)    # Hidden Layer 2 (Linear)\n",
    "        x = torch.tanh(self.fc3(x)) # Hidden Layer 3 (Tanh)\n",
    "        x = self.out(x) # Output Layer (Linear)\n",
    "        \n",
    "        return x\n",
    "    \n",
    "    def get_z(self, x):\n",
    "        \n",
    "        z = torch.tanh(self.fc1(x))\n",
    "        z = self.fc2(z)\n",
    "        \n",
    "        return z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "bS8b-mcViAT9",
    "outputId": "8b89196f-5f5d-496b-e6ac-61a6882bc7dd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "criterion = nn.MSELoss()\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rr4qrfn9iAUE"
   },
   "outputs": [],
   "source": [
    "ae1 = AutoEncoder(828, [500, 350, 500])\n",
    "ae1 = ae1.to(device)\n",
    "optimizer1 = optim.SGD(ae1.parameters(), lr=0.0001, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 4539
    },
    "colab_type": "code",
    "id": "O1O6lPmZiAUH",
    "outputId": "7e9925aa-7a7d-4f85-d2a6-605d06f0d1f2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 : Loss = 1.938248417594216\n",
      "inf\n",
      "Epoch 2 : Loss = 1.9155791374770077\n",
      "0.011834165278635199\n",
      "Epoch 3 : Loss = 1.8927836228500712\n",
      "0.012043381161874185\n",
      "Epoch 4 : Loss = 1.870415766130794\n",
      "0.011958761856219814\n",
      "Epoch 5 : Loss = 1.8479591933163726\n",
      "0.01215209345295153\n",
      "Epoch 6 : Loss = 1.8250269944017583\n",
      "0.01256540258580199\n",
      "Epoch 7 : Loss = 1.8012761148539456\n",
      "0.013185585125986393\n",
      "Epoch 8 : Loss = 1.776274201544848\n",
      "0.014075480737913681\n",
      "Epoch 9 : Loss = 1.7498579729687087\n",
      "0.015096212940826899\n",
      "Epoch 10 : Loss = 1.7217293788086283\n",
      "0.016337407322133474\n",
      "Epoch 11 : Loss = 1.6917392530224542\n",
      "0.01772739252375558\n",
      "Epoch 12 : Loss = 1.6597462025555696\n",
      "0.019275869056138695\n",
      "Epoch 13 : Loss = 1.6256525191393767\n",
      "0.02097230682128931\n",
      "Epoch 14 : Loss = 1.5894508714025668\n",
      "0.022776197986456012\n",
      "Epoch 15 : Loss = 1.5511432208798144\n",
      "0.024696398119204092\n",
      "Epoch 16 : Loss = 1.5108479125933216\n",
      "0.026670658211604605\n",
      "Epoch 17 : Loss = 1.468681863763115\n",
      "0.02871013108459525\n",
      "Epoch 18 : Loss = 1.4247781092470349\n",
      "0.03081445049663368\n",
      "Epoch 19 : Loss = 1.3794220685958862\n",
      "0.032880466163134926\n",
      "Epoch 20 : Loss = 1.33281411095099\n",
      "0.034969585977477846\n",
      "Epoch 21 : Loss = 1.285336581143466\n",
      "0.036937818859311314\n",
      "Epoch 22 : Loss = 1.2371813275597312\n",
      "0.03892335950358894\n",
      "Epoch 23 : Loss = 1.1887086331844328\n",
      "0.040777607751905434\n",
      "Epoch 24 : Loss = 1.1402599621902814\n",
      "0.042489145107829814\n",
      "Epoch 25 : Loss = 1.0920869816433296\n",
      "0.04411093745890362\n",
      "Epoch 26 : Loss = 1.0444821864366534\n",
      "0.045577412257344845\n",
      "Epoch 27 : Loss = 0.9977229698137805\n",
      "0.046865931764205254\n",
      "Epoch 28 : Loss = 0.9519875808195635\n",
      "0.04804200171901776\n",
      "Epoch 29 : Loss = 0.9075456898320806\n",
      "0.04896931524814553\n",
      "Epoch 30 : Loss = 0.8645239201459018\n",
      "0.0497635388491254\n",
      "Epoch 31 : Loss = 0.8230252807790582\n",
      "0.050422071272904136\n",
      "Epoch 32 : Loss = 0.78322643448006\n",
      "0.05081397223960965\n",
      "Epoch 33 : Loss = 0.745152940804308\n",
      "0.051094871389296206\n",
      "Epoch 34 : Loss = 0.7088182839480315\n",
      "0.051260891090304496\n",
      "Epoch 35 : Loss = 0.6743684370409357\n",
      "0.051084607485869855\n",
      "Epoch 36 : Loss = 0.6416744765910236\n",
      "0.05095100653465743\n",
      "Epoch 37 : Loss = 0.610807111317461\n",
      "0.050535373118011366\n",
      "Epoch 38 : Loss = 0.5817102884704417\n",
      "0.05001943995786452\n",
      "Epoch 39 : Loss = 0.5543285750529984\n",
      "0.049396178818357714\n",
      "Epoch 40 : Loss = 0.5286262048916384\n",
      "0.048621067066905374\n",
      "Epoch 41 : Loss = 0.5045793991197239\n",
      "0.04765712950997575\n",
      "Epoch 42 : Loss = 0.4820602719079363\n",
      "0.04671433952161133\n",
      "Epoch 43 : Loss = 0.4610315222631801\n",
      "0.045612390106271154\n",
      "Epoch 44 : Loss = 0.4414640590548516\n",
      "0.044324023229028675\n",
      "Epoch 45 : Loss = 0.42316271974281827\n",
      "0.04324894055685282\n",
      "Epoch 46 : Loss = 0.40622446889227076\n",
      "0.041696776407231816\n",
      "Epoch 47 : Loss = 0.3904393389821052\n",
      "0.04042914822906468\n",
      "Epoch 48 : Loss = 0.3757775486870247\n",
      "0.0390172067126127\n",
      "Epoch 49 : Loss = 0.36222109537233\n",
      "0.03742590778916367\n",
      "Epoch 50 : Loss = 0.34963726252317423\n",
      "0.03599110906641908\n",
      "Epoch 51 : Loss = 0.33799121122468595\n",
      "0.03445666902488287\n",
      "Epoch 52 : Loss = 0.32721244001930405\n",
      "0.03294120237221421\n",
      "Epoch 53 : Loss = 0.31725073876706045\n",
      "0.03140008843149748\n",
      "Epoch 54 : Loss = 0.3080427687276494\n",
      "0.029891855853146496\n",
      "Epoch 55 : Loss = 0.29955155605619616\n",
      "0.02834641483171028\n",
      "Epoch 56 : Loss = 0.2916980169036172\n",
      "0.0269235260353996\n",
      "Epoch 57 : Loss = 0.2844637307253751\n",
      "0.02543131301763787\n",
      "Epoch 58 : Loss = 0.2777897681702266\n",
      "0.024025228139644018\n",
      "Epoch 59 : Loss = 0.2716354511000894\n",
      "0.022656531189920108\n",
      "Epoch 60 : Loss = 0.26595835861834605\n",
      "0.021345794549326683\n",
      "Epoch 61 : Loss = 0.26073880019513046\n",
      "0.020018341801486386\n",
      "Epoch 62 : Loss = 0.2559287084097212\n",
      "0.01879465502443237\n",
      "Epoch 63 : Loss = 0.25150246410207316\n",
      "0.0175992085145203\n",
      "Epoch 64 : Loss = 0.24741874499754474\n",
      "0.016505293907981597\n",
      "Epoch 65 : Loss = 0.24367237768389963\n",
      "0.015374608108043456\n",
      "Epoch 66 : Loss = 0.2402208007194779\n",
      "0.014368351758398979\n",
      "Epoch 67 : Loss = 0.23704191500490362\n",
      "0.013410648131612168\n",
      "Epoch 68 : Loss = 0.23412440751086583\n",
      "0.012461355588918654\n",
      "Epoch 69 : Loss = 0.2314401807432825\n",
      "0.011597928929033864\n",
      "Epoch 70 : Loss = 0.22896896065636116\n",
      "0.010792816981993287\n",
      "Epoch 71 : Loss = 0.22669630260630091\n",
      "0.010025121821272602\n",
      "Epoch 72 : Loss = 0.22461261837319893\n",
      "0.009276790628209035\n",
      "Epoch 73 : Loss = 0.2226954874667255\n",
      "0.008608755068554737\n",
      "Epoch 74 : Loss = 0.22092986310070215\n",
      "0.007991786810724424\n",
      "Epoch 75 : Loss = 0.21931047094139175\n",
      "0.007384016606043219\n",
      "Epoch 76 : Loss = 0.21781923113898802\n",
      "0.006846226545773586\n",
      "Epoch 77 : Loss = 0.216451745480299\n",
      "0.006317739113882486\n",
      "Epoch 78 : Loss = 0.21519297937100582\n",
      "0.005849475726264241\n",
      "Epoch 79 : Loss = 0.21403651616790081\n",
      "0.005403111692389044\n",
      "Epoch 80 : Loss = 0.21297605505043818\n",
      "0.004979250447715783\n",
      "Epoch 81 : Loss = 0.21199375628070397\n",
      "0.004633621230021216\n",
      "Epoch 82 : Loss = 0.21110465140505272\n",
      "0.004211678282470886\n",
      "Epoch 83 : Loss = 0.2102770615707745\n",
      "0.003935711428037392\n",
      "Epoch 84 : Loss = 0.20951772921464665\n",
      "0.003624191417948838\n",
      "Epoch 85 : Loss = 0.20882069996812128\n",
      "0.003337931759790961\n",
      "Epoch 86 : Loss = 0.2081807001747868\n",
      "0.0030742513249169836\n",
      "Epoch 87 : Loss = 0.20759292183951905\n",
      "0.0028313987300690666\n",
      "Epoch 88 : Loss = 0.20705114338885652\n",
      "0.0026166407091267785\n",
      "Epoch 89 : Loss = 0.20655576986345375\n",
      "0.0023982555690903384\n",
      "Epoch 90 : Loss = 0.20609827881509601\n",
      "0.002219771319721614\n",
      "Epoch 91 : Loss = 0.20567444170063193\n",
      "0.0020607184390999864\n",
      "Epoch 92 : Loss = 0.20528879152102897\n",
      "0.0018785739676559707\n",
      "Epoch 93 : Loss = 0.20493377982215452\n",
      "0.0017323239691500968\n",
      "Epoch 94 : Loss = 0.20460511981086293\n",
      "0.0016063137207680855\n",
      "Epoch 95 : Loss = 0.20430540632117877\n",
      "0.0014669875608332953\n",
      "Epoch 96 : Loss = 0.20402769744396215\n",
      "0.0013611332220856693\n",
      "Epoch 97 : Loss = 0.2037722156806425\n",
      "0.0012537615222286119\n",
      "Epoch 98 : Loss = 0.20353827794844456\n",
      "0.001149354974189112\n",
      "Epoch 99 : Loss = 0.20332111113450746\n",
      "0.0010680977136379897\n",
      "Epoch 100 : Loss = 0.20312373611060053\n",
      "0.0009716984715142363\n",
      "Epoch 101 : Loss = 0.20293952185999267\n",
      "0.0009077297951600808\n",
      "Epoch 102 : Loss = 0.20277059620076957\n",
      "0.0008330875501093282\n",
      "Epoch 103 : Loss = 0.20261417905038048\n",
      "0.0007719950850536903\n",
      "Epoch 104 : Loss = 0.2024714648723602\n",
      "0.0007048606978284544\n",
      "Epoch 105 : Loss = 0.20233934237198392\n",
      "0.0006529748432876902\n",
      "Epoch 106 : Loss = 0.20221782001582064\n",
      "0.0006009478104044983\n",
      "Epoch 107 : Loss = 0.20210509442470287\n",
      "0.0005577572967106543\n",
      "Epoch 108 : Loss = 0.2019998309287158\n",
      "0.0005211068519370482\n",
      "Epoch 109 : Loss = 0.20190498537637966\n",
      "0.00046975339494131717\n",
      "Epoch 110 : Loss = 0.20181563835252422\n",
      "0.00044271605800620315\n",
      "Epoch 111 : Loss = 0.2017342817376961\n",
      "0.0004032860162752159\n",
      "Epoch 112 : Loss = 0.20165864513678983\n",
      "0.0003750724441045071\n",
      "Epoch 113 : Loss = 0.2015881226821379\n",
      "0.0003498343737399442\n",
      "Epoch 114 : Loss = 0.201522636142644\n",
      "0.00032495872794927153\n",
      "Epoch 115 : Loss = 0.20146419514309277\n",
      "0.00029008131946091825\n",
      "Epoch 116 : Loss = 0.2014067555693063\n",
      "0.00028519189251681985\n",
      "Epoch 117 : Loss = 0.20135608586398032\n",
      "0.0002516422839098878\n",
      "Epoch 118 : Loss = 0.2013077475130558\n",
      "0.00024012166209045463\n",
      "Epoch 119 : Loss = 0.20126319981434132\n",
      "0.00022134050713481073\n",
      "Epoch 120 : Loss = 0.20122198692776944\n",
      "0.00020481303858051018\n",
      "Epoch 121 : Loss = 0.20118322562087668\n",
      "0.00019266669362287645\n",
      "Epoch 122 : Loss = 0.20114643126726145\n",
      "0.00018292322356114172\n",
      "Epoch 123 : Loss = 0.20111390000039886\n",
      "0.00016175543740400527\n",
      "Epoch 124 : Loss = 0.2010819353163242\n",
      "0.00015896347936163174\n",
      "Epoch 125 : Loss = 0.20105418732220476\n",
      "0.00013801251537715934\n",
      "Epoch 126 : Loss = 0.20102571560577914\n",
      "0.00014163221028621923\n",
      "Epoch 127 : Loss = 0.20100015672770416\n",
      "0.00012715849823742887\n",
      "Epoch 128 : Loss = 0.2009765397418629\n",
      "0.00011751115762868541\n",
      "Epoch 129 : Loss = 0.20095361532135447\n",
      "0.00011407816909267264\n",
      "Epoch 130 : Loss = 0.20093307881192732\n",
      "0.00010220571718993514\n",
      "Epoch 131 : Loss = 0.20091237838972706\n",
      "0.00010303208974065783\n",
      "Epoch 132 : Loss = 0.20089377293532537\n",
      "9.261339527770009e-05\n",
      "Converged\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "old_loss = np.inf\n",
    "losses = []\n",
    "max_epoch = 500\n",
    "\n",
    "for epoch in range(max_epoch):\n",
    "\n",
    "    running_loss = 0.0\n",
    "    \n",
    "    for data in trainloader:\n",
    "        \n",
    "        X, _ = data[0].to(device), data[1].to(device)\n",
    "        \n",
    "        optimizer1.zero_grad()\n",
    "        \n",
    "        # Reconstructed Representation of X (forward)\n",
    "        X_hat = ae1(X)\n",
    "        \n",
    "        # Calculate Loss (MSE)\n",
    "        loss = criterion(X_hat, X)\n",
    "        \n",
    "        # Backpropagation\n",
    "        loss.backward()\n",
    "        \n",
    "        # Update Parameters\n",
    "        optimizer1.step()\n",
    "        \n",
    "        running_loss += loss.item()*len(X)/train_size\n",
    "    \n",
    "    print('Epoch', epoch+1, ': Loss =', running_loss)\n",
    "    losses.append(running_loss)\n",
    "    print(abs(running_loss-old_loss)/running_loss)\n",
    "    \n",
    "    if abs(running_loss-old_loss)/running_loss < 1e-4:\n",
    "        print('Converged')\n",
    "        break\n",
    "    \n",
    "    old_loss = running_loss\n",
    "\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "colab_type": "code",
    "id": "s_MgLSBL3Qfe",
    "outputId": "0f9a013e-639a-4508-aec0-0b24bcc94f53"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwV9bnH8c+TnUAISyJLgAQEVFQEjOCCotZa8Fq13tbiikpLbdVu3rb2ttXW9t7bzVZb63aVqr2KWlda677WgkBAQBCUsG9C2HdCkuf+MRM9xJMQQk7mnOT7fr3OKzPzm5nznIGcb2b7jbk7IiIidaVFXYCIiCQnBYSIiMSlgBARkbgUECIiEpcCQkRE4lJAiIhIXAoIkTbAzC41s5cifP+fmtn/RfX+0jQKCGkSM7vEzMrMbIeZrTWz581sZNR1SXzu/rC7n107bmZuZv2jrKk+ZrbMzM6Kug5RQEgTmNl3gduA/wa6AX2AO4Hzo6wrlpllRF1Da6Vt23YoIOSgmFk+cAtwrbs/5e473X2fu//N3b8XzpNtZreZ2ZrwdZuZZYdtp5vZKjO7wczWh3sfV4VtI8zsIzNLj3m/L5jZ3HA4zcxuNLPFZrbRzB43sy5hW0n4V/F4M1sBvGZm6WZ2q5ltMLOlZnZdOE9G7Wcxs/vDGlab2S9q39vMrjSzt83st2a2OVx+TExdXczsz+Hn22xmz8S0nWtms81si5lNMbPBDWxPN7NvmNkiM9tuZj83s8PD5baFnzErZv6vmlm5mW0ys8lm1rPOuq4J17XFzP5kZhb7ecLht8JF5oR7gF9u5LqvNbNFwKI4n6N2+08It8laM/uPBj73eWY2P6zzDTM7Kpz+F4I/OP4W1vb9+tYhLcDd9dKr0S9gNFAFZDQwzy3AO8BhQCEwBfh52HZ6uPwtQCZwDrAL6By2LwY+G7OuvwI3hsPfCtfbC8gG7gEmhW0lgAMPAe2BdsA1wPvh/J2BV8J5MsJlng7X0T6sdTrwtbDtSmAf8FUgHfg6sAawsP054LFwvZnAqHD6UGA9MCJcbhywDMiuZ1s58CzQETga2Au8CvQD8sP6x4XznglsAIaFn/+PwFt11vV3oBPBl2wFMDrm87xdZ97+MeONWffLQBegXZzPUbv9J4Xb89jw/c8K238K/F84PBDYCXw23HbfB8qBrLB9We1yekX8+x51AXql1gu4FPjoAPMsBs6JGf8csCwcPh3YTUzAhF+oJ4bDvwAmhsN54RdJcTi+APhMzHI9wi/xjJgvqH4x7a8RfuGH42eF82QQHBrbG/tlB1wMvB4OXwmUx7Tlhst2D9+3hjDU6nz2uwjDMGbaB4QBEmd+B06JGZ8J/CBm/FbgtnD4fuDXMW0dws9fErOukTHtj/NJuF5JwwHRmHWf2cC/ee32PzJm2q+B+8Ph2ID4CfB4zHxpwGrg9HBcAZEkLx1ikoO1ESg4wHHonsDymPHl4bSP1+HuVTHjuwi+kAAeAS4MD0ldCMxy99p1FQNPh4clthAERjXBl32tlXXqWFlPWzHBX69rY9Z3D8GeRK2PagfcfVc42AHoDWxy981xPnsxcEPtOsP19q7z+etaFzO8O8547bbZb7u6+w6Cf4+ieDWz/3Y9kMase2XdheKInafuv3t971UTLlcUZ16JkAJCDtZUgr+8L2hgnjUEX5S1+oTTDsjd3yf48hgDXEIQGLVWAmPcvVPMK8fdV8euImZ4LcHhpVq966xrL1AQs66O7n50I8pcCXQxs071tP1XnRpz3X1SI9Z7IPttVzNrD3Ql+Ou7JdbdmK6fY7dxff/udd/LwuVq30tdTCcJBYQcFHffCtwE/MnMLjCzXDPLNLMxZvbrcLZJwI/NrNDMCsL5D+Ya+EcIzjecRnAOotbdwH+ZWTFAuP6Grpx6HPiWmRWFX+Y/iPkca4GXgFvNrGN4AvxwMxt1oOLCZZ8H7jSzzuHnPy1s/l/gmvCEu5lZezP7NzPLa/zHr9ck4CozGxLuYf03MM3dlzVhXesIznM097p/Ev6fOBq4iuA8TV2PA/9mZp8xs0zgBoKwnlJPbRIRBYQcNHe/Ffgu8GOCE5ErgeuA2it5fgGUAXOB94BZ4bTGmgSMAl5z9w0x028HJgMvmdl2ghPWIxpYz/8ShMBc4F3gHwQnyKvD9iuALIITwZuBJwjOLzTG5QTH6BcSnEP5NoC7lxGc2L4jXGc5wfH/Q+burxAcv3+SYO/ocGBsE1f3U+DB8DDYRc247jcJPvOrwG/d/VM357n7B8BlBCfCNwCfBz7v7pXhLP9D8AfGloauhJLEq70iQ6TVCy9Tvdvdiw84sxwUMysBlgKZdc4vSQrTHoS0WmbWzszOMbMMMysCbia4tFVEGkEBIa2ZAT8jONTzLsFVTzdFWpFICtEhJhERiUt7ECIiElfCOt0ys94E3R50I7iu+V53v73OPEZwZUptdwtXuvussG0cwVUyAL9w9wcP9J4FBQVeUlLSbJ9BRKS1mzlz5gZ3L4zXlsheGauAG9x9VngN+Ewzezm8EarWGGBA+BpB0E3BCAs6YLsZKCUIl5lmNrmeO1c/VlJSQllZWSI+i4hIq2Rmy+trS9ghJndfW7s34O7bCU4Q1r2V/nzgIQ+8A3Qysx4Effe87O613Rm8TNBJnIiItJAWOQcRXiM9FJhWp6mI/ftuWRVOq296vHVPsODBNWUVFRXNVbKISJuX8IAwsw4Ed2d+2923Nff63f1edy9199LCwriH0UREpAkSGhBhPytPAg+7+1NxZlnN/p179Qqn1TddRERaSMICIrxC6X5ggbv/rp7ZJgNXhJ2anQhsDTtCexE4O+wIrTNwdjhNRERaSCKvYjqFoEOz98xsdjjtPwm6AMbd7yboPO0cgs69dhH0/oi7bzKznwMzwuVucfdNCaxVRETqSFhAuPvbBF0dNDSPA9fW0zYRmJiA0kREpBHa/J3U7s4fX13EvNVboy5FRCSptPmA2Lp7H5Omr+CqB2awctOuAy8gItJGtPmA6JSbxYNXD6eyqoZxE6ezaWflgRcSEWkD2nxAAAzolsd940pZvWU3Vz8wg12Vet6JiIgCInRCSRduHzuUuau2cP0j71JVXRN1SSIikVJAxBh9THd+dv4xvLpwPT96eh56VoaItGWJvA8iJV1+YjHrtu7hjtfL6Zafw3c/OzDqkkREIqGAiOOGsweybtse/vDqIrp1zObSEXrGvYi0PQqIOMyM/77wWCp27OUnz8yjsEM2Zx/dPeqyRERalM5B1CMzPY07Lx3GsUX5XD/pXWYuV08fItK2KCAakJuVwcQrT6BHfg7jHyyjfP2OqEsSEWkxCogD6Nohm4euHkFGmjFu4nTWbdsTdUkiIi1CAdEIfbrm8ucrh7NlVyXjJk5n2559UZckIpJwCohGOrZXPndddjzl63fwtYdmsreqOuqSREQSSgFxEE4bWMivvziYqUs2csPjc6ip0Y10ItJ66TLXg3ThsF6s27aXX72wkG4dc/jJuYOiLklEJCEUEE1wzah+rNu2h/vfXkqvzu246pS+UZckItLsdIipCcyMn5w7iLOO6sYvnlvAO0s2Rl2SiEizU0A0UXqa8bsvH0dxl1yue2QWa7fujrokEZFmpYA4BB1zMrn3iuPZXVnNNf83iz37dGWTiLQeCQsIM5toZuvNbF497d8zs9nha56ZVZtZl7BtmZm9F7aVJarG5tD/sDxuvWgIc1Zu4eZn56uLcBFpNRK5B/EAMLq+Rnf/jbsPcfchwA+BN909tsOjM8L20gTW2CxGH9Oda884nMfKVvLI9BVRlyMi0iwSFhDu/hbQ2B7uLgYmJaqWlvDdzx7BqIGF/HTyfOau2hJ1OSIihyzycxBmlkuwp/FkzGQHXjKzmWY24QDLTzCzMjMrq6ioSGSpDUpPM24fO4SCDtl8c9K77Nir51qLSGqLPCCAzwP/qnN4aaS7DwPGANea2Wn1Lezu97p7qbuXFhYWJrrWBnXKzeK2Lw9hxaZd3Pzs/EhrERE5VMkQEGOpc3jJ3VeHP9cDTwPDI6irSUb068r1Zw7gyVmreHb26qjLERFpskgDwszygVHAszHT2ptZXu0wcDYQ90qoZHX9mf0Z1qcTP3lmnroHF5GUlcjLXCcBU4EjzGyVmY03s2vM7JqY2b4AvOTuO2OmdQPeNrM5wHTgOXd/IVF1JkJGehq3XjSEyuoafvDkXF36KiIpKWF9Mbn7xY2Y5wGCy2Fjpy0BjktMVS2nb0F7fjjmKG6ePJ9HZ6zk4uF9oi5JROSgJMM5iFbr8hOLOaV/V/7ruQV8tFWHmkQktSggEigtzfifLwymqqaGnzw7T4eaRCSlKCASrE/XXL5z1kBefn8dL8z7KOpyREQaTQHRAsaP7MvRPTty0+T5bN2t51mLSGpQQLSAjPQ0fnnhYDbs2MvvX/4w6nJERBpFAdFCju2VzyXD+/DQ1GUsWLst6nJERA5IAdGCvve5I8hvl8lNOmEtIilAAdGCOuVm8YPRRzJj2Waenb0m6nJERBqkgGhhF5X25tiifH71wkJ2V+oJdCKSvBQQLSwtzfjxvx3F2q17uO+fS6IuR0SkXgqICIzo15XRR3fnrjcXqzM/EUlaCoiI/PCcI9lXXcOtL30QdSkiInEpICJS3LU9l59YwhMzV1G+fkfU5YiIfIoCIkLfOONwcjLT+f0runlORJKPAiJCBR2yufqUvjw3dy3z12yNuhwRkf0oICL21dP60TEng9+9pL0IEUkuCoiI5bfL5GujDufVheuZtWJz1OWIiHxMAZEErjy5hIIOWfz2RV3RJCLJQwGRBNpnZ/D10/szZfFGppRviLocEREggQFhZhPNbL2Zzaun/XQz22pms8PXTTFto83sAzMrN7MbE1VjMrl0RB965Ofwm5c+UEd+IpIUErkH8QAw+gDz/NPdh4SvWwDMLB34EzAGGARcbGaDElhnUsjJTOf6Mwfw7ootvP7B+qjLERFJXEC4+1vApiYsOhwod/cl7l4JPAqc36zFJakvlfaiqFM77nitXHsRIhK5qM9BnGRmc8zseTM7OpxWBKyMmWdVOC0uM5tgZmVmVlZRUZHIWhMuMz2Nr43qx6wVW5i2tCnZKiLSfKIMiFlAsbsfB/wReKYpK3H3e9291N1LCwsLm7XAKFxU2puCDlnc+cbiqEsRkTYusoBw923uviMc/geQaWYFwGqgd8ysvcJpbUJOZjpXj+zLWx9W8N4q3V0tItGJLCDMrLuZWTg8PKxlIzADGGBmfc0sCxgLTI6qzihcdmIxeTkZ3PVmedSliEgblpGoFZvZJOB0oMDMVgE3A5kA7n438EXg62ZWBewGxnpwZrbKzK4DXgTSgYnuPj9RdSajjjmZXHFSMXe+sZjy9Tvof1iHqEsSkTbIWtPVMqWlpV5WVhZ1Gc1iw469jPzVa3x+cE9+86Xjoi5HRFopM5vp7qXx2qK+iknqUdAhm7En9OHpd1ezesvuqMsRkTZIAZHEvnpaPwD+9y09u1pEWp4CIokVdWrHBUOLeHTGCjbtrIy6HBFpYxQQSW7Caf3Ys6+GR6Ytj7oUEWljFBBJbmC3PEYNLOTBqcvZW1UddTki0oYoIFLA+JF9qdi+l7/NWRt1KSLShiggUsCpAwoY2K0D97+9VJ34iUiLUUCkADPjKyP7sWDtNqYu3hh1OSLSRiggUsR5Q3pS0CGL+95eGnUpItJGKCBSRE5mOpedWMxrC9ezuGJH1OWISBuggEghl51YTFZGGhO1FyEiLUABkUIKOmTzhSFFPDlrFZt145yIJJgCIsWMP7Uve/bV8LBunBORBFNApJiB3fI4dUABf3lnOfuqa6IuR0RaMQVECrry5BLWbdvLi/M/iroUEWnFFBAp6PQjDqNPl1wenLIs6lJEpBVTQKSg9DTjipOKmbFsM/PX6LnVIpIYCogU9aXje9MuM52HpuhktYgkhgIiReXnZnLB0CKemb1al7yKSEIkLCDMbKKZrTezefW0X2pmc83sPTObYmbHxbQtC6fPNrPW8ZDpBBh3cjF7q2p4rGxl1KWISCuUyD2IB4DRDbQvBUa5+7HAz4F767Sf4e5D6nuYtsCR3TtyYr8u/GXqcqpr1MuriDSvhAWEu78FbGqgfYq7bw5H3wF6JaqW1uzKk0tYvWU3ryxYF3UpItLKJMs5iPHA8zHjDrxkZjPNbEJENaWEs47qRs/8HB6auizqUkSklYk8IMzsDIKA+EHM5JHuPgwYA1xrZqc1sPwEMyszs7KKiooEV5t8MtLTuOykYv5VvpFF67ZHXY6ItCKRBoSZDQbuA85394+fhOPuq8Of64GngeH1rcPd73X3UncvLSwsTHTJSWnsCX3IykjjwanLoi5FRFqRyALCzPoATwGXu/uHMdPbm1le7TBwNhD3SigJdGmfxXnH9eSpWavZtmdf1OWISCuRyMtcJwFTgSPMbJWZjTeza8zsmnCWm4CuwJ11LmftBrxtZnOA6cBz7v5CoupsLa48uYRdldX8tWxV1KWISCuRkagVu/vFB2j/CvCVONOXAMd9eglpyDFF+Rxf3Jm/TF3GVSeXkJZmUZckIiku8pPU0nzGnVzCso27eHNR2ztZLyLNTwHRiow5pjuH5WWrl1cRaRYKiFYkMz2NS0b04Y0PKli6YWfU5YhIilNAtDKXjOhDZrrpxjkROWQKiFbmsLwczjm2B0+UrWLn3qqoyxGRFKaAaIXGnVzC9r1VPPXu6qhLEZEUpoBohYb27sSxRfk8NGUZ7urlVUSaplEBEd7dnBYODzSz88wsM7GlSVOZGeNOLmHR+h1MWbzxwAuIiMTR2D2It4AcMysCXgIuJ3jegySpcwf3oEv7LF3yKiJN1tiAMHffBVwI3OnuXwKOTlxZcqhyMtO5eHhvXlmwjlWbd0VdjoikoEYHhJmdBFwKPBdOS09MSdJcLh1RjJnxl3eWR12KiKSgxgbEt4EfAk+7+3wz6we8nriypDn07NSOswd147EZK9mzrzrqckQkxTQqINz9TXc/z91/FZ6s3uDu30xwbdIMxp1cwpZd+5g8e03UpYhIimnsVUyPmFnH8PkM84D3zex7iS1NmsOIvl04olseD+iSVxE5SI09xDTI3bcBFxA8O7ovwZVMkuRqL3l9f+02ypZvjrocEUkhjQ2IzPC+hwuAye6+D9CfoynigqE96ZiTwQO65FVEDkJjA+IeYBnQHnjLzIqBbYkqSppXblYGXz6hNy/O+4iPtu6JuhwRSRGNPUn9B3cvcvdzPLAcOCPBtUkzuvzEEqrdeXiaLnkVkcZp7EnqfDP7nZmVha9bCfYmJEX06ZrLZ448jIenrdAlryLSKI09xDQR2A5cFL62AX9OVFGSGFeP7MumnZU8o15eRaQRGhsQh7v7ze6+JHz9DOh3oIXMbKKZrTezefW0m5n9wczKzWyumQ2LaRtnZovC17hG1ikNOKlfV47q0ZGJ/1qqS15F5IAaGxC7zWxk7YiZnQLsbsRyDwCjG2gfAwwIXxOAu8L1dwFuBkYAw4GbzaxzI2uVepgZ40f25cN1O/jnog1RlyMiSa6xAXEN8CczW2Zmy4A7gK8daCF3fwvY1MAs5wMPhSe+3wE6mVkP4HPAy+6+yd03Ay/TcNBII33+uB4UdMjm/reXRl2KiCS5xl7FNMfdjwMGA4PdfShwZjO8fxGwMmZ8VTitvumfYmYTak+eV1RUNENJrVt2RjpXnFTMmx9WUL5+e9TliEgSO6gnyrn7tvCOaoDvJqCeg+bu97p7qbuXFhYWRl1OSrh0RB+yMtK4/+1lUZciIknsUB45as3w/quB3jHjvcJp9U2XZtC1QzYXDi3iqVmr2LSzMupyRCRJHUpANMdlMJOBK8KrmU4Etrr7WuBF4Gwz6xyenD47nCbN5OqRfdlbVcMjunFOROqR0VCjmW0nfhAY0O5AKzezScDpQIGZrSK4MikTwN3vBv4BnAOUA7uAq8K2TWb2c2BGuKpb3L2hk91ykAZ2y+PUAQU8NHU5E047nKyMQ/lbQURaowYDwt3zDmXl7n7xAdoduLaetokEN+hJgowf2Zcr/zyDyXPW8MXje0VdjogkGf3Z2IaNGljIkd3zuOfNxdTU6MY5EdmfAqINMzOuGXU4i9bv4NWF66MuR0SSjAKijTt3cA96dW7HXW+Uq/sNEdmPAqKNy0hP46un9mPWii3MWKYnzonIJxQQwkWlvenSPou73iiPuhQRSSIKCKFdVjpXnVzC6x9UsGCtHhQoIgEFhABw+UnF5Galc8+bi6MuRUSShAJCAOiUm8Ulw/vwt7lrWblpV9TliEgSUEDIx8af2pc0g/v+uSTqUkQkCSgg5GM98ttxwZAiHp2xkvXb90RdjohETAEh+7n2jP5U1Th3v6G9CJG2TgEh+ykpaM8Xhhbx8LTlrN+mvQiRtkwBIZ9y/ZnBXsRduqJJpE1TQMinFHdtz4VDi3hk2grtRYi0YQoIiev6MwdQVePc+Yb2IkTaKgWExNWnay7/PqyIR6avYJ32IkTaJAWE1Ou6MwZQU+Pcpb0IkTZJASH1CvYievHI9BV8tFV7ESJtjQJCGnTdmf2pqXHueH1R1KWISAtLaECY2Wgz+8DMys3sxjjtvzez2eHrQzPbEtNWHdM2OZF1Sv16d8nl4uF9mDR9JYsrdkRdjoi0oIQFhJmlA38CxgCDgIvNbFDsPO7+HXcf4u5DgD8CT8U0765tc/fzElWnHNi3zhpAu8x0fv3CwqhLEZEWlMg9iOFAubsvcfdK4FHg/AbmvxiYlMB6pIkKOmRzzah+vDh/HTOWbYq6HBFpIYkMiCJgZcz4qnDap5hZMdAXeC1mco6ZlZnZO2Z2QX1vYmYTwvnKKioqmqNuiWP8yH5065jNf/9jgZ5dLdJGJMtJ6rHAE+5eHTOt2N1LgUuA28zs8HgLuvu97l7q7qWFhYUtUWub1C4rnRs+ewTvrtjC8/M+irocEWkBiQyI1UDvmPFe4bR4xlLn8JK7rw5/LgHeAIY2f4lyMP79+F4c0S2PX72wkMqqmqjLEZEES2RAzAAGmFlfM8siCIFPXY1kZkcCnYGpMdM6m1l2OFwAnAK8n8BapRHS04wbzzmS5Rt38ci05VGXIyIJlrCAcPcq4DrgRWAB8Li7zzezW8ws9qqkscCjvv+B7aOAMjObA7wO/NLdFRBJ4PSBhZzSvyu3v7qIrbv3RV2OiCSQtaYTjqWlpV5WVhZ1Ga3evNVb+fwdbzPupBJ+et7RUZcjIofAzGaG53s/JVlOUksKOaYon8tPLOahqcuYt3pr1OWISIIoIKRJbjj7CLq0z+LHz8yjpqb17IWKyCcUENIk+e0y+c9zjmL2yi08VrbywAuISMpRQEiTfWFoEcP7duFXLyxk087KqMsRkWamgJAmMzN+ccEx7NhTxa+eVz9NIq2NAkIOycBueYwf2ZfHylYyc7n6aRJpTRQQcsi++ZkB9MjP4UdPz9Md1iKtiAJCDln77AxuOf8YFn60nTvfKI+6HBFpJgoIaRafHdSN84f05I7Xynl/zbaoyxGRZqCAkGbz088fTafcTL73xBz2VetQk0iqU0BIs+ncPotfXHAs89ds44+v6VCTSKpTQEizGn1Md/59WC/ueG0RM5dvjrocETkECghpdj89bxA9O7XjO4/NZsfeqqjLEZEmUkBIs8vLyeT3Xx7Cqs27uOnZeVGXIyJNpICQhDihpAvXnzmAp2at5q/qq0kkJSkgJGG++ZkBnNSvKzc9O59F67ZHXY6IHCQFhCRMeppx+9ghtM9O5xsPz9L5CJEUo4CQhDqsYw5/GDuUxRU7+P4Tc2hNTzAUae0UEJJwJ/cv4MYxR/KP9z7i7jeXRF2OiDSSAkJaxFdP7ce5g3vwmxcX8trCdVGXIyKNkNCAMLPRZvaBmZWb2Y1x2q80swozmx2+vhLTNs7MFoWvcYmsUxLPzPj1FwczqGdHrn/kXfXXJJICEhYQZpYO/AkYAwwCLjazQXFmfczdh4Sv+8JluwA3AyOA4cDNZtY5UbVKy8jNyuC+K04gLyeT8Q/OYP22PVGXJCINSOQexHCg3N2XuHsl8ChwfiOX/RzwsrtvcvfNwMvA6ATVKS2oe34O940rZevufYz78wy27dkXdUkiUo9EBkQREHuH1KpwWl3/bmZzzewJM+t9kMtiZhPMrMzMyioqKpqjbkmwY4ryueuy41m0bjsTHipjz77qqEsSkTiiPkn9N6DE3QcT7CU8eLArcPd73b3U3UsLCwubvUBJjFEDC/ntl47jnSWb+Pajs6lS9+AiSSeRAbEa6B0z3iuc9jF33+jue8PR+4DjG7uspL4LhhZx07mDeGH+R9zw1zlU1+geCZFkksiAmAEMMLO+ZpYFjAUmx85gZj1iRs8DFoTDLwJnm1nn8OT02eE0aWWuHtmX748+gmdnr+EHT85VSIgkkYxErdjdq8zsOoIv9nRgorvPN7NbgDJ3nwx808zOA6qATcCV4bKbzOznBCEDcIu7b0pUrRKtb5zen8qqGm57ZRF79lXzu4uGkJUR9dFPEbHW1PVBaWmpl5WVRV2GNNHdby7ml88vZNTAQu6+7HjaZaVHXZJIq2dmM929NF6b/kyTpHHNqMP55YXH8s9FFVx2/zS27tIlsCJRUkBIUhk7vA93XDKM91Zt5cv3TmX9dt1MJxIVBYQknXOO7cHEK09gxaZdfOFPU5i3emvUJYm0SQoISUojBxTw6IQTqXHni3dP4dnZuspZpKUpICRpDe7VicnXjWRwUSe+9ehs/uu593VDnUgLUkBIUivMy+bhr45g3EnF/O8/lzLuz9PZuGPvgRcUkUOmgJCkl5mexs/OP4bffHEwM5Zt5nO3/ZNXF+iZEiKJpoCQlPGl0t5Mvu4UCvOyGf9gGTc+OVfPuRZJIAWEpJQju3fkmWtP5uunH87jZSsZc/tbTF+qm+xFEkEBISknOyOdH4w+kse/dhKGcdE9U/nBE3N1bkKkmSkgJGWVlnTh+W+dytdO68eTs1Zxxm/f4MEpy3Slk0gzUUBISmufncEPzzmKF759KoN7deLmyfM5949vM4d+KgwAAA0HSURBVKV8Q9SliaQ8BYS0Cv0Py+Mv44dz16XD2L6nikvum8bF976j8xMih0C9uUqrs2dfNY9MW8Gdbyxmw469nDqggG+fNZDjiztHXZpI0mmoN1cFhLRauyureXjacu56YzEbd1YyvKQLV51SwmcHdSMjXTvPIqCAkDZuV2UVj0xbwQNTlrFq826KOrXjipOKGXtCH/JzM6MuTyRSCggRoLrGefn9dTwwZSnvLNlEdkYanzu6OxcOK+LUAYWkp1nUJYq0uIYCImGPHBVJNulpxuhjujP6mO68v2Ybk6avYPKcNUyes4bD8rK5YGgR5x3Xk6N7dsRMYSGiPQhp0/ZWVfP6wvU8OWs1ry9cT1WNU9SpHWcf3Y2zB3XnhJLOOl8hrVpkh5jMbDRwO5AO3Ofuv6zT/l3gK0AVUAFc7e7Lw7Zq4L1w1hXuft6B3k8BIYdi085KXlmwjpfmf8RbizZQWVVD59xMTulfwMj+BZzSv4DeXXKjLlOkWUUSEGaWDnwIfBZYBcwALnb392PmOQOY5u67zOzrwOnu/uWwbYe7dziY91RASHPZubeKtz6s4OX31/F2+QbWbw+68SjpmstJhxdQWtyZYcWdKemaq8NRktKiOgcxHCh39yVhEY8C5wMfB4S7vx4z/zvAZQmsR6TR2mdnMObYHow5tgfuTvn6HbxdvoF/lW/g73PWMGn6CgC6tM9iWJ/ODO6Vz1E9OjKoZ0d65ucoNKRVSGRAFAErY8ZXASMamH888HzMeI6ZlREcfvqluz8TbyEzmwBMAOjTp88hFSwSj5kxoFseA7rlcdUpfampcRat38HM5ZuZuXwz767YzCsxz6fIb5fJkd3zPg6M/od1oG/X9nTKzVRwSEpJiquYzOwyoBQYFTO52N1Xm1k/4DUze8/dF9dd1t3vBe6F4BBTixQsbVpamnFE9zyO6J7HJSOCP0p27K3ig4+28f7a7by/ZhsL1m7jsRkr2b2v+uPlOuZk0LegPcVd21NS0J6Srrn06pxLj/wcunXMIStDJ8MluSQyIFYDvWPGe4XT9mNmZwE/Aka5+8f9Nbv76vDnEjN7AxgKfCogRJJBh+wMji/uwvHFXT6eVl3jLN+4k6UbgteyjTtZvnEXs1Zs5u9z11AT8+eMGRR0yKZnfg7d83Pokd+OwrxsurbPokv7LLp2yKJL+2y6dsgiLztDeyLSIhIZEDOAAWbWlyAYxgKXxM5gZkOBe4DR7r4+ZnpnYJe77zWzAuAU4NcJrFWk2aWnGf0KO9Cv8NPXWuytqmblpt2s2bKbtVt3s3brHtZu2cPabXtYUrGTf5VvrPdpeZnpRpf2QWB0zs0kLyeDjjmZ5OUEw7XjHdtlkJeTSYfsDHKz0snJTCc3K512WenkZKSTphsD5QASFhDuXmVm1wEvElzmOtHd55vZLUCZu08GfgN0AP4a/kVUeznrUcA9ZlZD0OPsL2OvfhJJddkZ6fQ/rAP9D6v/Qr09+6rZuLOSTTsq2bhzL5t2VrJxR2UwbedeNu6oZMvufWzYsJPte6rYvqfqoB7BmpOZRm5WBu0y0/cbzs5MIys9jcz0NLIyan/ax9MyM4L2oC2cHs6XnZFGRloa6WmQZkZGupFmRnpa+AqH09KMjLQ6bQ21m4EFe1ppZhifDEPd6Uaaob2sZqAb5URakeoaZ8feKrbv2ce23cHPHXur2L2vml2V1ewJf+6urGb3vuDnJ9Or2FVZTWV1DZVVNeyrrmFftVNZVUNldTAeOz1V1A0VqxMwtaESZtB+AVO3LXadAIbtNx5Mq53n0wH18XJ1lt9/Wuz8tv+0et6nS24Wj19z0gG2RHzqakOkjUhPM/LbZZLfLhMS2Lu5u4eh4ewLAyQ2PKprnBp3qmo+Ga6uiXm5U1MTtNeE47HtwfxQXVMTzh+8Z/DeUOOOxwzX1lTjwTQnGKbOfLXD/vHw/svEttV4OM4n7137/vv9JE5bnfH95qszT23tn55Wd10x71NnIC8nMV/lCggROWhmRnZGOtkZQHbU1Uii6Lo6ERGJSwEhIiJxKSBERCQuBYSIiMSlgBARkbgUECIiEpcCQkRE4lJAiIhIXK2qqw0zqwCWN3HxAmBDM5bTUlK1bkjd2lO1bkjd2lO1bkj+2ovdvTBeQ6sKiENhZmX19UeSzFK1bkjd2lO1bkjd2lO1bkjt2nWISURE4lJAiIhIXAqIT9wbdQFNlKp1Q+rWnqp1Q+rWnqp1QwrXrnMQIiISl/YgREQkLgWEiIjE1eYDwsxGm9kHZlZuZjdGXU9DzKy3mb1uZu+b2Xwz+1Y4vYuZvWxmi8KfCXyWWNOZWbqZvWtmfw/H+5rZtHDbP2ZmWVHXGI+ZdTKzJ8xsoZktMLOTUmGbm9l3wv8n88xskpnlJOs2N7OJZrbezObFTIu7jS3wh/AzzDWzYUlW92/C/ytzzexpM+sU0/bDsO4PzOxz0VTdeG06IMwsHfgTMAYYBFxsZoOirapBVcAN7j4IOBG4Nqz3RuBVdx8AvBqOJ6NvAQtixn8F/N7d+wObgfGRVHVgtwMvuPuRwHEEnyGpt7mZFQHfBErd/RggHRhL8m7zB4DRdabVt43HAAPC1wTgrhaqMZ4H+HTdLwPHuPtg4EPghwDh7+pY4OhwmTvD76Ck1aYDAhgOlLv7EnevBB4Fzo+4pnq5+1p3nxUObyf4oioiqPnBcLYHgQuiqbB+ZtYL+DfgvnDcgDOBJ8JZkrXufOA04H4Ad6909y2kwDYneKRwOzPLAHKBtSTpNnf3t4BNdSbXt43PBx7ywDtAJzPr0TKV7i9e3e7+krtXhaPvAL3C4fOBR919r7svBcoJvoOSVlsPiCJgZcz4qnBa0jOzEmAoMA3o5u5rw6aPgG4RldWQ24DvAzXheFdgS8wvUrJu+75ABfDn8PDYfWbWniTf5u6+GvgtsIIgGLYCM0mNbV6rvm2cSr+3VwPPh8OpVDeggEhJZtYBeBL4trtvi23z4LrlpLp22czOBda7+8yoa2mCDGAYcJe7DwV2UudwUpJu884Ef7H2BXoC7fn0oZCUkYzb+EDM7EcEh4UfjrqWpmrrAbEa6B0z3iuclrTMLJMgHB5296fCyetqd7HDn+ujqq8epwDnmdkygsN4ZxIc1+8UHv6A5N32q4BV7j4tHH+CIDCSfZufBSx19wp33wc8RfDvkArbvFZ92zjpf2/N7ErgXOBS/+Rms6Svu662HhAzgAHhlR1ZBCeQJkdcU73C4/b3Awvc/XcxTZOBceHwOODZlq6tIe7+Q3fv5e4lBNv4NXe/FHgd+GI4W9LVDeDuHwErzeyIcNJngPdJ8m1OcGjpRDPLDf/f1Nad9Ns8Rn3beDJwRXg104nA1phDUZEzs9EEh1PPc/ddMU2TgbFmlm1mfQlOsk+PosZGc/c2/QLOIbjSYDHwo6jrOUCtIwl2s+cCs8PXOQTH818FFgGvAF2irrWBz3A68PdwuB/BL0g58FcgO+r66ql5CFAWbvdngM6psM2BnwELgXnAX4DsZN3mwCSCcyX7CPbaxte3jQEjuPpwMfAewZVayVR3OcG5htrf0btj5v9RWPcHwJiot/uBXupqQ0RE4mrrh5hERKQeCggREYlLASEiInEpIEREJC4FhIiIxKWAkDbHzHaEP0vM7JJmWN8yM3syZvyLZvbAoa43XNdPzew/mmNdIgdLASFtWQlwUAERcxdyXccnW0/A4Y1k+h2XJtN/HmnLfgmcamazw2cnpId9+c8I+/L/GoCZnW5m/zSzyQR3I8dzK8FNUPupuwcQPpuhJHwtNLMHzOxDM3vYzM4ys3+Fzz+I7eXzODObGk7/asy6vhdT68/CaSXhswYeIrhBLrZrB5GDUt9fQyJtwY3Af7j7uQBmNoGg24YTzCwb+JeZvRTOO4ygj/+l9azrceAbZtb/IN6/P/Algh4/ZxDszYwEzgP+k0+6tx5M8PyP9sC7ZvYccAxBVw3DCe4snmxmpxF0sTEAGOdBV9giTaaAEPnE2cBgM6vtqyif4Mu2EpjeQDgAVAO/IXg4zPMNzBdrqbu/B2Bm8wkejuNm9h7B4a9az7r7bmC3mb1OEAojw3rfDefpENa6AliucJDmoIAQ+YQB17v7i/tNNDudoJvvA/kLQUDMi5lWxf6HcnNihvfGDNfEjNew/+9m3f5wPKz1f9z9njq1ljSyVpED0jkIacu2A3kx4y8CXw+7VMfMBoYPB2oUD7rV/j3wnZjJywgOTxE+O7lvE+o834LnSXcl6OxwRljr1eGzQTCzIjM7rAnrFqmX9iCkLZsLVJvZHIJnC99OcGhnVthFdgUH/0jO+4Efx4w/SdA19XyCp/992MQ6XwcKgJ+7+xpgjZkdBUwNSmUHcBnBoS6RZqHeXEVEJC4dYhIRkbgUECIiEpcCQkRE4lJAiIhIXAoIERGJSwEhIiJxKSBERCSu/wezO8c41t17NwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses)\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Iter Number')\n",
    "plt.title('Convergence monitor plot')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "G1w7UZJRiAUL"
   },
   "outputs": [],
   "source": [
    "class FinalNet(nn.Module):\n",
    "    \n",
    "    def __init__(self, input_size, hidden_sizes, num_classes):\n",
    "        super(FinalNet, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(input_size, hidden_sizes[0])\n",
    "        self.fc2 = nn.Linear(hidden_sizes[0], hidden_sizes[1])\n",
    "        # self.fc3 = nn.Linear(hidden_sizes[1], hidden_sizes[2])\n",
    "        #self.fc4 = nn.Linear(hidden_sizes[2], hidden_sizes[3])\n",
    "        self.out = nn.Linear(hidden_sizes[1], num_classes)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \n",
    "        x = torch.tanh(self.fc1(x))\n",
    "        x = torch.tanh(self.fc2(x))\n",
    "        # x = torch.tanh(self.fc3(x))\n",
    "        #x = torch.tanh(self.fc4(x))\n",
    "        x = self.out(x)\n",
    "        \n",
    "        return x\n",
    "    \n",
    "    def predict(self, X):\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            y_score = self.forward(X)\n",
    "            y_pred = torch.argmax(y_score, axis=1)\n",
    "            \n",
    "        return y_pred\n",
    "            \n",
    "    \n",
    "classifier = FinalNet(350, [200, 50], 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_VqwlpwniAUP"
   },
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(classifier.parameters(), lr=0.00001, momentum=0.9)\n",
    "classifier = classifier.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 2074
    },
    "colab_type": "code",
    "id": "FYJn45PGiAUS",
    "outputId": "18423f53-5eab-427f-b4ce-e90246380df8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 : Loss = 1.6237990260124209\n",
      "inf\n",
      "Epoch 501 : Loss = 1.5950649407776922\n",
      "3.013232683095418e-06\n",
      "Epoch 1001 : Loss = 1.5926135426217858\n",
      "2.1434707635386193e-07\n",
      "Epoch 1501 : Loss = 1.5900753519751807\n",
      "1.1794276708278314e-05\n",
      "Epoch 2001 : Loss = 1.5873279490254135\n",
      "5.832244723252967e-06\n",
      "Epoch 2501 : Loss = 1.5842348824847827\n",
      "1.388824477491742e-05\n",
      "Epoch 3001 : Loss = 1.5807421071962877\n",
      "3.45565208806856e-05\n",
      "Epoch 3501 : Loss = 1.5767012861641971\n",
      "2.1240347015329324e-05\n",
      "Epoch 4001 : Loss = 1.5715789849107913\n",
      "1.7194540496793242e-05\n",
      "Epoch 4501 : Loss = 1.5653905895623292\n",
      "3.65863549358868e-05\n",
      "Epoch 5001 : Loss = 1.557466325434771\n",
      "1.2039476603110335e-05\n",
      "Epoch 5501 : Loss = 1.5474964380264282\n",
      "1.822545984164254e-05\n",
      "Epoch 6001 : Loss = 1.5349144989793952\n",
      "7.715674758640677e-05\n",
      "Epoch 6501 : Loss = 1.5195008407939565\n",
      "8.83522607203795e-05\n",
      "Epoch 7001 : Loss = 1.5006075989116323\n",
      "1.4470842230043153e-05\n",
      "Epoch 7501 : Loss = 1.4788770648566159\n",
      "5.4370080997922436e-05\n",
      "Epoch 8001 : Loss = 1.4554963355714625\n",
      "1.6927828745971092e-05\n",
      "Epoch 8501 : Loss = 1.4321904968131673\n",
      "3.093528150120154e-05\n",
      "Epoch 9001 : Loss = 1.4103013033216647\n",
      "5.7350009864978204e-05\n",
      "Epoch 9501 : Loss = 1.3905549320307644\n",
      "0.00024319823032768382\n",
      "Epoch 10001 : Loss = 1.372413250533017\n",
      "0.0007724278975999065\n",
      "Epoch 10501 : Loss = 1.3557464236562902\n",
      "0.0003161003930038155\n",
      "Epoch 11001 : Loss = 1.340420023961501\n",
      "0.000615692019521111\n",
      "Epoch 11501 : Loss = 1.3249724344773721\n",
      "0.0001394389193158254\n",
      "Epoch 12001 : Loss = 1.3090498989278623\n",
      "0.0017105951345142475\n",
      "Epoch 12501 : Loss = 1.2893363697962328\n",
      "0.0001689436359930405\n",
      "Epoch 13001 : Loss = 1.2710706835443324\n",
      "0.00020647314397611883\n",
      "Epoch 13501 : Loss = 1.2523470412601123\n",
      "0.0007647395432672057\n",
      "Epoch 14001 : Loss = 1.234590855511752\n",
      "0.00152968510259775\n",
      "Epoch 14501 : Loss = 1.218685112216256\n",
      "0.0012398059403754145\n",
      "Epoch 15001 : Loss = 1.2003611678426915\n",
      "0.0002470298005259033\n",
      "Epoch 15501 : Loss = 1.1844968281008983\n",
      "0.0007519139554657124\n",
      "Epoch 16001 : Loss = 1.171122019941157\n",
      "0.0009006172021469496\n",
      "Epoch 16501 : Loss = 1.1567574211142282\n",
      "0.001428074656399163\n",
      "Epoch 17001 : Loss = 1.1431817425922917\n",
      "0.0013407467312339257\n",
      "Epoch 17501 : Loss = 1.1283577639948241\n",
      "0.0004744001202419543\n",
      "Epoch 18001 : Loss = 1.1136485771699387\n",
      "0.00022987069826442287\n",
      "Epoch 18501 : Loss = 1.102256860245358\n",
      "0.0010094984588619566\n",
      "Epoch 19001 : Loss = 1.0919735716147856\n",
      "0.003233347392726396\n",
      "Epoch 19501 : Loss = 1.0756619816476651\n",
      "0.0033534030352384407\n",
      "Epoch 20001 : Loss = 1.0652022727511146\n",
      "0.0008076854188708658\n",
      "Epoch 20501 : Loss = 1.0533648905428974\n",
      "0.001973916033861091\n",
      "Epoch 21001 : Loss = 1.0390413091941315\n",
      "0.003897477197856624\n",
      "Epoch 21501 : Loss = 1.032097885554487\n",
      "0.0036670508361219987\n",
      "Epoch 22001 : Loss = 1.0258777236396617\n",
      "0.005655547179677509\n",
      "Epoch 22501 : Loss = 1.0163289714943278\n",
      "0.005465749745708709\n",
      "Epoch 23001 : Loss = 1.001160845160484\n",
      "0.0029714847649306565\n",
      "Epoch 23501 : Loss = 0.9904223097996279\n",
      "0.0014268473257905818\n",
      "Epoch 24001 : Loss = 0.9802167293700307\n",
      "0.0007333813369862982\n",
      "Epoch 24501 : Loss = 0.9728793718598104\n",
      "0.002240359580152229\n",
      "Epoch 25001 : Loss = 0.961608893492005\n",
      "0.0008442415294785883\n",
      "Epoch 25501 : Loss = 0.9549292447892103\n",
      "0.0017081826210830979\n",
      "Epoch 26001 : Loss = 0.9451511082324116\n",
      "0.0008175552601980654\n",
      "Epoch 26501 : Loss = 0.939571892673319\n",
      "0.003515868618253953\n",
      "Epoch 27001 : Loss = 0.9342322661118074\n",
      "0.004329177081685119\n",
      "Epoch 27501 : Loss = 0.9280750141902403\n",
      "0.006960013856843262\n",
      "Epoch 28001 : Loss = 0.9172127233310178\n",
      "0.004046179530746873\n",
      "Epoch 28501 : Loss = 0.9095499800010162\n",
      "0.005600439153876015\n",
      "Epoch 29001 : Loss = 0.9112079400907861\n",
      "0.009327486344572575\n",
      "Epoch 29501 : Loss = 0.890836276791312\n",
      "0.005696762897290346\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "old_loss = np.inf\n",
    "\n",
    "max_epoch = 30000\n",
    "losses= []\n",
    "for epoch in range(max_epoch):\n",
    "\n",
    "    running_loss = 0.0\n",
    "    \n",
    "    for data in trainloader:\n",
    "        \n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        \n",
    "        # extracting encoder features from AE1 to use as input to the MLFFNN\n",
    "        with torch.no_grad():\n",
    "            Z = ae1.get_z(X)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Forward\n",
    "        y_hat = classifier(Z)\n",
    "        \n",
    "        # Calculate Loss (Cross Entropy)\n",
    "        loss = criterion(y_hat, y)\n",
    "        \n",
    "        # Backpropagation\n",
    "        loss.backward()\n",
    "        \n",
    "        # Update Parameters\n",
    "        optimizer.step()\n",
    "        \n",
    "        running_loss += loss.item()*len(X)/train_size\n",
    "    \n",
    "    if epoch%500 == 0:\n",
    "        print('Epoch', epoch+1, ': Loss =', running_loss)\n",
    "        print(abs(running_loss-old_loss)/running_loss)\n",
    "    losses.append(running_loss)\n",
    "\n",
    "    if abs(running_loss-old_loss)/running_loss < 1e-4 and running_loss<0.8:\n",
    "        print('Converged')\n",
    "        break\n",
    "    \n",
    "    old_loss = running_loss\n",
    "\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "colab_type": "code",
    "id": "28vxvQv_3y9E",
    "outputId": "a7530296-d739-4efe-b77f-b67c5a9115d0"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1fnH8c+ThC3sS3BhC4uKaFkkdSsK4gbGira2tVat1taq1Wp/2ha1UupGqrXVVq3autS20mqt1QoILghWEA2yoyBLEBAhEPYg6/P7Y27iZF/IzJ3JfN+v17y499wzd54zE+aZe8+955i7IyIiqSst7ABERCRcSgQiIilOiUBEJMUpEYiIpDglAhGRFKdEICKS4pQIRBoJM/uOmU0J8fXHmtnfwnp9qT8lAqmSmV1sZvlmtsPM1pnZJDMbEnZcUjl3/7u7n1WybmZuZn3CjKkqZlZgZmeEHYdEKBFIpczs/4AHgHuAQ4DuwCPAqDDjimZmGWHH0FjpvU0tSgRSgZm1Be4AfuTu/3b3ne6+193/6+4/Deo0M7MHzOzT4PGAmTULtg0zszVmdpOZbQiOJq4Itp1gZp+ZWXrU611gZvOD5TQzG21my81sk5k9Z2Ydgm3Zwa/cK83sE+BNM0s3s/vNbKOZrTSz64I6GSVtMbMnghjWmtldJa9tZpeb2f/M7Ddmtjl4/siouDqY2VNB+zab2X+itp1rZnPNbIuZzTCz/tW8n25m15rZx2a23czuNLPewfO2BW1sGlX/B2a2zMyKzOxlMzu83L6uDva1xcweNjOLbk+wPD14yrzgiO5btdz3j8zsY+DjStpR8v5fFbwn68zs5mrafZ6ZLQrifMvMjg7K/0rkh8V/g9h+VtU+JE7cXQ89yjyAEcA+IKOaOncA7wKdgSxgBnBnsG1Y8Pw7gCbAOUAx0D7Yvhw4M2pfzwOjg+Ubgv12BZoBjwHjg23ZgAPPAC2BFsDVwOKgfnvg9aBORvCcF4N9tAxifQ/4YbDtcmAv8AMgHbgG+BSwYPsE4J/BfpsAQ4PyQcAG4ITged8FCoBmVbxXDrwEtAGOAXYDbwC9gLZB/N8N6g4HNgLHBe3/AzC93L5eAdoR+TItBEZEted/5er2iVqvzb5fAzoALSppR8n7Pz54P78UvP4ZwfaxwN+C5SOBncCZwXv3M2AZ0DTYXlDyPD3Cf4QegB6J9wC+A3xWQ53lwDlR62cDBcHyMGAXUYkk+OI8MVi+C3gyWG4dfGH0CNY/BE6Pet5hwZd1RtQXUa+o7W8SfLEH62cEdTKInNLaHf2lBnwbmBosXw4si9qWGTz30OB1DxAkr3Jt/yNB0osqW0KQKCqp78BXotZnAz+PWr8feCBYfgK4N2pbq6D92VH7GhK1/Tm+SKKXU30iqM2+h1fzmZe8/32jyu4FngiWoxPB7cBzUfXSgLXAsGBdiSCBHjo1JJXZBHSq4Tzx4cCqqPVVQVnpPtx9X9R6MZEvHoBnga8Fp5K+Bnzg7iX76gG8GJxO2EIkMewn8qVeYnW5OFZXsa0HkV+j66L29xiRI4MSn5UsuHtxsNgK6AYUufvmStreA7ipZJ/BfruVa39566OWd1WyXvLelHlf3X0Hkc+jS2UxU/Z9rUlt9r26/JMqEV2n/Ode1WsdCJ7XpZK6EjIlAqnMTCK/pM+vps6nRL4QS3QPymrk7ouJfEmMBC4mkhhKrAZGunu7qEdzd18bvYuo5XVETguV6FZuX7uBTlH7auPux9QizNVABzNrV8W2u8vFmOnu42ux35qUeV/NrCXQkciv6XjsuzbDEUe/x1V97uVfy4LnlbyWhj1OIEoEUoG7bwXGAA+b2flmlmlmTcxspJndG1QbD/zCzLLMrFNQvy7XkD9LpD/gVCJ9BCUeBe42sx4Awf6ru1LpOeAGM+sSfGn/PKod64ApwP1m1iboiO5tZkNrCi547iTgETNrH7T/1GDzn4Crg45vM7OWZpZrZq1r3/wqjQeuMLOBwRHTPcAsdy+ox77WE+mHaOh93x78TRwDXEGkH6W854BcMzvdzJoANxFJyjOqiE1CpEQglXL3+4H/A35BpENwNXAdUHLlzF1APjAfWAB8EJTV1nhgKPCmu2+MKn8QeBmYYmbbiXQcn1DNfv5E5Mt+PjAHmEiko3p/sP0yoCmRDtnNwL+InP+vjUuJnEP/iEgfx40A7p5PpIP5oWCfy4icnz9o7v46kfPrLxA52ukNXFTP3Y0F/hKcvvpmA+57GpE2vwH8xt0r3MTm7kuAS4h0SG8Evgp81d33BFXGEfkhsaW6K48kPkqujhBpFILLPx919x41VpY6MbNsYCXQpFz/jyQ5HRFIUjOzFmZ2jpllmFkX4JdELhkVkVpSIpBkZ8CviJyimUPkKqMxoUYkkmR0akhEJMXpiEBEJMUl3cBSnTp18uzs7LDDEBFJKrNnz97o7lmVbUu6RJCdnU1+fn7YYYiIJBUzW1XVNp0aEhFJcUoEIiIpTolARCTFKRGIiKQ4JQIRkRSnRCAikuKUCEREUlzKJIIln23n/ilL2LRjd9ihiIgklJRJBCsKd/CHN5dRqEQgIlJGyiSCZk0iTf1874GQIxERSSwpkwiaZ6QDsHvv/hpqioiklpglAjN70sw2mNnCauoMM7O5ZrbIzKbFKhaIOiLYpyMCEZFosTwieBoYUdXGYKLxR4Dz3P0Y4BsxjIVmOiIQEalUzBKBu08HiqqpcjHwb3f/JKi/IVaxADTXEYGISKXC7CM4EmhvZm+Z2WwzuyyWL9a2RVMAJi/8jBnLNrK6qJi9+5UURETCnI8gAxgMnA60AGaa2bvuvrR8RTO7CrgKoHv37vV6sazWzTj1yCwmLFjHhAXrAEgzOKxtC3p0zKRb+0y6dWhBtw6ZdA2Ws1o1w8zq2TwRkeQQZiJYA2xy953ATjObDgwAKiQCd38ceBwgJyen3pMsP335l1m7ZReri4pZs3kXazYXs6qomE+Kinnjow1sLHePQbOMNLp1yKRHh0y6dcgku2MmPTq1pEeQLJpmpMxFVyLSiIWZCF4CHjKzDKApcALwu1i+YFqa0S34Uq/Mrj37WbO5mNWbi1ldFEkUnxQVs2pTMe+u2MTOPV90NKenGd07ZNKrU0t6d25Fn6xWHHFIK446tDWZTZNu4jcRSWEx+8Yys/HAMKCTma0Bfgk0AXD3R939QzN7FZgPHAD+7O5VXmoaDy2apnPEIa054pDWFba5Oxt37GHVpp0UbCpm5cYdrCjcycqNO3n7443sCfobzKBHh0z6HtqGow9rQ9/DWnP0oW3o2r4FaWk6zSQiicfc632mJRQ5OTmeaHMW7z/grC4qZsn67Xy0bjsffbaND9dtY1VRMSVvb+tmGfQ7vA0DurVjcI/2HNe9PVmtm4UbuIikDDOb7e45lW5TIoidnbv3sXT9dj5ct51Fn25l4dqtfPjZdvYEl7AedUhrTu7Tka/07sQJvTrQunmTkCMWkcZKiSCB7N63n/lrtpJfsJkZyzfy3soidu87QHqa0b9rW4b06cTwvp0Z0LWdTiWJSINRIkhgu/ft54NVW3hn2UbeWb6Reau3cMChdfMMvn18d0YceyiDurXTZawiclCUCJLIluI9vPHhBv6Zv5r8giIOOLTLbMLg7u351ahj6Nq+8iueRESqo0SQpLZ9vpdXF37GX2euYsHarQCcdlQWw/t25pITe+goQURqTYmgEVj86Tb+NXsNT76zsrRs9Mi+/PDUXkoIIlIjJYJGZGvxXm745xzeWlJYWvbUFV9m2JFZSggiUqXqEoHGSEgybTOb8PQVx7Ps7pF0adcCgCueep9LnpjFh+u2hRydiCQjHREkud379vPsrE944PWP2bprLwBzbj+T9i2bhhyZiCQSHRE0Ys0y0rniKz2Z9tNhpWWD7nyNp6P6EkREqqNE0Ei0y2xKQV4u//nRVwAY+9/FnH7/W2z7fG/IkYlIolMiaGQGdmvHkrtGkN0xk+WFO+k/dgo/evaDsMMSkQSmRNAINctI562fnsaz3z8BgAnz1zH25UUhRyUiiUqJoBE7uU8nxv/gRACenlHAPRM/ZP+B5Lo4QERiT4mgkTupd0c+vnskl53Ug8enr+D7f3lf/QYiUoYSQQpokp7GHaOO5a7zj2XqkkL6j53Cuys2hR2WiCQIJYIUcsmJPbhySE8ALnr8XT5evz3kiEQkESgRpJjbz+1X2ol85u+mc8u/54cckYiELWaJwMyeNLMNZlbpPMRmNszMtprZ3OAxJlaxSFkn9+nEwxcfB8D491Zz94TFIUckImGK5RHB08CIGuq87e4Dg8cdMYxFysntfxjfD04T/entlfxm8pKQIxKRsMQsEbj7dKAoVvuXg/eLc/vxh28PAuChqcv4+6xVIUckImEIu4/gJDObZ2aTzOyYqiqZ2VVmlm9m+YWFhVVVk3r46oDD+dpxXQC47cWFvLrws5AjEpF4CzMRfAD0cPcBwB+A/1RV0d0fd/ccd8/JysqKW4Cp4rffHEjPTi0BuPpvs1m3dVfIEYlIPIWWCNx9m7vvCJYnAk3MrFNY8aS6qTcPK10+adybFO/ZF14wIhJXoSUCMzvUgim1zOz4IBbd5RSigrzc0uV+YyaTbHNViEj9xPLy0fHATOAoM1tjZlea2dVmdnVQ5UJgoZnNA34PXOT65glddDLoecvEECMRkXjRDGVSwf4DTu9bv0gC0clBRJKTZiiTOklPM9677fTS9V/9V0NYizRmSgRSqc6tm/PYpYMBeOqdAsZN+jDkiEQkVpQIpEpnH3MomU3TAXhs2gq2FO8JOSIRiQUlAqnW4ju+GCVk4B2vhRiJiMSKEoHUKLqzOHv0hBAjEZFYUCKQWvn47pGly0oGIo2LEoHUSpP0tNL5jwEem7Y8xGhEpCEpEUitndS7IxcMigxQN27SR6wo3BFyRCLSEJQIpE5+962BpcvD75/G/gPJdUOiiFSkRCB1Ft15HH0HsogkJyUCqZdl6jwWaTSUCKReMtLTmHTDKaXrF/5xRojRiMjBUCKQejv6sDYc2qY5APmrNvPmR+tDjkhE6kOJQA7Ku7d+MTjd957OZ8O2z0OMRkTqQ4lADlp05/Hx97zBrj37Q4xGROpKiUAaRHQyOHrMqyFGIiJ1pUQgDeajO78YoC579ARNdSmSJJQIpME0b5JO/i/OKF3XVJciySGWcxY/aWYbzGxhDfW+bGb7zOzCWMUi8dOpVTOe/f4Jpes6MhBJfLE8IngaGFFdBTNLB34NTIlhHBJnJ/fpxI1nHFG6riMDkcQWs0Tg7tOBohqqXQ+8AGyIVRwSjhvPOJIvZ7cvXdfdxyKJK7Q+AjPrAlwA/LEWda8ys3wzyy8sLIx9cNIgnr/6ZPod1qZ0XclAJDGF2Vn8APBzdz9QU0V3f9zdc9w9JysrKw6hSUOZeMMpdGnXonRdfQYiiSfMRJAD/MPMCoALgUfM7PwQ45EYeWf08DLr6jMQSSyhJQJ37+nu2e6eDfwLuNbd/xNWPBJb0TecQeTIYNvne0OKRkSixfLy0fHATOAoM1tjZlea2dVmdnWsXlMSW0FeLif16li63n/sFPbur/HMoIjEmCXb+dqcnBzPz88POww5CD99fh7Pz15Tuv7mTUPpldUqxIhEGj8zm+3uOZVt053FEnf3fWMAN591ZOn68Pun8fK8T0OMSCS1KRFIKK4bfgT/vW5I6fqPx89h2H1TQ4xIJHUpEUhovtS1LR/cfmbpesGmYt1rIBICJQIJVYeWTVl618gyZbrXQCS+lAgkdE0z0lhxzzllynreMpH9B5QMROJBiUASQlqaVbjXoPetExk36cOQIhJJHUoEklAK8nLpndWydP2xaSu485XFIUYk0vgpEUjCeeOmYbx47cml60/8byVf/+OMECMSadyUCCQhDerengVjzypdn71qszqRRWJEiUASVuvmTVg5rmIncvGefSFFJNI4KRFIQjOr2Incb8xkDVgn0oCUCCQpFOTl0i6zSel6/7FTdPOZSANRIpCkMXfMWTx40cAyZUoGIgdPiUCSyqiBXXj7Z6eVKcsePYF9Gs5apN6UCCTpdOuQyUd3jihT1ue2SRzQncgi9aJEIEmpeZP0Cp3IvW7VFJgi9aFEIEmt/OWl2aMnsG7rrpCiEUlOsZyq8kkz22BmC6vYPsrM5pvZXDPLN7MhldUTqU5ll5eeNO5NZq3YFFJEIsknlkcETwMjqtn+BjDA3QcC3wP+HMNYpJEryMtlQNe2pevfevxdnn5nZYgRiSSPmCUCd58OFFWzfYd/MV5AS0A9fXJQXrpuCD8c2qt0fex/F/Pw1GUhRiSSHELtIzCzC8zsI2ACkaOCqupdFZw+yi8sLIxfgJJ0bhl5NPdd2L90/b7JSzjjt9NCjEgk8YWaCNz9RXfvC5wP3FlNvcfdPcfdc7KysuIXoCSlb+R0Y+6YL6bAXLZhh+ZDFqlGQlw1FJxG6mVmncKORRqHdplNee/W00vXNR+ySNVCSwRm1sfMLFg+DmgG6FIPaTCd2zRn2d0V50Peq7uQRcqI5eWj44GZwFFmtsbMrjSzq83s6qDK14GFZjYXeBj4lmuweWlgGelpFZLBEbdN0rwGIlGsNv8hzKwlsMvdD5jZkUBfYJK7x30s4JycHM/Pz4/3y0qS+3zvfvre/mqZsrljzqRdZtOQIhKJLzOb7e45lW2r7RHBdKC5mXUBpgCXErlPQCQpNG+Szop7yt6FPPCO1/h87/6QIhJJHLVNBObuxcDXgEfc/RvAMbELS6ThpaVZhSEp+t7+qk4TScqrdSIws5OA7xC55h8gPTYhicSOmVU4Muh5y0Sey18dUkQi4attIrgRuAV40d0XmVkvQBdmS1JKS6uYDH72r/ma00BSVq0SgbtPc/fz3P3XZpYGbHT3H8c4NpGYSUurOFhdn9smsaV4T0gRiYSnVonAzJ41szbB1UMLgcVm9tPYhiYSe+WTwcA7XmNF4Y6QohEJR21PDfVz921EhoKYBPQkcuWQSNIrnwyG3z+NGcs2hhSNSPzVNhE0MbMmRBLBy8H9A7rUQhqN8sng4j/P4p/vfxJSNCLxVdtE8BhQQGS46Olm1gPYFqugRMJQPhn8/IUF3PCPOSFFIxI/tbqzuNInmmW4+74GjqdGurNYYq384HQn9+7Iuf0P5+ITuocUkcjBO+g7i82srZn9tmROADO7n8jRgUijU/7IYMbyTdz64oKQohGJvdqeGnoS2A58M3hsA56KVVAiYSvIy+X8gYeXKdMw1tJY1TYR9Hb3X7r7iuDxK6BXjc8SSWIPXDSozNSXEEkGuvFMGpvaJoJdZjakZMXMvgLsik1IIonjlpFHc8GgLmXK+tw2KaRoRGKjtsNQDwCeAdoGRZuB77r7/BjGVil1FksY5q3ewqiH3ylTdkLPDvzzhyeFFJFI3Rx0Z7G7z3P3AUB/oL+7DwKGN2CMIgltQLd2ZeZBBpi1sohnZhaEEo9IQ6rTDGXuvi24wxjg/2IQj0jCapfZtMIw1mNeWsRX8t4MKSKRhnEwU1Vag0UhkiTMKs5psHbLLiYuWBdSRCIH72ASQbWdC2b2pJltMLOFVWz/jpnNN7MFZjYj6IcQSXhmVmEe5Gv//oEuL5WkVW0iMLPtZratksd24PDqnktkKssR1WxfCQx19y8BdwKP1yVwkTBlpKdRkJdLh5Zl5zzOHj1BM55J0qk2Ebh7a3dvU8mjtbtn1PDc6UBRNdtnuPvmYPVdoGudoxcJ2Qe3n8lNZx5ZpqznLRP5cJ2G4pLkcTCnhhrSlUSGt66UmV1VMrxFYWFhHMMSqdn1px/BmzcNLVM28sG3Gf3CfDbt2B1SVCK1F3oiMLPTiCSCn1dVx90fd/ccd8/JysqKX3AitdQrqxXLy01/+Y/3VzP4rtcp3hP3sRlF6iTURGBm/YE/A6PcfVOYsYgcrPS0ilcUAfQbM5mCjTtDiEikdkJLBGbWHfg3cKm7Lw0rDpGGZBaZC7lN87JdaMN+8xbfemwm7xdU2W0mEpqYJQIzGw/MBI4yszVmdqWZXW1mVwdVxgAdgUfMbK6ZadwIaTTmjz2bR75zXJmyWSuL+MajM3VVkSScek9MExaNNSTJ5MN12xj54Ntlyjq3bsbbPz+NZhnpIUUlqeigxxoSkfo5+rA2vHfb6WXKNmzfzVG/eJUFa7aGFJVIWUoEIjHWuXXzCrOeAXz1of+xpXhPCBGJlKVEIBInBXm5PHX5l8uUDbzjNXLuei2kiEQilAhE4ui0vp2Zc3vZ4aw37thD9ugJfL53f0hRSapTIhCJs/Ytm1Z6qqjv7a9qfgMJhRKBSEgK8nK57KQeZcrGvLSIk8a9EVJEkqqUCERCdMeoY5n3y7PKlK3b+jnZoyewdP12duzW8BQSe0oEIiFr26IJK+6pODTFWb+bzrG/nEzRTl1ZJLGlRCCSANLSIkNTtMtsUmHbcXfqqiKJLSUCkQQyd8xZvP2z0yqUZ4+ewNPvrAwhIkkFSgQiCaZbh0wW/ursCuVj/7tY02FKTCgRiCSgVs0yKMjLpU/nVhW2ZY+ewNbivSFEJY2VEoFIAnv9/4Yy8cenVCgfcMcUskdP0Axo0iCUCEQSXL/D2zB3zJmVbht81+uccM/rfLKpOM5RSWOiRCCSBNplRu5Gfvm6r1TYtn7bbk69b2oIUUljoUQgkkT6d23H/LFnVbpt2H1TmbhgnSa+kTpTIhBJMm2aN6EgL5e+h7YuU16wqZhr//4Bv39jWUiRSbKK5VSVT5rZBjNbWMX2vmY208x2m9nNsYpDpLF69cZTmXnL8Arlv3t9KdmjJ5BfUMSBAzo6kJrF8ojgaWBENduLgB8Dv4lhDCKN2mFtW1Q6kinAhY/O5OuPzohzRJKMYpYI3H06kS/7qrZvcPf3AV0QLXKQCvJy+fmIvhXK53yyhezRE1izWVcVSdXURyDSSFwzrDcrx1UcvA5gyK+n8ui05XGOSJJFUiQCM7vKzPLNLL+wsDDscEQSlllk8LoHLxpYYVvepI80RIVUKikSgbs/7u457p6TlZUVdjgiCW/UwC6VDm0NkSEqskdPYN3WXXGOShJVUiQCEam7kqGtB3VvV+n2k8a9GeeIJFFZrG4+MbPxwDCgE7Ae+CXQBMDdHzWzQ4F8oA1wANgB9HP3bdXtNycnx/Pz82MSs0hjVbxnH/3GTK5024m9OvDIdwbToWXTOEcl8WRms909p9JtyXYXohKBSP24O7e+uIDx762udPsr1w/h2C5t4xyVxIsSgYiU2rf/AH1um1Tl9rOPOYTHLq30+0KSWHWJQH0EIikmIz2NgrxcXrl+SKXbJy9az0NvfszuffvjHJmERUcEIils1579HD3m1WrrvHL9EGatLOJ7X8nGzOIUmTQ0HRGISKVaNE2nIC+X0SMr3pVc4tw//I87X1nMtKW6h6exUiIQEa4e2ptFlcyTHO3yp95nwZqtcYpI4kmJQEQAaBnMk/zGTUOrrPPVh/6nEU0bISUCESmjd1YrCvJyGfvVfpVu73XrRLJHT2D9ts/jHJnEijqLRaRaNY1PdMGgLow5tx/tdUNaQlNnsYjUW0FeLq/95NQqt784Zy2D7nxNU2QmMSUCEanREYe0rnICnBI9b5nIjf+Yw8qNO+MUlTQUnRoSkTqpbtyiEplN03n0ksFc87fZzLrtDFo1y4hTdFIVnRoSkQaT2TRyddGzPzihyjrFe/Zz2ZPvsXPPfpZv2BHH6KQ+lAhEpF5O7t2pyklwoo16+J04RST1pUQgIgdl1MAuNfYfZI+ewJLPtusehASlPgIRaTCbd+5h0J2vVVvnpjOPZNhRnTnq0NY0zdBv0XjRMNQiEle/mbyEh6Yuq7bOBYO6cOqRnTixV0cOa9siTpGlLiUCEYm7ffsPcP4j77BwbbWTDgLw9s9Oo1uHzDhElbp01ZCIxF1GehqvXH9Kjf0HAKfcO5XVRcVxiEoqE7NEYGZPmtkGM1tYxXYzs9+b2TIzm29mx8UqFhEJV0FeLjNGD6+2zin3TiV79ASmLPqM2auK4hSZQGwnrz+VyIT0z7j7sZVsPwe4HjgHOAF40N2rvjA5oFNDIsnL3Zm2tJDLn3q/xrpmcN6Aw3nwokFxiKzxC+XUkLtPB6pL66OIJAl393eBdmZ2WKziEZHwmRnDjurMinvO4TffGFBtXXd4ae6nfLZVo5zGWph9BF2A1VHra4KyCszsKjPLN7P8wkLNkiSS7NLSjAsHd6UgL5eJPz6l2ronjnuDH4+fE6fIUlNSdBa7++PunuPuOVlZWWGHIyINqN/hbSjIy+WsfodUWefleZ+SPXoCP3gmn/smf8QJ97zOuys2acTTBhJmIlgLdIta7xqUiUgKevSSwbx32+l8dcDhVdZ5bfF6Hp66nPXbdnPR4+/y2PQVfO2Rd5i9anMcI218wkwELwOXBVcPnQhsdfd1IcYjIiFKSzM6t27OH749iIK8XDKbptf4nLxJH/HBJ1v4+h9nsGffgThE2TjF8qqh8cAwoBOwHvgl0ATA3R81MwMeAkYAxcAV7l7j5UC6akgkdWzY9jnH3/NGreufckQnrhnam5P7dIphVMlJdxaLSNKracrMaEvuGkHT9DQivzcFlAhEpJFYtmE7Z/x2eq3qHtulDYXbdzP5xlNpl6n5lJUIRKRRcXduf2khf3v3k1rV/9NlOfxp+goGdW/HLeccHePoEpMSgYg0SisKdzD8/ml1es7Hd4+kSXpSXDnfoDTonIg0Sr2yWlGQl0tBXi5/vqzS77gKjrhtUoyjSj46IhCRRuPAAefuiR/yxP9W1qr+IW2a8e4tp/Pqws8Y3KM9nds0j3GE4dGpIRFJOfPXbOG8h+o2X3JthsxOVjo1JCIpp3/Xdky9eRjXDOtd6+d8/y/vM+KB6WSPnsCG7akz2J2OCEQkJWzdtZcBv5pS6/o9OmYy7aensbqomKzWzXhmZgGPvLWcuWPOil2QMVTdEUFGvIMREQlD2xZNKMjLZXVRMafcO7XG+qs2FZfexNYrqyUrCncCsHf/AdLNSEtrPDer6YhARFJSXW5OK69np5b85MwjOa+aAfISjbpLwtgAAAq5SURBVPoIRETK6dO5NQV5uSy/5xy+P6RnnZ67cuNOfjx+Dn+avqJ0KOwVhTv4fO/+WIQaczoiEBEhcrfyy/M+5YZ/zD2o/STqlUc6IhARqYGZMWpgFwrycnn2+zVOn16lqUs2MG/1ltL15YU72Pb53oYIMWZ0RCAiUoVtn++l/9jaX2kUreTIoKTD+YVrTmZwj/YNFltd6YYyEZGDtGrTTobe91adnjP0yCymLY3Ms37kIa2Y8pOhMYisdnRqSETkIPXo2JKCvFyW3jWSPp1b1eo5JUkAYOn6HTyXv5q1W3aVqfPBJ5t55K1lDRprXemIQESknjbt2M3gu16v8/MevGggnVo1Y+HarYyb9BEQ+05mnRoSEYmhut61XJV3Rg+nS7sWDRBRRaHdWWxmI4AHgXTgz+6eV257D+BJIAsoAi5x9zWxjElEpKGV3LUM8PbHhVz6xHv12s+1f5vNhu27ad08g6XrdzBvzFm0zWzSkKFWKpaT16cDS4EzgTXA+8C33X1xVJ3ngVfc/S9mNpzIBPaXVrdfHRGISKJ7v6CIWSs2Ubh9N3+Zueqg9vXEd3NYuHYbv3t9KeO+9iW+fXz3eu0nlFNDZnYSMNbdzw7WbwFw93FRdRYBI9x9tUVmmd7q7m2q268SgYgkmx/+NZ/Ji9Yf9H4OadOMWbeeUa/nhnVqqAuwOmp9DVD+Lo15wNeInD66AGhtZh3dfVN0JTO7CrgKoHv3+mVDEZGwPHZp5Pt3/wHnnAffZsn67SFHVFbYl4/eDAw1sznAUGAtUGGwDnd/3N1z3D0nKysr3jGKiDSI9DRj8k9OZc7tZ5Lb/7A6dwwfiNG1PbE8IlgLdIta7xqUlXL3T4kcEWBmrYCvu/sWREQasfYtm/LwxccBkek1/z1nLTc/P6/G5xVu3x2TeGKZCN4HjjCznkQSwEXAxdEVzKwTUOTuB4BbiFxBJCKSMtLSjAsHd+XCwV3ZsXsfcz/ZwiVPzIpvDLHasbvvA64DJgMfAs+5+yIzu8PMzguqDQOWmNlS4BDg7ljFIyKS6Fo1y2DIEZ1YOe4cfpF7NKceGZ9T4bqhTEQkgRXv2cdLcz/l9cXruf+bA2iX2bRe+9FUlSIiSSqzaQbfPr57ve8fqI2wrxoSEZGQKRGIiKQ4JQIRkRSnRCAikuKUCEREUpwSgYhIilMiEBFJcUoEIiIpLunuLDazQqC+Mz10AjY2YDhhUlsSU2NpS2NpB6gtJXq4e6VjViRdIjgYZpZf1S3WyUZtSUyNpS2NpR2gttSGTg2JiKQ4JQIRkRSXaong8bADaEBqS2JqLG1pLO0AtaVGKdVHICIiFaXaEYGIiJSjRCAikuJSJhGY2QgzW2Jmy8xsdNjxVMbMCsxsgZnNNbP8oKyDmb1mZh8H/7YPys3Mfh+0Z76ZHRe1n+8G9T82s+/GKfYnzWyDmS2MKmuw2M1scPDeLAuea3Fuy1gzWxt8NnPN7JyobbcEcS0xs7Ojyiv9mzOznmY2Kyj/p5nVb8qpmtvRzcymmtliM1tkZjcE5Un3uVTTlmT8XJqb2XtmNi9oy6+qe30zaxasLwu2Z9e3jVVy90b/ANKB5UAvoCkwD+gXdlyVxFkAdCpXdi8wOlgeDfw6WD4HmAQYcCIwKyjvAKwI/m0fLLePQ+ynAscBC2MRO/BeUNeC546Mc1vGAjdXUrdf8PfUDOgZ/J2lV/c3BzwHXBQsPwpcE6N2HAYcFyy3BpYG8Sbd51JNW5LxczGgVbDcBJgVvIeVvj5wLfBosHwR8M/6trGqR6ocERwPLHP3Fe6+B/gHMCrkmGprFPCXYPkvwPlR5c94xLtAOzM7DDgbeM3di9x9M/AaMCLWQbr7dKAoFrEH29q4+7se+R/wTNS+4tWWqowC/uHuu919JbCMyN9bpX9zwS/m4cC/gudHvy8Nyt3XufsHwfJ24EOgC0n4uVTTlqok8ufi7r4jWG0SPLya14/+vP4FnB7EW6c2VhdTqiSCLsDqqPU1VP9HFBYHppjZbDO7Kig7xN3XBcufAYcEy1W1KZHa2lCxdwmWy5fH23XBKZMnS06nUPe2dAS2uPu+cuUxFZxOGETk12dSfy7l2gJJ+LmYWbqZzQU2EEmsy6t5/dKYg+1bg3gb7DsgVRJBshji7scBI4Efmdmp0RuDX11Jeb1vMsce+CPQGxgIrAPuDzec2jOzVsALwI3uvi16W7J9LpW0JSk/F3ff7+4Dga5EfsH3DTOeVEkEa4FuUetdg7KE4u5rg383AC8S+QNZHxyCE/y7IaheVZsSqa0NFfvaYLl8edy4+/rgP+8B4E9EPhuoe1s2ETnlklGuPCbMrAmRL86/u/u/g+Kk/Fwqa0uyfi4l3H0LMBU4qZrXL4052N42iLfhvgNi0RmSaA8gg0gHV0++6Dw5Juy4ysXYEmgdtTyDyLn9+yjbsXdvsJxL2Y6994LyDsBKIp167YPlDnFqQzZlO1gbLHYqdkqeE+e2HBa1/BMi52YBjqFsh90KIp11Vf7NAc9TtlPw2hi1wYict3+gXHnSfS7VtCUZP5csoF2w3AJ4Gzi3qtcHfkTZzuLn6tvGKmOK5X+mRHoQuSJiKZFzcbeFHU8l8fUKPrB5wKKSGImcC3wD+Bh4Peo/oAEPB+1ZAORE7et7RDqOlgFXxCn+8UQOzfcSOSd5ZUPGDuQAC4PnPERwV3wc2/LXINb5wMvlvoBuC+JaQtRVM1X9zQWf9XtBG58HmsWoHUOInPaZD8wNHuck4+dSTVuS8XPpD8wJYl4IjKnu9YHmwfqyYHuv+raxqoeGmBARSXGp0kcgIiJVUCIQEUlxSgQiIilOiUBEJMUpEYiIpDglAmm0zGxH8G+2mV3cAPsrMLMXotYvNLOnD3a/wb7GmtnNDbEvkbpSIpBUkA3UKRFE3eFZ3mAz63fQETWgYPho/V+WetMfj6SCPOCUYLz6nwQDft1nZu8Hg5X9EMDMhpnZ22b2MrC4in3dT+QmnjLK/6I3s4XBkUi2mX1kZk+b2VIz+7uZnWFm71hkbP/jo3YzwMxmBuU/iNrXT6NiLRm7PjsYb/4ZIjclRQ8pIFInVf3qEWlMRhMZs/5cgGBk163u/mUzawa8Y2ZTgrrHAcd6ZFjfyjwHXGtmferw+n2AbxC5O/d9IkcnQ4DzgFv5Yrjh/kSGa2gJzDGzCcCxwBFExtAx4OVgMMJPgvLvemTIaJF6UyKQVHQW0N/MLgzW2xL5Ut1DZHydqpIAwH4iY/XcQmRsndpY6e4LAMxsEfCGu7uZLSBy2qrES+6+C9hlZlOJfPkPCeKdE9RpFcT6CbBKSUAaghKBpCIDrnf3yWUKzYYBO2vx/L8SSQQLo8r2UfZUa/Oo5d1Rywei1g9Q9v9g+fFePIh1nLs/Vi7W7FrGKlIj9RFIKthOZHrDEpOBa4JhjTGzI82sZW135u57gd8RGe2yRAGR00pYZK7fnvWIc1Qwn21HYBiR00iTge8F4/BjZl3MrHM99i1SJR0RSCqYD+w3s3nA08CDRE7JfBBM+VdI3aclfAL4RdT6C8BlwamfWURGfqxPnFOBTsCd7v4p8KmZHQ3MjITKDuASIqeoRBqERh8VEUlxOjUkIpLilAhERFKcEoGISIpTIhARSXFKBCIiKU6JQEQkxSkRiIikuP8HDKQa95CTbv4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses)\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Iter Number')\n",
    "plt.title('Convergence monitor plot')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "colab_type": "code",
    "id": "Uebx8Dm6iAUX",
    "outputId": "bd7d68f6-9d8d-4948-c822-cf28307d8eca"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Loss = 0.8810939788818359\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>189</td>\n",
       "      <td>15</td>\n",
       "      <td>65</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>240</td>\n",
       "      <td>34</td>\n",
       "      <td>4</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>28</td>\n",
       "      <td>49</td>\n",
       "      <td>183</td>\n",
       "      <td>12</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>131</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>56</td>\n",
       "      <td>25</td>\n",
       "      <td>15</td>\n",
       "      <td>185</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2    3    4\n",
       "0  189   15   65   10    6\n",
       "1    6  240   34    4   43\n",
       "2   28   49  183   12   32\n",
       "3   12    6   14  131   45\n",
       "4    3   56   25   15  185"
      ]
     },
     "execution_count": 15,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    \n",
    "    train_loss = 0.0\n",
    "    y_train = []\n",
    "    y_train_pred = []\n",
    "\n",
    "    for data in trainloader:\n",
    "\n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        Z = ae1.get_z(X)\n",
    "        y_hat = classifier(Z)      \n",
    "        train_loss += criterion(y_hat, y)*len(X)/train_size\n",
    "        \n",
    "        y_train.extend(list(y.cpu().detach().numpy()))\n",
    "        y_train_pred.extend(list(torch.argmax(y_hat, axis=1).cpu().detach().numpy()))\n",
    "\n",
    "print('Train Loss =', train_loss.item())\n",
    "pd.DataFrame(confusion_matrix(y_train, y_train_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "ItZO18ZIjAyQ",
    "outputId": "16c72291-7588-4a5d-86fb-5e5ea5c26ee7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy = 0.6590909090909091 Train Precision = 0.6752864044727527 Train F1 = 0.6624632924854645\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score, precision_score\n",
    "acc = accuracy_score(y_train, y_train_pred)\n",
    "prec = precision_score(y_train, y_train_pred, average='macro')\n",
    "f1 = f1_score(y_train, y_train_pred, average='macro')\n",
    "\n",
    "print('Train Accuracy =', acc, 'Train Precision =', prec, 'Train F1 =', f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "colab_type": "code",
    "id": "q2F2G4QbiAUa",
    "outputId": "cdc17a00-f578-447e-d4df-ad0770c5f684"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss = 16.27732276916504\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>31</td>\n",
       "      <td>5</td>\n",
       "      <td>23</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>59</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>35</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>32</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3   4\n",
       "0  31   5  23   3   9\n",
       "1   5  59  10   3   6\n",
       "2  12  12  35   7   4\n",
       "3   6   1   3  32  10\n",
       "4   3  20   9   2  42"
      ]
     },
     "execution_count": 22,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    \n",
    "    test_loss = 0.0\n",
    "    y_test = []\n",
    "    y_test_pred = []\n",
    "\n",
    "    for data in testloader:\n",
    "\n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        Z = ae1.get_z(X)\n",
    "        y_hat = classifier(Z)*len(X)/test_size\n",
    "        test_loss += criterion(y_hat, y)\n",
    "        \n",
    "        y_test.extend(list(y.cpu().detach().numpy()))\n",
    "        y_test_pred.extend(list(torch.argmax(y_hat, axis=1).cpu().detach().numpy()))\n",
    "\n",
    "print('Test Loss =', test_loss.item())\n",
    "pd.DataFrame(confusion_matrix(y_test, y_test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "tjCtqyVJiAUf",
    "outputId": "ff2f3566-2fd1-48e0-af77-fa656a4a8259"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy = 0.5653409090909091 Test Precision = 0.572401486281531 Test F1 = 0.564898088023088\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score, precision_score\n",
    "acc = accuracy_score(y_test, y_test_pred)\n",
    "prec = precision_score(y_test, y_test_pred, average='macro')\n",
    "f1 = f1_score(y_test, y_test_pred, average='macro')\n",
    "\n",
    "print('Test Accuracy =', acc, 'Test Precision =', prec, 'Test F1 =', f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iecwF67CiAUj"
   },
   "outputs": [],
   "source": [
    "torch.save(classifier.state_dict(), 'a2_q1_wts.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1Ms1L078iAUm"
   },
   "source": [
    "# PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "n260pNqViAUn"
   },
   "outputs": [],
   "source": [
    "# To fit the PCA model we load all the training points in a single batch\n",
    "# train, test = train_test_loader('Data_Set_1(Colored_Images)', label_dict, train_fraction=0.8, batch_size=2000, num_workers=0)\n",
    "train, test, train_size, test_size = train_test_loader('/content/drive/My Drive/Data_Set_1(Colored_Images)', label_dict, train_fraction=0.8, batch_size=2000, num_workers=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 312
    },
    "colab_type": "code",
    "id": "86fyUeEoyZB5",
    "outputId": "a68715b1-ca20-49ba-e820-483908f84aaf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99% Variance Explained: 296\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZgdZZn38e8v3Uk6+77vhARJEAQiEBBBUGQTHGREUAE3dNxQcX1fF9zmHdxnRkYEBQQVRQYlsgooi0IkCUtICAkhZN86W2fppNf7/aOqw0mnu3MScvqc7vp9rquurr3uUzmp+zzPU/WUIgIzM8uuLsUOwMzMisuJwMws45wIzMwyzonAzCzjnAjMzDLOicDMLOOcCCyTJF0n6WvFjqPQJI2XFJLK81j3ZEkLCxTHzZK+U4h922vnRGB5k7RU0k5J2yWtS/9z985Z/nZJj0naJqlS0qOSzmu2j1PTC9OX9uO470mPrWbzyyWtl3Tu/n6WiPhYRHx7f7crFEmXS2pIz23uMLK9YoiIxyPisPY6npUOJwLbX++IiN7AMcA04KsAki4E/gDcAowGhgFfB97RbPvLgE3ApftxzD8B/YFTms0/Ewjg/v35AJLK9mf9dvRkRPRuNqwudlDW+TkR2AGJiFXAfcAR6S/1HwHfjohfRERVRDRGxKMR8ZGmbST1Ai4EPgFMkjQtz2PtAm5n7+RxKfDbiKiX9AdJayVVpaWSqTnHvVnSzyTdK2kH8JbcqgpJAyTdnZZiNqfjo3O2f0TStyX9Iy3t/EXS4Jzlb5L0hKQtklZIujyd313SDyQtT0tQ10nqsR+nuWn/EyVtknRMOj0yjfXUnPj+n6SnJG2VdJekga3s6wOSFqSfY4mkj+YsO1XSypzppZI+L2luel5/L6kiZ/m5kp5NP/cTko7MWXa0pKfT4/we2L2dlR4nAjsgksYAZwPPAIcBY4A79rHZBcB2kpLDAySlg6b9jU0vKGNb2fZXwIVNF1JJ/UhKG79Kl98HTAKGAk8Dv2m2/SXAd4E+wN+bLesC3ASMA8YCO4GftrD9B9L9dwM+n8YxLj32fwNDgDcAz6bb/AcwOZ13KDCKpJS0XyLiZeBLwK8l9Uxj/VVEPJKz2qXAB4ERQD3wX63sbj1wLtA3/Tw/bkowrXg3SclrAnAkcDkkF3rgRuCjwCDg58CMNPl1IynF3QoMJPn3ftd+fWhrXxHhwUNeA7CU5EK+BVgG/A/QAziJpIqmYh/bPwT8JB2/GKgEuu7H8V8CLknHPwI818p6/dN4+qXTNwO3NFvnZuA7rWz/BmBzzvQjwFdzpj8O3J+OfwX4Ywv7ELADmJgzbzrwSivHvJzkAr4lZ3i52TozgOeBuUD3ZvH9R870FKAWKAPGp+eivJXj/gm4Mh0/FVjZ7N/7fTnT3wOuS8d/RlICzN3XQpLquzcDqwHlLHuitfPtofiDSwS2v94ZEf0jYlxEfDwidgIb02UjWtsoLUG8hVd/qd9FUl1wzn4c+xZerR56fzqNpDJJ/yHpZUlbSS5gAINztl3RRmw9Jf1c0rJ0+8eA/s3aEtbmjFcDTY3kY4CXW9jtEKAnMCct6WwhacsY0sbnm5me26ZhYrPlNwBHAP8dETXNluV+vmVAV/b8/E2f9SxJM9Oqpi0kpbq91svR2uceB1zV9NnSfY0BRqbDqkgzQE5MVqKcCOxgWEhyIWqr+P9+ku/bnyWtBZaQJILL2timuVuB0yVNB07g1aRyCXA+8FagH8mvYEh+lTdpq5vdq0iqt46PiL4kv2ibb9+aFUDzCzbABpIqpqk5F/Z+kTS07zcld2f9BPglcHULbQBjcsbHAnVpDLn76A78L/ADYFhE9AfuJb/P2dwK4LvNElfPiLgNWAOManaXV2tVflYCnAjsNUt/+X0O+FraGNlXUpe0EfX6dLXLgG+SVLs0De8CzpY0KM/jLCWp378NeDAimn6t9gFqSEomPYF/38+P0Ifkor0lvcB+Yz+2/Q3wVknvVnI76yBJb4iIRpJf8D+WNBRA0ihJb9/P2Jr8JzA7Ij4M3ANc12z5+yRNSdsQvgXcERENzdbpBnQnqZKrl3QWcMYBxnMD8DFJxyvRS9I5kvoAT5JUc31aUldJFwDHHeBxrB04EdhBERF3ABeRNFiuBtYB3wHuknQCSVXCtRGxNmeYASwGLk4bi7e30Vjc5Ffpvm7JmXcLSdXDKuAFYOZ+hv8TkraODem2ed+OGhHLSapXriK5LfZZ4Kh08ZdIPt/MtMrpIZKSR2uma+/nCN4o6XySBtt/S9f7HHCMpPfmbHsrSbvHWpKS1qdbiHVbOv92YDNJSWpGvp+12b5mk7TT/DTd12LShuSIqCW5MeByknNyEXDngRzH2of2rMYzs45G0iPAryPiF8WOxTomlwjMzDLOicDMLONcNWRmlnEuEZiZZdw+u6YtNYMHD47x48cXOwwzsw5lzpw5GyKixQcaO1wiGD9+PLNnzy52GGZmHYqkVp/udtWQmVnGORGYmWWcE4GZWcY5EZiZZZwTgZlZxhUsEUi6UcmLxee1slyS/kvS4vRVeG29JcnMzAqkkCWCm0l6TGzNWSSvFpwEXEHyxiMzM2tnBXuOICIekzS+jVXOJ3l9YJB009tf0oiIWFOomCybdtU1sLO2gZ11ybArHXbWNlLb0EB9Q9AYQX1j0JAO9Y1BY9PfiN3rNKZdskS8+qabZDzI7a0leQVgOp6uk4zHHtviLl5sP5x++DCOGtP/oO+3mA+UjWLP1+utTOftlQgkXUFSamDsWL/oKOsigi3Vdazduou1VbtYu3UX67fWsLm6lk07al/9u6OWTdW17KprLHbIbdKBvB/MMmlo34pOlwjyFhHXA9cDTJs2zT+hOrmGxmBN1U6Wb6pm+cZqlm+qZtWWnayp2sW69OJfU7/3xb1PRTkDe3VjQM9uDOtbweEj+jKwVzf69ehKr25l9OhWRkXXV4ceXcvoVt6F8i6iLHdQ8re87NXxsi6iSxfRRdr9XkcJmqaaLua7/6J0edP8Ztv56m8lpJiJYBV7vmd1dDrPMiAiqNxew+J121m0bhsvV+5ILvybqlm5uZq6hlfzfXkXMaJ/BSP69uDI0f15+9QKhvWtYHjfCob3S4YhvbvTrdw3wZkdiGImghnAJyX9DjgeqHL7QOdUtbOO+aurWLh2G4vWbWfx+uRv1c663ev0qShn3KCeTBnRl7dPHc64QT0ZOzAZRvSroLzMF3mzQilYIpB0G3AqMFjSSpIXgncFiIjrgHtJ3vW6GKgGPlCoWKz9bKmuZd6qrTy/qop5q6uYt6qKZRurdy/v37Mrk4f24ZwjRzB5aG8mDevDpKG9GdKnu6tLzIqkkHcNXbyP5QF8olDHt8KLCJZurGbW0k3MWbqZWcs2saRyx+7lYwb24IiR/Xj3tDG8flQ/Dh/Rl8G9u/mCb1ZiOkRjsZWGiGDJhh08vqiSmUs2MXvZJjZsrwWSX/rTxg3gwmNHc+So/hwxqi/9e3YrcsRmlg8nAmtTVXUdf1+8gcdfquTxlzawastOAEYP6MGbJw1h2viBvHH8ACYO6U2XLv6lb9YRORHYXtZt3cVf5q/lgfnreHLJRhoagz4V5Zw0cTAff8tETj50CGMH9Sx2mGZ2kDgRGAAbttdw17OruXvuap5ZvgWAQ4b04oo3H8JbDx/KUaP7+84ds07KiSDDauob+OuC9fzv0yt5ZGEl9Y3B1JF9+fwZkznziOEcOrRPsUM0s3bgRJBBa6p2cuuTy7jtqeVsrq5jWN/ufOjkCVx4zGgmDfPF3yxrnAgy5LkVW7j+8SXcP28tEcHbpgzjkuPH8aZDB1Pmhl6zzHIiyIB5q6r4yUOLeGjBevpUlPPBk8Zz6fTxjBnoBl8zcyLo1FZsqubf713AffPW0reinM+fMZnLThxPn4quxQ7NzEqIE0EnVN/QyM8eeZmf/m0xZV3EladP4kMnT6CvE4CZtcCJoJNZtWUnn/ndM8xauplzXj+Cr557OCP69Sh2WGZWwpwIOpGnXtnEFbfOpq6+kR9fdBT/cvToYodkZh2AE0Encc/cNXz29mcZPaAHN172RsYP7lXskMysg3Ai6ARun7WCL905l2PHDuCGS6cxoJc7ezOz/DkRdHB3zFnJl+6cy8mThnD9+4+lomtZsUMysw7Gncd0YH9+bjVfuOM53nToYCcBMztgTgQd1MwlG7nq9ud44/iB3HDpNCcBMztgTgQd0OL127jiltmMHdSTG97vJGBmr40TQQezaUctl980i27lZdx0+Rvp19MPiZnZa+PG4g6kvqGRT932NOu31fCHj053X0FmdlC4RNCBfP8vC/nH4o18951HcNSY/sUOx8w6CSeCDuKeuWv4+aNLeN8JY/nXaWOKHY6ZdSJOBB3AKxt28MU7nuPosf35+rlTix2OmXUyTgQlrra+kU/f9gzlZV249pJj6FbufzIzO7jcWFzifviXhTy/qorr3ncsI/u7F1EzO/j887KEPf5SJT9/bAmXHD+WM48YXuxwzKyTciIoUVXVdVx1+3NMGtqbr50zpdjhmFkn5qqhEvWde15g445abrz8jfTo5ieHzaxwXCIoQY8uquQPc1bysVMO4YhR/Yodjpl1ck4EJWZ7TT3/587nmTikF586bVKxwzGzDHDVUIm55r4XWV21kzs+dqI7kzOzduESQQmZvXQTt85cxgdPmsCx4wYUOxwzywgnghJR39DI1+6az8h+FVx1xuRih2NmGeJEUCJ+PXMZC9Zs5WvnTqFnN9fYmVn7cSIoAZXbavjhg4s4edJgPzhmZu3OiaAEXHP/i+yqa+Dq86YiqdjhmFnGFDQRSDpT0kJJiyV9uYXlYyX9TdIzkuZKOruQ8ZSiOcs2cceclXz45EOYOKR3scMxswwqWCKQVAZcC5wFTAEultS8r4SvArdHxNHAe4D/KVQ8pSgi+NbdCxjet4JPnXZoscMxs4wqZIngOGBxRCyJiFrgd8D5zdYJoG863g9YXcB4Ss49z6/huRVbuOqMyW4gNrOiKWQiGAWsyJlemc7LdTXwPkkrgXuBT7W0I0lXSJotaXZlZWUhYm13NfUNfO/+hbxueB8uOGZ0scMxswwrdmPxxcDNETEaOBu4VdJeMUXE9RExLSKmDRkypN2DLIRfz1zO8k3VfOXswynr4gZiMyueQiaCVUDuy3VHp/NyfQi4HSAingQqgMEFjKkkVO2s47//+hInTxrMKZM7R2Izs46rkIlgFjBJ0gRJ3Ugag2c0W2c5cDqApMNJEkHnqPtpw/WPvUzVzjq+fNbrih2KmVnhEkFE1AOfBB4AFpDcHTRf0rcknZeudhXwEUnPAbcBl0dEFCqmUrBpRy03/2Mp5x45kqkj3cW0mRVfQW9ViYh7SRqBc+d9PWf8BeCkQsZQam54fAnVdQ182reLmlmJKHZjcaZs2lHLr55ISgOThvUpdjhmZoATQbu6/rEl7Kxr4MrTXRows9LhRNBONm6v4ZYnl/KOI0dy6FCXBsysdLTaRiDpzyRP/rYoIs5rbZnt7Zd/f4WddQ18+nS/ftLMSktbjcU/SP9eAAwHfp1OXwysK2RQnc22XXXcOnMZZ04dzqFD3bGcmZWWVhNBRDwKIOmHETEtZ9GfJc0ueGSdyG1PLWfbrno+dsrEYodiZraXfNoIekk6pGlC0gSgV+FC6lxq6hv45d9f4cSJgzhqTP9ih2Nmtpd8niP4LPCIpCWAgHHARwsaVSdy1zOrWbe1hu9feFSxQzEza9E+E0FE3C9pEtDUH8KLEVFT2LA6h8bG4LrHXmbqyL6cPKnTd6FkZh3UPquGJPUEvgB8MiKeA8ZKOrfgkXUCDy5Yx5LKHXz0lIl+BaWZlax82ghuAmqB6en0KuA7BYuoE7nhsSWMGdiDs/1CejMrYfkkgokR8T2gDiAiqknaCqwN81ZVMXvZZi4/cQLlZX5uz8xKVz5XqFpJPUgfLpM0EXAbwT7c/MRSenYr41+n+e1jZlba8rlr6BvA/cAYSb8h6S308kIG1dFt3F7DjOdW8+5po+lb0bXY4ZiZtSmfu4YelPQ0cAJJldCVEbGh4JF1YL+btYLa+kYunT6+2KGYme1Tvu8jqAA2p+tPkUREPFa4sDqu+oZGfjNzGSdOHMRkdzVtZh3APhOBpGuAi4D5QGM6OwAnghY8tGAdq6t28Y3zphY7FDOzvORTIngncJgfIsvPzU8sZVT/Hrz18GHFDsXMLC/53DW0BHCLZx4Wr9/GzCWbeN8J4yjr4jtszaxjyKdEUA08K+lhcm4bjYhPFyyqDuq2p1ZQ3kW+ZdTMOpR8EsGMdLA21NQ3cOfTK3nblGEM7t292OGYmeUtn9tHf9UegXR0f5m/js3VdbznuLHFDsXMbL+09arK2yPi3ZKep4VXVkbEkQWNrIP5/awVjOrfg5MPdS+jZtaxtFUiuDL9655G92H5xmr+vngDn3vbZLq4kdjMOpi2XlW5Jv27rP3C6Zh+P3s5XYQbic2sQ8rnfQQnSJolabukWkkNkra2R3AdQX1DI3+YvZJTDxvKiH49ih2Omdl+y+c5gp8CFwMvAT2ADwPXFjKojuQfL29k/bYa3u3SgJl1UHl1lB8Ri4GyiGiIiJuAMwsbVsfxx6dX0q9HV97yuqHFDsXM7IDk9UCZpG4kD5V9D1hDngmks9teU8/989dywTGj6V5eVuxwzMwOSD4X9PcDZcAngR3AGOBdhQyqo7h/3lp21TVywdGjih2KmdkBy+eBsqa7hnYC3yxsOB3LH59ZydiBPTl23IBih2JmdsDaeqCsxQfJmmT9gbI1VTt54uWNfOq0SUh+dsDMOq62SgR+kKwNM55dTQT8i6uFzKyDa+uBst0PkkkaDhxHUkKYFRFr2yG2kvbnuas5akx/JgzuVexQzMxek3weKPsw8BRwAXAhMFPSBwsdWClbumEH81Zt5R1Hjih2KGZmr1k+dw19ATg6Ii6PiMuAY4Ev5bNzSWdKWihpsaQvt7LOuyW9IGm+pN/mH3rx3PP8GgDOer0TgZl1fPk8R7AR2JYzvS2d1yZJZSRPIL8NWAnMkjQjIl7IWWcS8BXgpIjYLKlDPJV1z9w1HD22P6P6u0sJM+v48kkEi4F/SrqLpI3gfGCupM8BRMSPWtnuOGBxRCwBkPS7dNsXctb5CHBtRGxO97X+gD5FO1pSuZ0X1mzlq+ccXuxQzMwOinwSwcvp0OSu9G+ffWw3CliRM70SOL7ZOpMBJP2D5KG1qyPi/jxiKpp702qhs10tZGadRD6J4JqI2JU7Q9LgiNhwkI4/CTgVGA08Jun1EbGl2fGuAK4AGDu2uG8Au3vuGo4dN4CRrhYys04in8bipySd0DQh6V3AE3lst4qkO4omo9N5uVYCMyKiLiJeARaRJIY9RMT1ETEtIqYNGTIkj0MXxuL123lx7TbOcWnAzDqRfEoE7wVulPQIMBIYBJyWx3azgEmSJpAkgPcAlzRb508kXVzfJGkwSVXRkvxCb3/3uVrIzDqhfPoael7Sd4FbSe4YenNErMxju3pJnwQeIKn/vzEi5kv6FjA7Imaky86Q9ALQAHwhIvZ5R1Kx/OWFdRw9tj/D+1UUOxQzs4Nmn4lA0i+BicCRJL/Y75b03xGxz5fTRMS9wL3N5n09ZzyAz6VDSVu9ZSfPr6rii2ceVuxQzMwOqnzaCJ4H3hIRr0TEAyR3/hxT2LBKz0ML1gFwxpThRY7EzOzgajURSOoLEBE/SX+5k05XkcHuqB98YR2HDO7FoUN7FzsUM7ODqq0SwSNNI5IebrbsTwWJpkRt3VXHzCUbeduUYcUOxczsoGsrEeR2sj+wjWWd3iMLK6lrCCcCM+uU2koE0cp4S9Od2oMvrGNQr24cPdZvIjOzzqetu4aGpv0JKWecdLp4T3W1s9r6Rh55cT1nv34EZV0yVRAys4xoKxHcwKv9CeWOA/yiYBGVmH++spFtNfWuFjKzTqutN5Rl7s6gljy8YD3dy7vwpkmDix2KmVlB5PMcQaY9tqiS6RMHUdG1rNihmJkVhBNBG1ZsqmbJhh2cMjkzTSJmlkFOBG14dFElgBOBmXVq+by8fpikX0q6L52eIulDhQ+t+B5dVMmYgT2YMLhXsUMxMyuYfEoEN5P0EjoynV4EfKZQAZWK2vpGnli8gVMmD0HybaNm1nnlkwgGR8TtQCMk3UuTdBndqc1ZtpkdtQ28eZKrhcysc8snEeyQNIj0aeL0bWVVBY2qBDy6qJLyLuLEQ33bqJl1bvm8oexzwAxgYvqS+SHAhQWNqgQ8uqiSaeMH0Lt7PqfIzKzjyucNZU9LOgU4jKR7iYURUVfwyIqoclsNC9Zs9UtozCwT8rlr6BNA74iYHxHzgN6SPl740IrnySXJ2zJPmuhqITPr/PJpI/hIRGxpmoiIzcBHChdS8T358kb6dC9n6si+xQ7FzKzg8kkEZcq5f1JSGdCtcCEV38wlGzluwkDKy/y8nZl1fvlc6e4Hfi/pdEmnA7el8zqltVW7eGXDDqZPHFTsUMzM2kU+t8R8Cfgo8G/p9IN04m6on1yyAYATDnEiMLNsyOeuoUbgZ+nQ6T2xeCP9enRlygi3D5hZNuwzEUg6CbgaGJeuLyAi4pDChlYcTy7ZyAmHDKSL30ZmZhmRT9XQL4HPAnPo5F1LrNhUzcrNO/nwmyYUOxQzs3aTTyKoioj7Ch5JCWh6fmC6nx8wswzJJxH8TdL3gTuBmqaZEfF0waIqkplLNjKoVzcmD+td7FDMzNpNPong+PTvtJx5AZx28MMprqeXbebYcQPc7bSZZUo+dw29pT0CKbYN22tYurGaS44fW+xQzMzaVV5da0o6B5gKVDTNi4hvFSqoYnh62WYAjh03oMiRmJm1r3w6nbsOuAj4FMmto/9KcitppzJn+Wa6lXVh6sh+xQ7FzKxd5dPFxIkRcSmwOSK+CUwHJhc2rPb39LLNHDGqLxVdy4odiplZu8onEexM/1ZLGgnUASMKF1L7q6lv4LmVVa4WMrNMyqeN4G5J/YHvA0+T3DHUqfoamr96K7X1jU4EZpZJ+dw19O109H8l3Q1URESnemdxU0PxMWOdCMwse1pNBJJOi4i/SrqghWVExJ2FDa39zFtVxfC+FQztW7Hvlc3MOpm22ghOSf++o4Xh3Hx2LulMSQslLZb05TbWe5ekkDSttXUKad7qrRwxyr2Nmlk2tVoiiIhvSOoC3BcRt+/vjtM3mV0LvA1YCcySNCMiXmi2Xh/gSuCf+3uMg6G6tp6XK7dzzus7Vfu3mVne2rxrKH0XwRcPcN/HAYsjYklE1AK/A85vYb1vA9cAuw7wOK/JgjVbiYAjRvn5ATPLpnxuH31I0ucljZE0sGnIY7tRwIqc6ZXpvN0kHQOMiYh72tqRpCskzZY0u7KyMo9D52/eqq0Arhoys8zK5/bRi9K/n8iZF8BrejFNWu30I+Dyfa0bEdcD1wNMmzYtXstxm5u/uopBvbox3A3FZpZR+dw+eqBvaVkFjMmZHp3Oa9IHOAJ4JO3tczgwQ9J5ETH7AI+53+at2srUUf3c46iZZVa+nc4dAUxhz07nbtnHZrOASZImkCSA9wCX5GxfBex+A4ykR4DPt2cSqKlvYNG6bZx62JD2OqSZWcnJ553F3wBOJUkE9wJnAX8H2kwEEVEv6ZPAA0AZcGNEzJf0LWB2RMx4jbG/ZovWbqe+MdxQbGaZlk+J4ELgKOCZiPiApGHAr/PZeUTcS5I8cud9vZV1T81nnwfTgrVJQ/HhI9xQbGbZlVenc+ltpPWS+gLr2bPuv8N6ef12upV1YcyAHsUOxcysaPIpEcxOO527AZgDbAeeLGhU7WTx+u0cMqQX5WX55EMzs86prb6GrgV+GxEfT2ddJ+l+oG9EzG2X6ArspfXbef1otw+YWba19VN4EfADSUslfU/S0RGxtLMkgV11DazYXM2hQ3oXOxQzs6JqNRFExH9GxHSSzuc2AjdKelHSNyR1+DeULancQQQcOtSJwMyybZ+V4xGxLCKuiYijgYuBdwILCh5ZgS2u3A44EZiZ5fPy+nJJ75D0G+A+YCGw1zsKOpplG3YAMGFwryJHYmZWXG01Fr+NpARwNvAUSe+hV0TEjnaKraCWb6pmaJ/uflm9mWVeW7ePfgX4LXBVRGxup3jazfJN1Ywd2LPYYZiZFV1bL6Y5rT0DaW8rN+/k+An59KZtZta5ZfJJqtr6RlZX7WS0SwRmZtlMBKu27CQCVw2ZmZHRRLBiUzXgRGBmBhlNBMudCMzMdstkIlixqZpu5V0Y2qd7sUMxMyu6bCaCzdWMHtCDLl38ekozs0wmAj9DYGb2qkwmglWbdzKqv19GY2YGGUwENfUNbK6uY3jfimKHYmZWEjKXCNZvrQFgWD8nAjMzyGAiWLd1FwDDXCIwMwMymAjWponAVUNmZonsJYKqphKBnyEwM4MMJoL122roXt6Ffj26FjsUM7OSkLlEsLZqF8P6ViD5YTIzM8hgIli3dZfbB8zMcmQyEQx1+4CZ2W6ZSgQRwVqXCMzM9pCpRLB1Vz276hr9DIGZWY5MJYLdD5P5qWIzs92ymQj8HgIzs90ylQgqtyX9DA111ZCZ2W6ZSgRVO+sA6O+HyczMdstkIuhTUV7kSMzMSkemEsHWnfX07l5OeVmmPraZWZsydUXcuquOvi4NmJntoaCJQNKZkhZKWizpyy0s/5ykFyTNlfSwpHGFjKdqZx193T5gZraHgiUCSWXAtcBZwBTgYklTmq32DDAtIo4E7gC+V6h4ALbvqnf7gJlZM4UsERwHLI6IJRFRC/wOOD93hYj4W0RUp5MzgdEFjIfqugZ6dnMiMDPLVchEMApYkTO9Mp3Xmg8B97W0QNIVkmZLml1ZWXnAAVXX1NOzW9kBb29m1hmVRGOxpPcB04Dvt7Q8Iq6PiGkRMW3IkCEHfJzqWpcIzMyaK+RVcRUwJmd6dDpvD5LeCvxf4JSIqClgPFTXukRgZtZcIUsEs4BJkiZI6ga8B5iRu4Kko4GfA+dFxPoCxgLAjtoGenZ3IjAzy1WwRBAR9cAngQeABcDtETFf0nIuWHkAAAicSURBVLcknZeu9n2gN/AHSc9KmtHK7l6z+oZGausb6dnVVUNmZrkKelWMiHuBe5vN+3rO+FsLefxc1XUNAPRyicDMbA8l0VjcHnbVJomgoqsTgZlZrswkgpr6RgC6l2fmI5uZ5SUzV8WmRNDNicDMbA+ZuSrW1CdVQ93LXTVkZpYrM4mg1lVDZmYtysxV0YnAzKxlmbkquo3AzKxlmbkqvloicBuBmVmuzCQClwjMzFqWmatibUPTXUOZ+chmZnnJzFWxps4lAjOzlmTmqljb4LuGzMxakpmrYq3bCMzMWpSZq+LYgT0564jhvmvIzKyZzHTOf8bU4ZwxdXixwzAzKzmZKRGYmVnLnAjMzDLOicDMLOOcCMzMMs6JwMws45wIzMwyzonAzCzjnAjMzDJOEVHsGPaLpEpg2QFuPhjYcBDD6ax8nvLj87RvPkf5aY/zNC4ihrS0oMMlgtdC0uyImFbsOEqdz1N+fJ72zecoP8U+T64aMjPLOCcCM7OMy1oiuL7YAXQQPk/58XnaN5+j/BT1PGWqjcDMzPaWtRKBmZk140RgZpZxmUkEks6UtFDSYklfLnY8xSJpjKS/SXpB0nxJV6bzB0p6UNJL6d8B6XxJ+q/0vM2VdExxP0H7klQm6RlJd6fTEyT9Mz0fv5fULZ3fPZ1enC4fX8y424uk/pLukPSipAWSpvu7tDdJn03/v82TdJukilL6LmUiEUgqA64FzgKmABdLmlLcqIqmHrgqIqYAJwCfSM/Fl4GHI2IS8HA6Dck5m5QOVwA/a/+Qi+pKYEHO9DXAjyPiUGAz8KF0/oeAzen8H6frZcF/AvdHxOuAo0jOlb9LOSSNAj4NTIuII4Ay4D2U0ncpIjr9AEwHHsiZ/grwlWLHVQoDcBfwNmAhMCKdNwJYmI7/HLg4Z/3d63X2ARhNciE7DbgbEMnTn+Xp8t3fK+ABYHo6Xp6up2J/hgKfn37AK80/p79Le52nUcAKYGD63bgbeHspfZcyUSLg1X+IJivTeZmWFjmPBv4JDIuINemitcCwdDzL5+4nwBeBxnR6ELAlIurT6dxzsfs8pcur0vU7swlAJXBTWn32C0m98HdpDxGxCvgBsBxYQ/LdmEMJfZeykgisGUm9gf8FPhMRW3OXRfJTJNP3FUs6F1gfEXOKHUsJKweOAX4WEUcDO3i1GgjwdwkgbSM5nyRxjgR6AWcWNahmspIIVgFjcqZHp/MySVJXkiTwm4i4M529TtKIdPkIYH06P6vn7iTgPElLgd+RVA/9J9BfUnm6Tu652H2e0uX9gI3tGXARrARWRsQ/0+k7SBKDv0t7eivwSkRURkQdcCfJ96tkvktZSQSzgElpK303koaaGUWOqSgkCfglsCAifpSzaAZwWTp+GUnbQdP8S9M7Pk4AqnKK/Z1WRHwlIkZHxHiS78tfI+K9wN+AC9PVmp+npvN3Ybp+p/4lHBFrgRWSDktnnQ68gL9LzS0HTpDUM/3/13SeSue7VOyGlHZssDkbWAS8DPzfYsdTxPPwJpKi+lzg2XQ4m6QO8mHgJeAhYGC6vkjuuHoZeJ7kzoeif452PmenAnen44cATwGLgT8A3dP5Fen04nT5IcWOu53OzRuA2en36U/AAH+XWjxP3wReBOYBtwLdS+m75C4mzMwyLitVQ2Zm1gonAjOzjHMiMDPLOCcCM7OMcyIwM8s4JwIrSZJC0g9zpj8v6eoCHOe2tCfMzx7sfZcSSW+QdHax47DS5ERgpaoGuEDS4EIdQNJw4I0RcWRE/LhQxykRbyB5XsRsL04EVqrqSd7jutcvdUnjJf01/SX/sKSxbe0o7fv9JknPp52jvSVd9BdglKRnJZ3cbJthkv4o6bl0ODGd/7m0T/l5kj6TE8+Lkm6WtEjSbyS9VdI/0j75j0vXu1rSrZKeTOd/JJ0vSd9P9/m8pIvS+adKekSv9vf/m/TJVCQdK+lRSXMkPZDTpcMjkq6R9FQay8np0/TfAi5KP+tFkk5Jx59Nz0mfA/2Hsk6g2E/cefDQ0gBsB/oCS0n6Wvk8cHW67M/AZen4B4E/7WNfVwE3puOvI3nkvwIYD8xrZZvfk3TIB0n/8f2AY0meiO0F9Abmk/TeOp4kcb2e5MfVHOBGkidpz2+KD7gaeA7oAQwm6WFyJPAu4MH0OMPS+EaQPNFcRdIPTRfgSZInw7sCTwBD0v1elPP5HgF+mI6fDTyUjl8O/DTn8/0ZOCkd703aHbKHbA4uEVjJiqRX1FtIXuqRazrw23T8VpKLY1veBPw63eeLwDJg8j62OY30xSkR0RARVel+/hgROyJiO0nnYU0liVci4vmIaCRJEA9HRJAkjvE5+70rInZGxAaSvmaOS/d7W3qcdcCjwBvT9Z+KiJXpfp9N93UYcATwoKRnga+SJIsmTR0Jzml27Fz/AH4k6dNA/3i1O2TLoPJ9r2JWVD8BngZuKnYg+1CTM96YM93Inv/Pmvfpsq8+XnL325DuS8D8iJi+j22a1t9LRPyHpHtISg3/kPT2NElaBrlEYCUtIjYBt/Pqa/wgqRZ5Tzr+XuDxfezm8XQ9JE0GxpK8HastDwP/lm5TJqlfup93pr1I9gL+JY9jN3d+2mYxiKTqZ1a6j4vS4wwB3kzS2VhrFgJDJE1P4+sqaeo+jrsN2N0OIGliWoK5Jo3hdfv5OawTcSKwjuCHJHXqTT4FfEDSXOD9JO8VRtLHJH2she3/B+gi6XmSuv/LI6KmhfVyXQm8Jd1mDjAlIp4Gbia5SP8T+EVEPLOfn2UuSZXQTODbEbEa+GM6/zngr8AXI+niuUURUUvSPfE1kp4jqTI6cR/H/RswpamxGPhM2jg9F6gD7tvPz2GdiHsfNWsn6XMQ2yPiB8WOxSyXSwRmZhnnEoGZWca5RGBmlnFOBGZmGedEYGaWcU4EZmYZ50RgZpZx/x+OX6ryVP7IaQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "kk = 0\n",
    "for i in train:\n",
    "    temp = i[0]\n",
    "\n",
    "pca1 = PCA(n_components=temp.shape[1])\n",
    "pca1.fit(temp)\n",
    "\n",
    "print('99% Variance Explained:', np.where(np.cumsum(pca1.explained_variance_ratio_)>=0.99)[0][0]+1)\n",
    "explained_var = np.cumsum(pca1.explained_variance_ratio_)\n",
    "plt.plot(explained_var)\n",
    "plt.title('PCA: Variance Explained')\n",
    "plt.xlabel('No. of components')\n",
    "plt.ylabel('Variance Explained')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xODHGEIziAUq"
   },
   "outputs": [],
   "source": [
    "def return_pca(temp):\n",
    "    pca = PCA(n_components=0.99)\n",
    "    pca.fit(temp)\n",
    "    return pca\n",
    "\n",
    "PCA_model = return_pca(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WYPOH8V0iAUv"
   },
   "outputs": [],
   "source": [
    "reduced_dimension = PCA_model.transform(temp).shape[1]\n",
    "pca_clf = FinalNet(reduced_dimension, [150, 50], 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "Ia5lUuzsiAUz",
    "outputId": "37105d0b-cc12-4fdd-fe2d-042b39052d3c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "296"
      ]
     },
     "execution_count": 24,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reduced_dimension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5s9qE34QiAU2"
   },
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(pca_clf.parameters(), lr=0.001, momentum=0.9)\n",
    "pca_clf = pca_clf.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 6364
    },
    "colab_type": "code",
    "id": "AxWdHntPiAU5",
    "outputId": "8833352d-2c7d-41a7-bc1f-9d81d1dd3b04"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 : Loss = 1.6132414449344978\n",
      "Epoch 2 : Loss = 1.559115195816213\n",
      "Epoch 3 : Loss = 1.5051277090202677\n",
      "Epoch 4 : Loss = 1.4514944472096183\n",
      "Epoch 5 : Loss = 1.4014036790891131\n",
      "Epoch 6 : Loss = 1.3560702448541466\n",
      "Epoch 7 : Loss = 1.3161314400759607\n",
      "Epoch 8 : Loss = 1.2789940021254798\n",
      "Epoch 9 : Loss = 1.2426278889179228\n",
      "Epoch 10 : Loss = 1.2068441943688824\n",
      "Epoch 11 : Loss = 1.171247065067291\n",
      "Epoch 12 : Loss = 1.1365510577505282\n",
      "Epoch 13 : Loss = 1.102829074317759\n",
      "Epoch 14 : Loss = 1.0702505098147825\n",
      "Epoch 15 : Loss = 1.0394562889229169\n",
      "Epoch 16 : Loss = 1.0099370425397698\n",
      "Epoch 17 : Loss = 0.9814402921633285\n",
      "Epoch 18 : Loss = 0.9542509466409687\n",
      "Epoch 19 : Loss = 0.9291637743061238\n",
      "Epoch 20 : Loss = 0.9047065295956351\n",
      "Epoch 21 : Loss = 0.8826249607584694\n",
      "Epoch 22 : Loss = 0.8607305938547307\n",
      "Epoch 23 : Loss = 0.8405540897087616\n",
      "Epoch 24 : Loss = 0.8213727582584727\n",
      "Epoch 25 : Loss = 0.8031308935447173\n",
      "Epoch 26 : Loss = 0.7851167510856282\n",
      "Epoch 27 : Loss = 0.767833185466853\n",
      "Epoch 28 : Loss = 0.7524041885679416\n",
      "Epoch 29 : Loss = 0.7364228421991521\n",
      "Epoch 30 : Loss = 0.7218800634145736\n",
      "Epoch 31 : Loss = 0.707443032752384\n",
      "Epoch 32 : Loss = 0.6943461976268076\n",
      "Epoch 33 : Loss = 0.6813012104142795\n",
      "Epoch 34 : Loss = 0.6672595502300697\n",
      "Epoch 35 : Loss = 0.6549456227909434\n",
      "Epoch 36 : Loss = 0.6432998783209107\n",
      "Epoch 37 : Loss = 0.6320721981200302\n",
      "Epoch 38 : Loss = 0.619659314778718\n",
      "Epoch 39 : Loss = 0.6083684035322883\n",
      "Epoch 40 : Loss = 0.5989550054073335\n",
      "Epoch 41 : Loss = 0.5877962789752267\n",
      "Epoch 42 : Loss = 0.5765357349406589\n",
      "Epoch 43 : Loss = 0.5655385269360109\n",
      "Epoch 44 : Loss = 0.5567387593063441\n",
      "Epoch 45 : Loss = 0.5466414988040924\n",
      "Epoch 46 : Loss = 0.5380864671685477\n",
      "Epoch 47 : Loss = 0.5283479094505311\n",
      "Epoch 48 : Loss = 0.518478093499487\n",
      "Epoch 49 : Loss = 0.5098911916667765\n",
      "Epoch 50 : Loss = 0.4997859407554971\n",
      "Epoch 51 : Loss = 0.49226355891336104\n",
      "Epoch 52 : Loss = 0.4831135489723899\n",
      "Epoch 53 : Loss = 0.47528860040686355\n",
      "Epoch 54 : Loss = 0.46666617149656453\n",
      "Epoch 55 : Loss = 0.4600685801018369\n",
      "Epoch 56 : Loss = 0.4506540887735107\n",
      "Epoch 57 : Loss = 0.4431898790326986\n",
      "Epoch 58 : Loss = 0.4357372728938406\n",
      "Epoch 59 : Loss = 0.42792707004330377\n",
      "Epoch 60 : Loss = 0.42105984484607534\n",
      "Epoch 61 : Loss = 0.4125621769238602\n",
      "Epoch 62 : Loss = 0.4041191163388166\n",
      "Epoch 63 : Loss = 0.39744394543496037\n",
      "Epoch 64 : Loss = 0.39129705523902725\n",
      "Epoch 65 : Loss = 0.38372014666145493\n",
      "Epoch 66 : Loss = 0.37753833390095015\n",
      "Epoch 67 : Loss = 0.37064832686023275\n",
      "Epoch 68 : Loss = 0.3638288588686423\n",
      "Epoch 69 : Loss = 0.35681519860571076\n",
      "Epoch 70 : Loss = 0.35097958621653635\n",
      "Epoch 71 : Loss = 0.34401789883320977\n",
      "Epoch 72 : Loss = 0.33764980259266775\n",
      "Epoch 73 : Loss = 0.3321842138062825\n",
      "Epoch 74 : Loss = 0.32422411001541396\n",
      "Epoch 75 : Loss = 0.3184758079322902\n",
      "Epoch 76 : Loss = 0.3121851801194928\n",
      "Epoch 77 : Loss = 0.305859293111346\n",
      "Epoch 78 : Loss = 0.2990428297357126\n",
      "Epoch 79 : Loss = 0.29361338642510504\n",
      "Epoch 80 : Loss = 0.287195551124486\n",
      "Epoch 81 : Loss = 0.2805693051354452\n",
      "Epoch 82 : Loss = 0.27584152871912176\n",
      "Epoch 83 : Loss = 0.26972230760888627\n",
      "Epoch 84 : Loss = 0.2653910185803067\n",
      "Epoch 85 : Loss = 0.2576223022558472\n",
      "Epoch 86 : Loss = 0.253162021664056\n",
      "Epoch 87 : Loss = 0.24695279483090754\n",
      "Epoch 88 : Loss = 0.24242785742337047\n",
      "Epoch 89 : Loss = 0.23673649606379601\n",
      "Epoch 90 : Loss = 0.23090966418385497\n",
      "Epoch 91 : Loss = 0.22638067874041473\n",
      "Epoch 92 : Loss = 0.22134169919246965\n",
      "Epoch 93 : Loss = 0.21587422659451305\n",
      "Epoch 94 : Loss = 0.2128798140382225\n",
      "Epoch 95 : Loss = 0.20654186064546762\n",
      "Epoch 96 : Loss = 0.20072829892689528\n",
      "Epoch 97 : Loss = 0.19624402306296604\n",
      "Epoch 98 : Loss = 0.19129919904199516\n",
      "Epoch 99 : Loss = 0.1869011340154843\n",
      "Epoch 100 : Loss = 0.18245919116518716\n",
      "Epoch 101 : Loss = 0.1781877956607125\n",
      "Epoch 102 : Loss = 0.17409068972549654\n",
      "Epoch 103 : Loss = 0.1690372247248888\n",
      "Epoch 104 : Loss = 0.1649227958511223\n",
      "Epoch 105 : Loss = 0.16188679957254368\n",
      "Epoch 106 : Loss = 0.15715588798577132\n",
      "Epoch 107 : Loss = 0.15312809734181929\n",
      "Epoch 108 : Loss = 0.14928330548784946\n",
      "Epoch 109 : Loss = 0.14602637426419693\n",
      "Epoch 110 : Loss = 0.14178609204563225\n",
      "Epoch 111 : Loss = 0.13909426941113037\n",
      "Epoch 112 : Loss = 0.13543933765454727\n",
      "Epoch 113 : Loss = 0.13145645369182934\n",
      "Epoch 114 : Loss = 0.1277211123908108\n",
      "Epoch 115 : Loss = 0.12532738020474257\n",
      "Epoch 116 : Loss = 0.1220364501191811\n",
      "Epoch 117 : Loss = 0.1184554799375209\n",
      "Epoch 118 : Loss = 0.11573755038394169\n",
      "Epoch 119 : Loss = 0.11310104246843947\n",
      "Epoch 120 : Loss = 0.11003032487563114\n",
      "Epoch 121 : Loss = 0.10766817011277784\n",
      "Epoch 122 : Loss = 0.1049437622793696\n",
      "Epoch 123 : Loss = 0.10226929035376417\n",
      "Epoch 124 : Loss = 0.09989143213765186\n",
      "Epoch 125 : Loss = 0.09715388969264246\n",
      "Epoch 126 : Loss = 0.09497331556948749\n",
      "Epoch 127 : Loss = 0.09241637062620031\n",
      "Epoch 128 : Loss = 0.09016954331574117\n",
      "Epoch 129 : Loss = 0.08805930462073194\n",
      "Epoch 130 : Loss = 0.08571644466031683\n",
      "Epoch 131 : Loss = 0.08397721770134838\n",
      "Epoch 132 : Loss = 0.0819442340765487\n",
      "Epoch 133 : Loss = 0.07975930407304654\n",
      "Epoch 134 : Loss = 0.07792771184308964\n",
      "Epoch 135 : Loss = 0.07595788535069334\n",
      "Epoch 136 : Loss = 0.0747293917123567\n",
      "Epoch 137 : Loss = 0.07254878575490283\n",
      "Epoch 138 : Loss = 0.07123077322136273\n",
      "Epoch 139 : Loss = 0.06965588295662946\n",
      "Epoch 140 : Loss = 0.06804504042321985\n",
      "Epoch 141 : Loss = 0.06660473473708739\n",
      "Epoch 142 : Loss = 0.06486646814102476\n",
      "Epoch 143 : Loss = 0.0634921859441833\n",
      "Epoch 144 : Loss = 0.061972637152807285\n",
      "Epoch 145 : Loss = 0.06063222673467616\n",
      "Epoch 146 : Loss = 0.05933196927336127\n",
      "Epoch 147 : Loss = 0.05782602456482974\n",
      "Epoch 148 : Loss = 0.05672644222663208\n",
      "Epoch 149 : Loss = 0.05557826525447042\n",
      "Epoch 150 : Loss = 0.05443676230920987\n",
      "Epoch 151 : Loss = 0.05314097947187043\n",
      "Epoch 152 : Loss = 0.05212629006498239\n",
      "Epoch 153 : Loss = 0.05107733996754341\n",
      "Epoch 154 : Loss = 0.05002034023742784\n",
      "Epoch 155 : Loss = 0.048977057915180915\n",
      "Epoch 156 : Loss = 0.047862126957625144\n",
      "Epoch 157 : Loss = 0.04703306394036522\n",
      "Epoch 158 : Loss = 0.04606373578479344\n",
      "Epoch 159 : Loss = 0.0451549421995878\n",
      "Epoch 160 : Loss = 0.044243169101801774\n",
      "Epoch 161 : Loss = 0.04351390144703063\n",
      "Epoch 162 : Loss = 0.042713546490466055\n",
      "Epoch 163 : Loss = 0.041756179818714205\n",
      "Epoch 164 : Loss = 0.04112471301447262\n",
      "Epoch 165 : Loss = 0.04030929391526364\n",
      "Epoch 166 : Loss = 0.03949433738704433\n",
      "Epoch 167 : Loss = 0.038788662588393155\n",
      "Epoch 168 : Loss = 0.03807604837823998\n",
      "Epoch 169 : Loss = 0.0373739713210274\n",
      "Epoch 170 : Loss = 0.03685764511200515\n",
      "Epoch 171 : Loss = 0.03611218247731978\n",
      "Epoch 172 : Loss = 0.035459310641850934\n",
      "Epoch 173 : Loss = 0.0349002174863761\n",
      "Epoch 174 : Loss = 0.034205999902703545\n",
      "Epoch 175 : Loss = 0.03364814132113349\n",
      "Epoch 176 : Loss = 0.033063302578573886\n",
      "Epoch 177 : Loss = 0.032622493126175614\n",
      "Epoch 178 : Loss = 0.03205315806818279\n",
      "Epoch 179 : Loss = 0.031548047819259496\n",
      "Epoch 180 : Loss = 0.031058597039769993\n",
      "Epoch 181 : Loss = 0.03055690707300197\n",
      "Epoch 182 : Loss = 0.030058337587185874\n",
      "Epoch 183 : Loss = 0.02965632885355841\n",
      "Epoch 184 : Loss = 0.02916695219887929\n",
      "Epoch 185 : Loss = 0.02869315610521218\n",
      "Epoch 186 : Loss = 0.02828655561262911\n",
      "Epoch 187 : Loss = 0.02785545638339086\n",
      "Epoch 188 : Loss = 0.02749126677570695\n",
      "Epoch 189 : Loss = 0.02704440556805242\n",
      "Epoch 190 : Loss = 0.026658150663768705\n",
      "Epoch 191 : Loss = 0.02619434332221069\n",
      "Epoch 192 : Loss = 0.02586502598767931\n",
      "Epoch 193 : Loss = 0.02547773155807094\n",
      "Epoch 194 : Loss = 0.025124820372597733\n",
      "Epoch 195 : Loss = 0.024791628884320908\n",
      "Epoch 196 : Loss = 0.024461097613146368\n",
      "Epoch 197 : Loss = 0.024114505768838255\n",
      "Epoch 198 : Loss = 0.02381391086700288\n",
      "Epoch 199 : Loss = 0.023455023511566902\n",
      "Epoch 200 : Loss = 0.02315694426017052\n",
      "Epoch 201 : Loss = 0.022830045528032562\n",
      "Epoch 202 : Loss = 0.02253255677748133\n",
      "Epoch 203 : Loss = 0.022262999999590895\n",
      "Epoch 204 : Loss = 0.02196036886677823\n",
      "Epoch 205 : Loss = 0.021658099806782873\n",
      "Epoch 206 : Loss = 0.02141266030428761\n",
      "Epoch 207 : Loss = 0.021104424620385868\n",
      "Epoch 208 : Loss = 0.0208724900148809\n",
      "Epoch 209 : Loss = 0.02060197590088302\n",
      "Epoch 210 : Loss = 0.020353861153125763\n",
      "Epoch 211 : Loss = 0.020095439183271748\n",
      "Epoch 212 : Loss = 0.01983996499753134\n",
      "Epoch 213 : Loss = 0.019627690759741443\n",
      "Epoch 214 : Loss = 0.01940981607714837\n",
      "Epoch 215 : Loss = 0.01919423763386228\n",
      "Epoch 216 : Loss = 0.018923623381521208\n",
      "Epoch 217 : Loss = 0.01870703908868812\n",
      "Epoch 218 : Loss = 0.01850458372130313\n",
      "Epoch 219 : Loss = 0.018292680128731517\n",
      "Epoch 220 : Loss = 0.018102855336937035\n",
      "Epoch 221 : Loss = 0.0178779613819312\n",
      "Epoch 222 : Loss = 0.017643244048072538\n",
      "Epoch 223 : Loss = 0.017490698850121014\n",
      "Epoch 224 : Loss = 0.017274673118002033\n",
      "Epoch 225 : Loss = 0.01709243865810673\n",
      "Epoch 226 : Loss = 0.016917604741386386\n",
      "Epoch 227 : Loss = 0.01672073250467127\n",
      "Epoch 228 : Loss = 0.016533377411013295\n",
      "Epoch 229 : Loss = 0.016400159950452773\n",
      "Epoch 230 : Loss = 0.016203246608545836\n",
      "Epoch 231 : Loss = 0.016032102411951535\n",
      "Epoch 232 : Loss = 0.01584734829058024\n",
      "Epoch 233 : Loss = 0.015703463482416493\n",
      "Epoch 234 : Loss = 0.015556165101853283\n",
      "Epoch 235 : Loss = 0.01539098308421671\n",
      "Epoch 236 : Loss = 0.015220173528756606\n",
      "Epoch 237 : Loss = 0.015059913741424678\n",
      "Epoch 238 : Loss = 0.01492095410569825\n",
      "Epoch 239 : Loss = 0.014785286293111063\n",
      "Epoch 240 : Loss = 0.014638282104649323\n",
      "Epoch 241 : Loss = 0.014488605295561932\n",
      "Epoch 242 : Loss = 0.014343837614763865\n",
      "Epoch 243 : Loss = 0.014198509743437168\n",
      "Epoch 244 : Loss = 0.014073512580415058\n",
      "Epoch 245 : Loss = 0.013926624362780294\n",
      "Epoch 246 : Loss = 0.013793998842381618\n",
      "Epoch 247 : Loss = 0.013696925460614937\n",
      "Epoch 248 : Loss = 0.013535291252827103\n",
      "Epoch 249 : Loss = 0.013430008241398766\n",
      "Epoch 250 : Loss = 0.013300116724249994\n",
      "Epoch 251 : Loss = 0.013180220944129609\n",
      "Epoch 252 : Loss = 0.013049672823399309\n",
      "Epoch 253 : Loss = 0.01293659127656032\n",
      "Epoch 254 : Loss = 0.012832406675443052\n",
      "Epoch 255 : Loss = 0.012705990663644941\n",
      "Epoch 256 : Loss = 0.012604866557839241\n",
      "Epoch 257 : Loss = 0.012480465991591867\n",
      "Epoch 258 : Loss = 0.012367229273712095\n",
      "Epoch 259 : Loss = 0.012256685729053887\n",
      "Epoch 260 : Loss = 0.012159311360764233\n",
      "Epoch 261 : Loss = 0.012077071703970434\n",
      "Epoch 262 : Loss = 0.011973964833569798\n",
      "Epoch 263 : Loss = 0.011846643152900713\n",
      "Epoch 264 : Loss = 0.011757822104052388\n",
      "Epoch 265 : Loss = 0.011661314045671712\n",
      "Epoch 266 : Loss = 0.011558755089274862\n",
      "Epoch 267 : Loss = 0.01146096597552638\n",
      "Epoch 268 : Loss = 0.011375746689736845\n",
      "Epoch 269 : Loss = 0.01127962816760621\n",
      "Epoch 270 : Loss = 0.011188545666465707\n",
      "Epoch 271 : Loss = 0.011095917233350603\n",
      "Epoch 272 : Loss = 0.011006956115703693\n",
      "Epoch 273 : Loss = 0.010919978154230523\n",
      "Epoch 274 : Loss = 0.01083756317596205\n",
      "Epoch 275 : Loss = 0.010744761601513757\n",
      "Epoch 276 : Loss = 0.010666169088587842\n",
      "Epoch 277 : Loss = 0.010577247340486134\n",
      "Epoch 278 : Loss = 0.010494198406707832\n",
      "Epoch 279 : Loss = 0.010417918727563863\n",
      "Epoch 280 : Loss = 0.01034138822631741\n",
      "Epoch 281 : Loss = 0.010264202105728062\n",
      "Epoch 282 : Loss = 0.010182558188468896\n",
      "Epoch 283 : Loss = 0.010105896804651078\n",
      "Epoch 284 : Loss = 0.010030897556465457\n",
      "Epoch 285 : Loss = 0.009958461540835826\n",
      "Epoch 286 : Loss = 0.00989073884292421\n",
      "Epoch 287 : Loss = 0.009813827600076118\n",
      "Epoch 288 : Loss = 0.009737782106226818\n",
      "Epoch 289 : Loss = 0.00967283320443874\n",
      "Epoch 290 : Loss = 0.009602940948256714\n",
      "Epoch 291 : Loss = 0.009536754742095414\n",
      "Epoch 292 : Loss = 0.009450772544369098\n",
      "Epoch 293 : Loss = 0.009385740225711326\n",
      "Epoch 294 : Loss = 0.009320244034328925\n",
      "Epoch 295 : Loss = 0.009255930686115544\n",
      "Epoch 296 : Loss = 0.00919574885417453\n",
      "Epoch 297 : Loss = 0.009132612962275743\n",
      "Epoch 298 : Loss = 0.009060487352226946\n",
      "Epoch 299 : Loss = 0.009004671856845644\n",
      "Epoch 300 : Loss = 0.008939393213950098\n",
      "Epoch 301 : Loss = 0.00887982314452529\n",
      "Epoch 302 : Loss = 0.008820851174691184\n",
      "Epoch 303 : Loss = 0.008756647002883255\n",
      "Epoch 304 : Loss = 0.008701811586929991\n",
      "Epoch 305 : Loss = 0.0086388041222976\n",
      "Epoch 306 : Loss = 0.008585503896359693\n",
      "Epoch 307 : Loss = 0.008531802661970934\n",
      "Epoch 308 : Loss = 0.008467488191937182\n",
      "Epoch 309 : Loss = 0.00840854872314429\n",
      "Epoch 310 : Loss = 0.008363567186858168\n",
      "Epoch 311 : Loss = 0.00830245589498769\n",
      "Epoch 312 : Loss = 0.008250635403039103\n",
      "Epoch 313 : Loss = 0.008195714210160077\n",
      "Epoch 314 : Loss = 0.008149246843955056\n",
      "Epoch 315 : Loss = 0.008090838080864736\n",
      "Epoch 316 : Loss = 0.008040234308943829\n",
      "Epoch 317 : Loss = 0.007989200255410236\n",
      "Epoch 318 : Loss = 0.007944432891566646\n",
      "Epoch 319 : Loss = 0.007893836072815413\n",
      "Epoch 320 : Loss = 0.007845441186377271\n",
      "Epoch 321 : Loss = 0.007792710495943373\n",
      "Epoch 322 : Loss = 0.007743051360276613\n",
      "Epoch 323 : Loss = 0.007695220019245013\n",
      "Epoch 324 : Loss = 0.007647367151962085\n",
      "Epoch 325 : Loss = 0.007599059492349625\n",
      "Epoch 326 : Loss = 0.007553588958795776\n",
      "Epoch 327 : Loss = 0.007506698888557202\n",
      "Epoch 328 : Loss = 0.007465618175708436\n",
      "Epoch 329 : Loss = 0.007422103811140088\n",
      "Epoch 330 : Loss = 0.007380860106257553\n",
      "Epoch 331 : Loss = 0.0073352932760661314\n",
      "Epoch 332 : Loss = 0.007284517103636807\n",
      "Epoch 333 : Loss = 0.0072478953939439225\n",
      "Epoch 334 : Loss = 0.007204318641346286\n",
      "Epoch 335 : Loss = 0.007163819497112522\n",
      "Epoch 336 : Loss = 0.007124676273881714\n",
      "Epoch 337 : Loss = 0.007079944395544856\n",
      "Epoch 338 : Loss = 0.007037124914032491\n",
      "Epoch 339 : Loss = 0.007000057286032562\n",
      "Epoch 340 : Loss = 0.006960441739383068\n",
      "Epoch 341 : Loss = 0.006918073039163242\n",
      "Epoch 342 : Loss = 0.00688245366539129\n",
      "Epoch 343 : Loss = 0.0068458324954421695\n",
      "Epoch 344 : Loss = 0.006806069043126297\n",
      "Epoch 345 : Loss = 0.006766267988661476\n",
      "Epoch 346 : Loss = 0.00672735934759575\n",
      "Epoch 347 : Loss = 0.006692502531222999\n",
      "Epoch 348 : Loss = 0.006653279246000403\n",
      "Epoch 349 : Loss = 0.006618950971063565\n",
      "Epoch 350 : Loss = 0.006579420386432582\n",
      "Epoch 351 : Loss = 0.006545715594918215\n",
      "Epoch 352 : Loss = 0.0065080771243877025\n",
      "Epoch 353 : Loss = 0.006475396136837927\n",
      "Epoch 354 : Loss = 0.006436234805732965\n",
      "Epoch 355 : Loss = 0.006410766255364498\n",
      "Converged\n",
      "Finished Training\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxddZ3/8dc7W9N9Swpt09KWlqVA2QIqoFZFLaiUUVAQRlAEEXFwRv2JM44ijM4og6IjsqgVZLQMqGhVEAQRhLKFrdBSSqCFtrQ0pfuWZvn8/jgn7W2apGnpzUly38/H4z5ylu8993NP0/vOOd9zz1cRgZmZFa6irAswM7NsOQjMzAqcg8DMrMA5CMzMCpyDwMyswDkIzMwKnIPArJeQdJakuzN8/csk/W9Wr297zkFg7ZL0cUk1kjZIWibpTkknZF2XtS0ifhkR72uZlxSSJmZZU3skLZJ0YtZ1WMJBYG2S9C/A1cC3gX2AscCPgelZ1pVLUknWNfRW3reFxUFgO5E0GLgc+FxE/DYiNkZEQ0T8ISK+nLbpI+lqSa+lj6sl9UnXTZW0RNIXJa1IjyY+ma57i6TlkopzXu8fJM1Jp4skXSrpJUlvSLpV0rB03bj0r9zzJL0K/FVSsaSrJK2UtFDSxWmbkpb3IulnaQ1LJf1Hy2tLOlfSg5L+W9Lq9Pkn5dQ1TNLP0/e3WtLvctZ9UNLTktZImi1pSgf7MyRdJOlFSeslXSFp//R569L3WJbT/nxJtZJWSZolaVSrbV2YbmuNpGskKff9pNMPpE95Jj2i+1gnt/05SS8CL7bxPlr2/wXpPlkm6UsdvO9TJM1N6/ybpIPT5TeT/GHxh7S2/9feNqyLRIQffuzwAKYBjUBJB20uBx4BRgCVwGzginTd1PT5lwOlwMnAJmBouv4l4L0527oNuDSdviTdbhXQB7gemJmuGwcE8AugP9AXuBCYl7YfCtyTtilJn3N7uo3+aa2PAZ9J150LNADnA8XAZ4HXAKXr/wT8X7rdUuCd6fIjgRXAW9LnnQMsAvq0s68C+D0wCDgEqAfuBSYAg9P6z0nbvhtYCRyVvv//AR5ota0/AkNIPkzrgGk57+fBVm0n5sx3Ztt/AYYBfdt4Hy37f2a6Pw9LX//EdP1lwP+m0wcAG4H3pvvu/wG1QFm6flHL8/zI/pF5AX50vwdwFrB8F21eAk7OmX8/sCidngpsJidI0g/Ot6bT/wHMSKcHph8Y+6XzzwPvyXneyPTDuiTng2hCzvq/kn6wp/Mnpm1KSE5p1ed+qAFnAvel0+cCtTnr+qXP3Td93WbS8Gr13q8lDb2cZS+QBkUb7QM4Pmf+CeArOfNXAVen0z8DvpuzbkD6/sflbOuEnPW3sj1Ez6XjIOjMtt/dwb95y/4/KGfZd4GfpdO5QfDvwK057YqApcDUdN5B0I0ePjVkbXkDqNjFeeJRwCs586+ky7ZtIyIac+Y3kXzwAPwK+HB6KunDwJMR0bKt/YDb09MJa0iCoYnkQ73F4lZ1LG5n3X4kf40uy9ne9SRHBi2Wt0xExKZ0cgAwBlgVEavbeO/7AV9s2Wa63TGt3n9rr+dMb25jvmXf7LBfI2IDyb/H6LZqZsf9uiud2fbi1k9qQ26b1v/u7b1Wc/q80W20tYw5CKwtD5P8JX1qB21eI/lAbDE2XbZLETGP5EPiJODjJMHQYjFwUkQMyXmUR8TS3E3kTC8jOS3UYkyrbdUDFTnbGhQRh3SizMXAMElD2ln3rVY19ouImZ3Y7q7ssF8l9QeGk/w13RXb7sztiHP3cXv/7q1fS+nzWl7Ltz3uRhwEtpOIWAt8HbhG0qmS+kkqlXSSpO+mzWYCX5NUKakibb8715D/iqQ/4B0kfQQtrgO+JWk/gHT7HV2pdCtwiaTR6Yf2V3LexzLgbuAqSYPSjuj9Jb1zV8Wlz70T+LGkoen7f0e6+ifAhWnHtyT1l/QBSQM7//bbNRP4pKQj0iOmbwOPRsSiPdjW6yT9EHt72/+e/k4cAnySpB+ltVuBD0h6j6RS4IskoTy7ndosQw4Ca1NEXAX8C/A1kg7BxcDFQMuVM/8B1ABzgGeBJ9NlnTUTeCfw14hYmbP8B8As4G5J60k6jt/SwXZ+QvJhPwd4CriDpKO6KV3/CaCMpEN2NfBrkvP/nfGPJOfQ55P0cXwBICJqSDqYf5Rus5bk/PybFhH3kJxf/w3J0c7+wBl7uLnLgJvS01cf3Yvbvp/kPd8L/HdE7PQltoh4ATibpEN6JfAh4EMRsTVt8p8kf0is6ejKI+saLVdHmPUK6eWf10XEfrtsbLtF0jhgIVDaqv/HejgfEViPJqmvpJMllUgaDXyD5JJRM+skB4H1dAK+SXKK5imSq4y+nmlFZj2MTw2ZmRU4HxGYmRW4HndjqYqKihg3blzWZZiZ9ShPPPHEyoiobGtdjwuCcePGUVNTk3UZZmY9iqRX2lvnU0NmZgXOQWBmVuAcBGZmBc5BYGZW4BwEZmYFzkFgZlbgHARmZgUub0EgaYaSgcuf66DN1HQA8LmS7s9XLQDzl6/jO3+ez9rNDfl8GTOzHiefRwQ3kgyC3qZ0EJEfA6ekI0adnsdaWLxqM9f+7SUWrdyYz5cxM+tx8hYEEfEAsKqDJh8HfhsRr6btV+SrFoCxw/oBsHj1pl20NDMrLFn2ERwADJX0N0lPSPpEew0lXSCpRlJNXV3dHr1Y1dC+ALy6ykFgZpYryyAoAY4GPgC8n2Qc1APaahgRN0REdURUV1a2ec+kXerfp4SKAWUsdhCYme0gy5vOLQHeiIiNwEZJDwCHAwvy9YJjhvXzEYGZWStZHhH8HjghHWKwH8kA5c/n8wXHOgjMzHaStyMCSTOBqUCFpCUkY8mWAkTEdRHxvKQ/A3OAZuCnEdHupaZ7w9hh/fjjnGU0NjVTUuyvUJiZQR6DICLO7ESbK4Er81VDa2OG9qOpOVi2dgtj0quIzMwKXUH9Wdzy4e/TQ2Zm2xVUEIwd7iAwM2utoIJg30HllBbLl5CameUoqCAoLhJVQ33lkJlZroIKAki+YewjAjOz7QouCPxdAjOzHRVkEKze1MD6Lb4dtZkZFGgQQHJbajMzK8Ag8HcJzMx2VLBB4A5jM7NEwQXB4L6lDO5b6iMCM7NUwQUBJP0EHqnMzCxRsEHgIwIzs0RBBkHVsL4sWbWZ5ubIuhQzs8wVZBCMHdaPrU3NvL5+S9almJllLm9BIGmGpBWSOhxsRtIxkholnZavWlpr+S7Bq2/49JCZWT6PCG4EpnXUQFIx8B3g7jzWsZNtXypb7S+VmZnlLQgi4gFg1S6afR74DbAiX3W0ZdSQvhTJXyozM4MM+wgkjQb+Abi2E20vkFQjqaauru5Nv3ZpcRGjh/Zl0cqNb3pbZmY9XZadxVcDX4mI5l01jIgbIqI6IqorKyv3yotPrBxA7YoNe2VbZmY9Wd4Gr++EauAWSQAVwMmSGiPid13x4pP2Gcjsl96gqTkoLlJXvKSZWbeUWRBExPiWaUk3An/sqhAAmDhiAPWNzSxZvYn9hvfvqpc1M+t28hYEkmYCU4EKSUuAbwClABFxXb5et7MmjRgAwIuvb3AQmFlBy1sQRMSZu9H23HzV0Z6JLUGwYgMnTt6nq1/ezKzbKMhvFgMMLC9l5OByXnx9fdalmJllqmCDAJKjghd95ZCZFbiCDoJJIwZSu2KDbz5nZgWtsINgnwFsbmhi6RrfasLMCldhB8G2DmP3E5hZ4SroIDhg34EAPL/MQWBmhaugg2BQeSn7De/H3NfWZl2KmVlmCjoIAA4ZNYjnlq7Lugwzs8w4CEYN5tVVm1i7uSHrUszMMuEgGDUIgHmv+ajAzAqTg2DUYAD3E5hZwSr4IKgc2Id9BvXxEYGZFayCDwJIjgqe8xGBmRUoBwFw6KhBvFS3kS0NTVmXYmbW5RwEwORRg2lqDuYv9xfLzKzwOAjYfuXQc0t9esjMCk/egkDSDEkrJD3XzvqzJM2R9Kyk2ZIOz1ctu1I1tC+D+5Yy1x3GZlaA8nlEcCMwrYP1C4F3RsRhwBXADXmspUOSOGTUIF9CamYFKW9BEBEPAKs6WD87Ilans48AVfmqpTMOHT2Y+cvX09DUnGUZZmZdrrv0EZwH3NneSkkXSKqRVFNXV5eXAg4ZNYitjc28VOcRy8yssGQeBJLeRRIEX2mvTUTcEBHVEVFdWVmZlzq2dxi7n8DMCkumQSBpCvBTYHpEvJFlLeMrBtC3tNj9BGZWcDILAkljgd8C/xgRC7Kqo0VxkTh45EDm+ojAzApMSb42LGkmMBWokLQE+AZQChAR1wFfB4YDP5YE0BgR1fmqpzOmVA3h1prFNDUHxUXKshQzsy6TtyCIiDN3sf7TwKfz9fp7YkrVYG6cvYjaFRs4MB3G0syst8u8s7g7mVI1BIBnlqzJuBIzs67jIMgxoaI/A/uUMMdBYGYFxEGQo6hIHFY1mDlLfOWQmRUOB0ErU6qG8PyyddQ3+pbUZlYYHAStHF41mIam4PllviW1mRUGB0ErU8YkHcbuJzCzQuEgaGXU4HIqBpTxzGL3E5hZYXAQtCKJKVVDfERgZgXDQdCGw6uGUFu3gQ31jVmXYmaWdw6CNkwZM5gIeNaXkZpZAXAQtOGI9BvGT766ehctzcx6PgdBG4b2L2PSiAE8vqjdAdbMzHoNB0E7qscN44lXVtPUHFmXYmaWVw6Cdhw7fijrtzTywnJ/sczMejcHQTuq9xsGQM0rPj1kZr2bg6AdVUP7MnJwOY8vcoexmfVueQsCSTMkrZD0XDvrJemHkmolzZF0VL5q2ROSqB43jMcXriLC/QRm1nvl84jgRmBaB+tPAialjwuAa/NYyx45ZtxQlq/bwpLVm7Muxcwsb/IWBBHxANDRCfbpwC8i8QgwRNLIfNWzJ44Zl/QT+DJSM+vNsuwjGA0szplfki7biaQLJNVIqqmrq+uS4gAO2GcgA8tL3E9gZr1aj+gsjogbIqI6IqorKyu77HWLi0T1fkN9RGBmvVqWQbAUGJMzX5Uu61beMmE4tSs2sGLdlqxLMTPLiyyDYBbwifTqobcCayNiWYb1tOmEiRUAPPTSyowrMTPLj3xePjoTeBg4UNISSedJulDShWmTO4CXgVrgJ8BF+arlzZg8chBD+5Xy9xcdBGbWO5Xka8MRceYu1gfwuXy9/t5SVCSOm1jBQ7UriQgkZV2Smdle1SM6i7N2wsQKXl9XT+2KDVmXYma21zkIOqGln+DBWp8eMrPex0HQCWOG9WPc8H7cv6DrvsNgZtZVHASd9K6DRjD7pTfYtNXjGJtZ7+Ig6KQTD96HrY3NPOirh8ysl3EQdNIx44YxsE8J9z6/IutSzMz2KgdBJ5WVFPGOAyu5d/4Kmj18pZn1Ig6C3XDiwSNYuaGeOUvXZl2Kmdle4yDYDVMPGEFxkbhr7vKsSzEz22s6FQSS+ksqSqcPkHSKpNL8ltb9DO1fxnH7D+eOZ5d51DIz6zU6e0TwAFAuaTRwN/CPJCOQFZwPHDaSV97YxNzX1mVdipnZXtHZIFBEbAI+DPw4Ik4HDslfWd3X+w7Zl+Iiccez3e5GqWZme6TTQSDpbcBZwJ/SZcX5Kal7G+bTQ2bWy3Q2CL4AfBW4PSLmSpoA3Je/srq3Dxw2kkVvbOKZJb56yMx6vk4FQUTcHxGnRMR30k7jlRHxT3murds6ecpIykuL+PUTi3fd2Mysm+vsVUO/kjRIUn/gOWCepC934nnTJL0gqVbSpW2sHyvpPklPSZoj6eTdfwtdb1B5KdMO2ZdZT7/GloamrMsxM3tTOntqaHJErANOBe4ExpNcOdQuScXANcBJwGTgTEmTWzX7GnBrRBwJnAH8eDdqz9Tp1WNYt6WRu+e9nnUpZmZvSmeDoDT93sCpwKyIaAB21VN6LFAbES9HxFbgFmB6qzYBDEqnBwOvdbKezL1twnBGD+nLbTU+PWRmPVtng+B6YBHQH3hA0n7Ari6kHw3kfkouSZflugw4W9ISkjGMP9/WhiRdIKlGUk1dXfcYE6CoSHzk6CoerF3Ja2s2Z12Omdke62xn8Q8jYnREnByJV4B37YXXPxO4MSKqgJOBm1u+wdzq9W+IiOqIqK6srNwLL7t3nH50FRHw2yeXZF2Kmdke62xn8WBJ32v5q1zSVSRHBx1ZCozJma9Kl+U6D7gVICIeBsqBik5V3g2MGdaP4/YfzszHFtPY1Jx1OWZme6Szp4ZmAOuBj6aPdcDPd/Gcx4FJksZLKiPpDJ7Vqs2rwHsAJB1MEgTd49xPJ33ibeNYumYz93icAjProTobBPtHxDfSjt+XI+KbwISOnhARjcDFwF3A8yRXB82VdLmkU9JmXwTOl/QMMBM4N3rY13VPPHgEo4f05abZi7Iuxcxsj5R0st1mSSdExIMAko4HdtlDGhF3kHQC5y77es70POD4zpfb/ZQUF3H2W/fjO3+ez/zl6zho30G7fpKZWTfS2SOCC4FrJC2StAj4EfCZvFXVw5xxzBjKS4v42d8XZl2Kmdlu6+xVQ89ExOHAFGBK+gWwd+e1sh5kaP8yPlo9ht89vZTla7dkXY6Z2W7ZrRHKImJd+g1jgH/JQz091qdPmEBTc/Dz2T4qMLOe5c0MVam9VkUvMHZ4P046bCS/euRV1m1pyLocM7NOezNB0KOu7ukKn33n/qyvb2TGgz4qMLOeo8MgkLRe0ro2HuuBUV1UY49x6OjBvP+QffjZ3xeydpOPCsysZ+gwCCJiYEQMauMxMCI6e+lpQfnCiQewvr6Rn/z95axLMTPrlDdzasjacPDIQXzgsJHMeGghdevrsy7HzGyXHAR58KX3H8jWxmauvmdB1qWYme2SgyAPxlf056y3jOWWxxdTu2JD1uWYmXXIQZAnn3/PJPqWFvPdP8/PuhQzsw45CPKkYkAfLnznBO6e9zqPL1qVdTlmZu1yEOTReSdMYJ9Bffj2Hc/Tw26qamYFxEGQR33Livniew/kqVfX8Pune8xwzGZWYBwEefaRo6s4YswQrvjjPNZs2pp1OWZmO3EQ5FlxkfjPDx/Gms0NfPuO57Mux8xsJ3kNAknTJL0gqVbSpe20+aikeZLmSvpVPuvJysEjB3H+2ydwa80SHnn5jazLMTPbQd6CQFIxcA1wEjAZOFPS5FZtJgFfBY6PiEOAL+Srnqxd8p5JjBnWl3+9/VnqG5uyLsfMbJt8HhEcC9SmYxxvBW4Bprdqcz5wTUSsBoiIXjsCfN+yYr516mG8XLeRH9/3UtblmJltk88gGA0szplfki7LdQBwgKSHJD0iaVpbG5J0gaQaSTV1dXV5Kjf/3nFAJdOPGMW1f3vJ3zg2s24j687iEmASMBU4E/iJpCGtG0XEDRFRHRHVlZWVXVzi3vXvH5xM37Ji/vX2Z2lu9ncLzCx7+QyCpcCYnPmqdFmuJcCsiGiIiIXAApJg6LUqBvThX08+iMcWruK2Jxbv+glmZnmWzyB4HJgkabykMuAMYFarNr8jORpAUgXJqaJefyP/j1aP4djxw/j2HfNZsd6D3ZtZtvIWBBHRCFwM3AU8D9waEXMlXS7plLTZXcAbkuYB9wFfjohef32lJL79D4dR39jEJTOfpsmniMwsQ+pp98Cprq6OmpqarMvYK379xBK+dNszXPyuiXzp/QdmXY6Z9WKSnoiI6rbWZd1ZXNBOO7qKj1WP4Uf31XLf/F575ayZdXMOgox9c/ohHDxyEP9869MsW7s563LMrAA5CDJWXlrMNR8/kq2NzXzhFvcXmFnXcxB0AxMqB3D59EN5dOEqfvTX2qzLMbMC4yDoJj5y1GhOPWIUP7h3AY8t9IhmZtZ1HATdhCSuOPVQxgzrxyW3PMWqjR67wMy6hoOgGxlYXsr/nHkkb2zcyidvfJxNWxuzLsnMCoCDoJuZUjWEH515JHOWrOFLtz3jsY7NLO8cBN3Q+w7Zl6+edBB3PLucH97rzmMzy6+SrAuwtp3/9gnMX76e79+zgAP3HcC0Q0dmXZKZ9VI+IuimWu5HdMSYIXx+5lPcM+/1rEsys17KQdCNlZcWc9Onjk2+efx/T/PKGxuzLsnMeiEHQTc3uG8pPz7rKIqKxDkzHmPlhvqsSzKzXsZB0ANUDe3HjHOrWb5uC+fMeIz1WxqyLsnMehEHQQ9x9H7DuPbso3lh+XrO/0UNWxqasi7JzHqJvAaBpGmSXpBUK+nSDtp9RFJIavNe2ZZ414Ej+O/TD+eRl1dxyS1P0djUnHVJZtYL5C0IJBUD1wAnAZOBMyVNbqPdQOAS4NF81dKbnHrkaL7xocncNfd1/v33c/2FMzN70/J5RHAsUBsRL0fEVuAWYHob7a4AvgN48N5O+uTx47lo6v7MfOxVrru/1w/xbGZ5ls8gGA0szplfki7bRtJRwJiI+FNHG5J0gaQaSTV1dXV7v9Ie6EvvO5APThnJd/48nx/c86KPDMxsj2X2zWJJRcD3gHN31TYibgBugGTM4vxW1jMUFYmrP3YEfUqK+f49C9i4tZGvnnQQkrIuzcx6mHwGwVJgTM58VbqsxUDgUOBv6YfXvsAsSadERO8YnT7PSoqLuPK0KfQrK+aGB15mY30jV0w/lKIih4GZdV4+g+BxYJKk8SQBcAbw8ZaVEbEWqGiZl/Q34EsOgd1TVCQun34I/foUc/39L7N5axPfPW0KJcW+MtjMOidvQRARjZIuBu4CioEZETFX0uVATUTMytdrFxpJXDrtIAaUlXDVXxawuaGJH5xxJGUlDgMz27W89hFExB3AHa2Wfb2dtlPzWUtvJ4nPv2cS/fqUcMUf57H55hquO/toykuLsy7NzLo5/8nYy5x3wnj+68OHcf+COs6Z8Rgb6j3KmZl1zEHQC51x7Fiu/tgR1LyymrN++qhvVGdmHXIQ9FLTjxjNtWcdxfxl6/jQ/zzI4lWbsi7JzLopB0Ev9r5D9uU3nz2ODfWNnDPjMYeBmbXJQdDLHTp6MD8/9xhWbqjnw9fO5tkla7Muycy6GQdBAageN4zffPY4yoqL+NgND/PAAt+mw8y2cxAUiEn7DOT2i45j3PD+fPqmGu54dlnWJZlZN+EgKCAjBpUz8/y3cljVYC765ZNcc1+tb1ZnZg6CQjO4Xym//PRbmH7EKK686wW+MWsuWxs9wI1ZIcvs7qOWnfLSYr7/0SOoHNCHnz64kKcXr+Gnn6hmxKDyrEszswz4iKBAFRWJr31wMtedfTQvvr6B9139APe9sCLrsswsAw6CAjft0H2ZdfHxjBrclwtvfoI/zXEnslmhcRAYk/YZyM3nHcuB+w7kc796km/+YS4NTe43MCsUDgIDYPiAPvz2s8fxyePH8fOHFnHqNQ+xcOXGrMsysy7gILBtSoqL+MaHDuG6s49m6ZrNnPI/D/Ln53yqyKy3cxDYTqYdui9//PwJTBgxgAv/90ku/8M8X2Jq1ovlNQgkTZP0gqRaSZe2sf5fJM2TNEfSvZL2y2c91nlVQ/tx22fexrnHjWPGQws5/fqHeWzhqqzLMrM8yFsQSCoGrgFOAiYDZ0qa3KrZU0B1REwBfg18N1/12O4rKynislMO4dqzjmLxqk189PqH+d5fFvjbyGa9TD6PCI4FaiPi5YjYCtwCTM9tEBH3RUTLvZEfAaryWI/toZMOG8nsS9/N6UdX8cN7X+SfbnmaNZu2Zl2Wme0l+QyC0cDinPkl6bL2nAfc2dYKSRdIqpFUU1fnO2dmoby0mO+eNoUvv/9A7nx2GSd+737+8MxrPjow6wW6RWexpLOBauDKttZHxA0RUR0R1ZWVlV1bnG0jic+9ayKzLj6BUUP68vmZT/Hpm2p4bc3mrEszszchn0GwFBiTM1+VLtuBpBOBfwNOiQgPrtsDTB41iNsvOp6vfeBgZr/0Bu/93v384uFFNDf76MCsJ8pnEDwOTJI0XlIZcAYwK7eBpCOB60lCwDe66UGKi8Sn3z6Bu//5HRy131C+/vu5nH79w8x9zSOgmfU0eQuCiGgELgbuAp4Hbo2IuZIul3RK2uxKYABwm6SnJc1qZ3PWTY0Z1o9ffOpYvv+xw3m5bgMf+OGDfObmGpat9ekis55CPa2zr7q6OmpqarIuw9qwZtNWbpr9CtfeX0uxxEXvmsinjh9P37LirEszK3iSnoiI6rbWdYvOYusdhvQr45ITJ3H3F97J2/av4Mq7XuCdV97HzY+8wpaGpqzLM7N2+IjA8ubxRav4rzvn88Qrqxnev4zz3zGBc48bR3mpjxDMulpHRwQOAsuriOCRl1dx7f0v8cCCOkYP6csnjx/HaUdXMaRfWdblmRUMB4F1C7NrV3LVXxbwxCurKS8t4iNHVfHJ48cxccTArEsz6/U6CgKPWWxd5riJFRw3sYJ5r63jptmLuK1mCb989FUOrxrMh4+q4kOHj2JYfx8lmHU1HxFYZlZuqOd3Ty3lt08uZd6ydZQUiakHjuAjR43m3QePoE+J+xLM9hafGrJub/7yddz+5FJuf2opK9bXM6i8hA8ePooPHzmao8YOpahIWZdo1qM5CKzHaGoOZr+0kt8+uZQ/P7eczQ1NVAzow9QDK3nXgSM4YVIFg/uWZl2mWY/jILAeaWN9I3+Z9zp/nb+C+xfUsXZzA8VFonq/oUw9cARHjh3C4VVD/IU1s05wEFiP19jUzNOL1/DX+Su474U6nl+2DoDSYjGlaghHjNn+qBraF8mnksxyOQis16lbX8+zS9fw6MJV1CxazXNL11Kfjqs8vH8ZR4wZwuFpMBw8chAVA8ocDlbQfPmo9TqVA/vw7oP24d0H7QNAQ1MzLyxfz1OL1/DM4jXJ0cMLK2j5O2dgeQkTKvozvqI/EyoHpD+T+X5l/m9ghc1HBNZrrdvSwLNL1rLg9fUsXLmRl+s2snDlRpa2Gkhn30Hl20Jh7LB+jBjUhxEDyxkxMPk5qG+Jjyasx/MRgRWkQeWlHD+xguMnVuywfPPWJha9kYTCwpUbealuAwtXbuSPc5axdnPDTtvpU1JE5cA+24IhCYpkujJnemi/UkqKfR9H63kcBFZw+pYVc/DIQRw8ctBO69ZvaWDF+npWrL0XgKcAAApMSURBVKtnxfot1K2vT+e3sGJ9PbV1G5j90krWbWlsc9sD+5QwuF8pg/uWMqRfKUP6ljGobzI/sLyEAX2SR/8+Jdvmc6f7lRX76MO6nIPALMfA8lIGlpeyf+WADtttaWhKQ2JLGhr1rNnUwJrNW1m7uYG1mxpYs7mB+WvXsXZzI2s3b6WhadenYSXoX1ZC37JiykuL6FtaTHnuo6QoWVdSTN+yYvrktmlZl86XlRRRVlxEaXERpcWitLiIspLt8y3rSlrWFRf5i3sFKq9BIGka8AOgGPhpRPxXq/V9gF8ARwNvAB+LiEX5rMlsbygvLWbMsH6MGdav08+pb2xiw5ZGNtSnj9zpdH5jfSPr6xvZ0tDEloZmNm9tYktjE5u3NrF2cwOvp/NbGprSdc1sTa+W2huKi7Q9NFpCpKTVfLo+d7qkWBRJlBSJoqLkZ3HLQ6K4qIjiInb8Ke30vGJBcXG6Lmdb235qx20XSRQJlP4sKsqdT5dJKP25fbrjNjtss+V5RezUXrSa76FHc3kLAknFwDXAe4ElwOOSZkXEvJxm5wGrI2KipDOA7wAfy1dNZlnqU1JMnwHFDB/QZ69ut6k5qG9Mg6Nhe0hsbWqmsSloaGpma1MzDY3NNOTO5yzbNp8+Z/v6Vu1bnt/YzKatjdvmm5ojeUTQ2BQ0R9DYHDQ37/izKWJb295oe5i0ESRqHSQ7h1ASJtsDRhJKtyuJM44Zw6ffPmGv153PI4JjgdqIeBlA0i3AdCA3CKYDl6XTvwZ+JEnR0y5lMstQcZHoV1ZCTxreISJoDnYIkKamNEiam2luZsefETS1WtbUHATQ3Jxsq2WbzZEEUWybJp3PXZ+070yblu23t81kWQfPj7aen7v9dF36XiLdPwHb60yXVezlPyJa5DMIRgOLc+aXAG9pr01ENEpaCwwHVuY2knQBcAHA2LFj81WvmXURKT0N5D6JbqFHXOsWETdERHVEVFdWVmZdjplZr5LPIFgKjMmZr0qXtdlGUgkwmKTT2MzMukg+g+BxYJKk8ZLKgDOAWa3azALOSadPA/7q/gEzs66Vtz6C9Jz/xcBdJJePzoiIuZIuB2oiYhbwM+BmSbXAKpKwMDOzLpTX7xFExB3AHa2WfT1negtwej5rMDOzjvWIzmIzM8sfB4GZWYFzEJiZFbgeNx6BpDrglT18egWtvqzWjbnW/HCt+eFa82Nv1rpfRLT5RaweFwRvhqSa9gZm6G5ca3641vxwrfnRVbX61JCZWYFzEJiZFbhCC4Ibsi5gN7jW/HCt+eFa86NLai2oPgIzM9tZoR0RmJlZKw4CM7MCVzBBIGmapBck1Uq6NOt6WpO0SNKzkp6WVJMuGybpL5JeTH8Ozai2GZJWSHouZ1mbtSnxw3Q/z5F0VDeo9TJJS9N9+7Skk3PWfTWt9QVJ7+/iWsdIuk/SPElzJV2SLu92+7aDWrvdvpVULukxSc+ktX4zXT5e0qNpTf+X3hUZSX3S+dp0/bhuUOuNkhbm7Ncj0uX5+R2IbcOt9d4Hyd1PXwImAGXAM8DkrOtqVeMioKLVsu8Cl6bTlwLfyai2dwBHAc/tqjbgZOBOQMBbgUe7Qa2XAV9qo+3k9HehDzA+/R0p7sJaRwJHpdMDgQVpTd1u33ZQa7fbt+n+GZBOlwKPpvvrVuCMdPl1wGfT6YuA69LpM4D/68L92l6tNwKntdE+L78DhXJEsG385IjYCrSMn9zdTQduSqdvAk7NooiIeIDkNuG52qttOvCLSDwCDJE0smsqbbfW9kwHbomI+ohYCNSS/K50iYhYFhFPptPrgedJhm/tdvu2g1rbk9m+TffPhnS2NH0E8G6SsdFh5/3asr9/DbxHUpeModlBre3Jy+9AoQRBW+Mnd/RLnIUA7pb0hJIxmgH2iYhl6fRyYJ9sSmtTe7V11319cXooPSPnFFu3qTU9HXEkyV+E3XrftqoVuuG+lVQs6WlgBfAXkiOSNRHR2EY9O4ydDrSMnZ5JrRHRsl+/le7X70tqGbU+L/u1UIKgJzghIo4CTgI+J+kduSsjOS7sltf6dufaUtcC+wNHAMuAq7ItZ0eSBgC/Ab4QEety13W3fdtGrd1y30ZEU0QcQTJE7rHAQRmX1K7WtUo6FPgqSc3HAMOAr+SzhkIJgs6Mn5ypiFia/lwB3E7yy/t6y2Ff+nNFdhXupL3aut2+jojX0/9szcBP2H6KIvNaJZWSfLD+MiJ+my7ulvu2rVq7875N61sD3Ae8jeQ0SstgXLn1dIux03NqnZaeiouIqAd+Tp73a6EEQWfGT86MpP6SBrZMA+8DnmPHMZ3PAX6fTYVtaq+2WcAn0qsb3gqszTnNkYlW51D/gWTfQlLrGelVI+OBScBjXViXSIZrfT4ivpezqtvt2/Zq7Y77VlKlpCHpdF/gvSR9GveRjI0OO+/XTMZOb6fW+Tl/CIikLyN3v+7934F89oh3pwdJb/sCknOF/5Z1Pa1qm0ByhcUzwNyW+kjOU94LvAjcAwzLqL6ZJIf9DSTnJM9rrzaSqxmuSffzs0B1N6j15rSWOel/pJE57f8trfUF4KQurvUEktM+c4Cn08fJ3XHfdlBrt9u3wBTgqbSm54Cvp8snkIRRLXAb0CddXp7O16brJ3SDWv+a7tfngP9l+5VFefkd8C0mzMwKXKGcGjIzs3Y4CMzMCpyDwMyswDkIzMwKnIPAzKzAOQis15K0If05TtLH98L2Fkn6Tc78aZJufLPbTbd1maQv7Y1tme0uB4EVgnHAbgVBzjdQWzta0uQ3XdFelH65yP+XbY/5l8cKwX8Bb0/v6/7P6U2+rpT0eHpTr88ASJoq6e+SZgHz2tnWVSRflNpB67/oJT2XHomMkzQ/vb/8Akm/lHSipIeUjDeQe0fOwyU9nC4/P2dbX86pteV+9eOU3Of/FyRfOsq97YDZbmnvrx6z3uRSknvmfxAgvbvr2og4Jr2r40OS7k7bHgUcGsmtk9tyK3CRpIm78foTgdOBT5Hc7uTjJN/UPQX4V7bfDnkKyT3m+wNPSfoTcCjJ7RmOJflW6az0hoSvpsvPieR2xGZ7zEFgheh9wBRJLfedGUzyoboVeKyDEABoAq4kuTvknZ18vYUR8SyApLnAvRERkp4lOW3V4vcRsRnYLOk+kg//E9J6n0rbDEhrfRV4xSFge4ODwAqRgM9HxF07LJSmAhs78fybSYLguZxljex4qrU8Z7o+Z7o5Z76ZHf8Ptr7fS6S1/mdEXN+q1nGdrNVsl9xHYIVgPcnwii3uAj6b3lYZSQekd33tlIhoAL4P/HPO4kUkp5VQMo7s+D2oc7qSMWyHA1NJTiPdBXwqHQcASaMljdiDbZu1y0cEVgjmAE2SniEZC/YHJKdknkxv81vH7g8D+jPgaznzvyG5PfBckpG7FuxhnfcBFcAVEfEa8Jqkg4GHk1LZAJxNcorKbK/w3UfNzAqcTw2ZmRU4B4GZWYFzEJiZFTgHgZlZgXMQmJkVOAeBmVmBcxCYmRW4/w82Sa+bAa84ygAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "old_loss = np.inf\n",
    "\n",
    "max_epoch = 500\n",
    "losses= []\n",
    "for epoch in range(max_epoch):\n",
    "\n",
    "    running_loss = 0.0\n",
    "    \n",
    "    for data in trainloader:\n",
    "        \n",
    "        X, y = data[0], data[1].to(device)\n",
    "        \n",
    "        # applying PCA on the data to use as input to the MLFFNN\n",
    "        Z = torch.Tensor(PCA_model.transform(X)).to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Forward\n",
    "        y_hat = pca_clf(Z)\n",
    "        \n",
    "        # Calculate Loss (Cross Entropy)\n",
    "        loss = criterion(y_hat, y)\n",
    "        \n",
    "        # Backpropagation\n",
    "        loss.backward()\n",
    "        \n",
    "        # Update Parameters\n",
    "        optimizer.step()\n",
    "        \n",
    "        running_loss += loss.item()*len(X)/train_size\n",
    "    \n",
    "    print('Epoch', epoch+1, ': Loss =', running_loss)\n",
    "    losses.append(running_loss)\n",
    "\n",
    "    if abs(running_loss-old_loss)/running_loss < 0.005 and running_loss<0.01:\n",
    "        print('Converged')\n",
    "        break\n",
    "    \n",
    "    old_loss = running_loss\n",
    "\n",
    "print('Finished Training')\n",
    "plt.plot(losses)\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Iter Number')\n",
    "plt.title('Convergence monitor plot')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "colab_type": "code",
    "id": "VoXwDQ8biAU9",
    "outputId": "78a388fb-5e6c-4315-e01b-b150ee3d9718"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Loss = 0.006342008709907532\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>285</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>327</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>304</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>208</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>284</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2    3    4\n",
       "0  285    0    0    0    0\n",
       "1    0  327    0    0    0\n",
       "2    0    0  304    0    0\n",
       "3    0    0    0  208    0\n",
       "4    0    0    0    0  284"
      ]
     },
     "execution_count": 27,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    \n",
    "    train_loss = 0.0\n",
    "    y_train = []\n",
    "    y_train_pred = []\n",
    "\n",
    "    for data in trainloader:\n",
    "\n",
    "        X, y = data[0], data[1].to(device)\n",
    "        Z = torch.Tensor(PCA_model.transform(X)).to(device)\n",
    "        y_hat = pca_clf(Z)      \n",
    "        train_loss += criterion(y_hat, y)*len(X)/train_size\n",
    "        \n",
    "        y_train.extend(list(y.cpu().detach().numpy()))\n",
    "        y_train_pred.extend(list(torch.argmax(y_hat, axis=1).cpu().detach().numpy()))\n",
    "\n",
    "print('Train Loss =', train_loss.item())\n",
    "pd.DataFrame(confusion_matrix(y_train, y_train_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "267e_aGo3RCs",
    "outputId": "e461bcee-b020-45fd-ce52-121e57e2e4c4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy = 1.0 Train Precision = 1.0 Train F1 = 1.0\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score, precision_score\n",
    "acc = accuracy_score(y_train, y_train_pred)\n",
    "prec = precision_score(y_train, y_train_pred, average='macro')\n",
    "f1 = f1_score(y_train, y_train_pred, average='macro')\n",
    "\n",
    "print('Train Accuracy =', acc, 'Train Precision =', prec, 'Train F1 =', f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "colab_type": "code",
    "id": "EV0KYYroiAVB",
    "outputId": "1b4b1898-906d-43a0-cfb7-7e76ec379d82"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss = 2.3745405673980713\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>40</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>60</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14</td>\n",
       "      <td>13</td>\n",
       "      <td>32</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>36</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>11</td>\n",
       "      <td>8</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3   4\n",
       "0  40   3  18   6   4\n",
       "1   4  60   5   3  11\n",
       "2  14  13  32   4   7\n",
       "3   6   1   4  36   5\n",
       "4   3  17  11   8  37"
      ]
     },
     "execution_count": 29,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    \n",
    "    test_loss = 0.0\n",
    "    y_test = []\n",
    "    y_test_pred = []\n",
    "\n",
    "    for data in testloader:\n",
    "\n",
    "        X, y = data[0], data[1].to(device)\n",
    "        Z = torch.Tensor(PCA_model.transform(X)).to(device)\n",
    "        y_hat = pca_clf(Z)      \n",
    "        test_loss += criterion(y_hat, y)*len(X)/test_size\n",
    "        \n",
    "        y_test.extend(list(y.cpu().detach().numpy()))\n",
    "        y_test_pred.extend(list(torch.argmax(y_hat, axis=1).cpu().detach().numpy()))\n",
    "\n",
    "print('Test Loss =', test_loss.item())\n",
    "pd.DataFrame(confusion_matrix(y_test, y_test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "lku7GRBriAVG",
    "outputId": "4bd7e15b-3931-497f-db1f-d4fa32f4a7f2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy = 0.5823863636363636 Test Precision = 0.5804319204449676 Test F1 = 0.5807881982104666\n"
     ]
    }
   ],
   "source": [
    "acc = accuracy_score(y_test, y_test_pred)\n",
    "prec = precision_score(y_test, y_test_pred, average='macro')\n",
    "f1 = f1_score(y_test, y_test_pred, average='macro')\n",
    "\n",
    "print('Test Accuracy =', acc, 'Test Precision =', prec, 'Test F1 =', f1)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Q1.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}