LDL.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "LDL.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "5LLTBDU7Nuwm",
        "yCmYn3riunMF",
        "Uvhz4XTkuwWm",
        "Sy6qaOoUGjcH",
        "UcXvONf6N_ni",
        "sMQy1zWwXwYb",
        "uSD_adunU1IY",
        "XVDqqOTpBQbY",
        "ATa9pqdBcp53",
        "YYFDULfKGN-K"
      ],
      "toc_visible": true,
      "mount_file_id": "1FQMWsDB5NwR3zHj6r8SkSQtyyeiQDyge",
      "authorship_tag": "ABX9TyOOM4BauzMGnktOiaScG3aI",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/DreamWalker1412/Colab/blob/main/LDL.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rGUjPwOwsRQ2"
      },
      "source": [
        "GPU information"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PzwsDT0crfPq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b7674e62-d4e5-4341-cf37-b871f96c1900"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Thu Dec 24 09:08:27 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.27.04    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   34C    P0    26W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CZ8MI3tpBiM8"
      },
      "source": [
        "# Resource preparation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kqT52g81N14a"
      },
      "source": [
        "## Import required libraries"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KWJXxHY3f7sG"
      },
      "source": [
        "Update pandas and pandas_profiling libraries before loading instance of pandas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UVNqclCff08S"
      },
      "source": [
        "!pip install --upgrade pandas\n",
        "!pip install --upgrade pandas_profiling"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EYpiYkmOgMIe"
      },
      "source": [
        "Call tersorflow 2.x version to support auto-keras and other related libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VZIrxQOatTPR"
      },
      "source": [
        "%tensorflow_version 2.x\n",
        "import sklearn\n",
        "from sklearn import preprocessing\n",
        "from sklearn import model_selection\n",
        "from sklearn.decomposition import PCA\n",
        "from sklearn.metrics import *\n",
        "\n",
        "import scipy\n",
        "from scipy import io as spio\n",
        "from scipy import stats\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "# Drawing tools\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RxBCeEplOGtp"
      },
      "source": [
        "## Download datasets"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WS6H3sXHgld8"
      },
      "source": [
        "Decompress the data stored in the Google disk to Colab, or you can download the data from Dropbox.\n",
        "\n",
        "\n",
        "```\n",
        "!wget -c https://www.dropbox.com/sh/oxk6087vnsazr60/AABgsh2rrC79aWDaXeC-4VXIa?dl=0 -O CADA_VAE.zip\n",
        "!unzip CADA_VAE.zip\n",
        "```\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "beU9GPLc3mVb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f3d34e22-c027-4fdb-c5ca-fb5daa2a177e"
      },
      "source": [
        "!unzip /content/drive/'My Drive'/data/LDL/LDL_datasets_mat.zip\n",
        "!unzip /content/drive/'My Drive'/data/MLL/MLL_datasets_mat.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/drive/My Drive/data/LDL/LDL_datasets_mat.zip\n",
            "   creating: data_mat/\n",
            "  inflating: data_mat/Flickr.mat     \n",
            "  inflating: data_mat/Human_Gene.mat  \n",
            "  inflating: data_mat/JAFFE(Fear Excluded).mat  \n",
            "  inflating: data_mat/JAFFE.mat      \n",
            "  inflating: data_mat/Movie.mat      \n",
            "  inflating: data_mat/Natural_Scene.mat  \n",
            "  inflating: data_mat/SBU_3DFE.mat   \n",
            "  inflating: data_mat/SJAFFE.mat     \n",
            "  inflating: data_mat/Twitter.mat    \n",
            "  inflating: data_mat/Yeast_alpha.mat  \n",
            "  inflating: data_mat/Yeast_cdc.mat  \n",
            "  inflating: data_mat/Yeast_cold.mat  \n",
            "  inflating: data_mat/Yeast_diau.mat  \n",
            "  inflating: data_mat/Yeast_dtt.mat  \n",
            "  inflating: data_mat/Yeast_elu.mat  \n",
            "  inflating: data_mat/Yeast_heat.mat  \n",
            "  inflating: data_mat/Yeast_spo.mat  \n",
            "  inflating: data_mat/Yeast_spo5.mat  \n",
            "  inflating: data_mat/Yeast_spoem.mat  \n",
            "Archive:  /content/drive/My Drive/data/MLL/MLL_datasets_mat.zip\n",
            "   creating: ML_datasets/\n",
            "  inflating: ML_datasets/Arts.mat    \n",
            "  inflating: ML_datasets/business.mat  \n",
            "  inflating: ML_datasets/cal500.mat  \n",
            "  inflating: ML_datasets/enron.mat   \n",
            "  inflating: ML_datasets/flags.mat   \n",
            "  inflating: ML_datasets/genbase.mat  \n",
            "  inflating: ML_datasets/health.mat  \n",
            "  inflating: ML_datasets/Image.mat   \n",
            "  inflating: ML_datasets/llog.mat    \n",
            "  inflating: ML_datasets/medical.mat  \n",
            "  inflating: ML_datasets/slashdot.mat  \n",
            "  inflating: ML_datasets/society.mat  \n",
            "  inflating: ML_datasets/yeast.mat   \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BsbG4Qq0gciB"
      },
      "source": [
        "Store the dataset path in a dictionary structure"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hRDCt3E97e6C"
      },
      "source": [
        "data_path = dict(\n",
        "    # Label distribution learning datasets, beginning with an uppercase letter.\n",
        "    Human_Gene='/content/data_mat/Human_Gene.mat',\n",
        "    Movie='/content/data_mat/Movie.mat',\n",
        "    Natural_Scene='/content/data_mat/Natural_Scene.mat',\n",
        "    SBU_3DFE='/content/data_mat/SBU_3DFE.mat',\n",
        "    SJAFFE='/content/data_mat/SJAFFE.mat',\n",
        "    Yeast_alpha='/content/data_mat/Yeast_alpha.mat',\n",
        "    Yeast_cdc='/content/data_mat/Yeast_cdc.mat',\n",
        "    Yeast_cold='/content/data_mat/Yeast_cold.mat',\n",
        "    Yeast_diau='/content/data_mat/Yeast_diau.mat',\n",
        "    Yeast_dtt='/content/data_mat/Yeast_dtt.mat',\n",
        "    Yeast_elu='/content/data_mat/Yeast_elu.mat',\n",
        "    Yeast_heat='/content/data_mat/Yeast_heat.mat',\n",
        "    Yeast_spo='/content/data_mat/Yeast_spo.mat',\n",
        "    Yeast_spo5='/content/data_mat/Yeast_spo5.mat',\n",
        "    Yeast_spoem='/content/data_mat/Yeast_spoem.mat',\n",
        "    Flickr='/content/data_mat/Flickr.mat',\n",
        "    Twitter='/content/data_mat/Twitter.mat',\n",
        "\n",
        "    # Muti-label learning datasets, beginning with a lowercase letter.\n",
        "    yahoo_arts='/content/ML_datasets/Arts.mat', \n",
        "    image='/content/ML_datasets/Image.mat',\n",
        "    yahoo_business='/content/ML_datasets/business.mat',\n",
        "    cal500='/content/ML_datasets/cal500.mat',\n",
        "    enron='/content/ML_datasets/enron.mat',\n",
        "    flags='/content/ML_datasets/flags.mat',\n",
        "    genbase='/content/ML_datasets/genbase.mat',\n",
        "    health='/content/ML_datasets/health.mat',\n",
        "    llog='/content/ML_datasets/llog.mat',\n",
        "    medical='/content/ML_datasets/medical.mat',\n",
        "    slashdot='/content/ML_datasets/slashdot.mat',\n",
        "    society='/content/ML_datasets/society.mat',\n",
        "    yeast='/content/ML_datasets/yeast.mat'\n",
        "    )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BWAAx3vSBxdM"
      },
      "source": [
        "## Load data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YD3x-OuKrkfk"
      },
      "source": [
        "Use \"loadmat\" function of scipy to read data in mat format. If it is csv format data, use pandas to read."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CpJ3S7Ml4_Ra"
      },
      "source": [
        "dataset_name = 'SBU_3DFE'\n",
        "data = spio.loadmat(data_path[dataset_name])\n",
        "X = data['features']\n",
        "y = data['labels']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZqMYyXRw9aWq",
        "outputId": "13c35993-32db-4622-d2f8-e9a304d5ef88"
      },
      "source": [
        "X_train,X_test,y_train,y_test = model_selection.train_test_split(X,y,test_size=0.2)\n",
        "print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(2000, 243) (500, 243) (2000, 6) (500, 6)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mRO-MtdQa02q"
      },
      "source": [
        "convent to single label and muli-label"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hJ7R0IAJU2At"
      },
      "source": [
        "import numpy as np\n",
        "import copy\n",
        "\n",
        "# single label\n",
        "def get_single_label(y):\n",
        "  maxInRows = np.amax(y,axis=1)\n",
        "  result = [np.where(y[i] == maxInRows[i],1,0) for i in range(len(y))]\n",
        "  single_label = np.array(result)\n",
        "  return single_label\n",
        "\n",
        "# multi-label\n",
        "def get_multi_label(y,set_threshold=0.3):\n",
        "  ls = list() \n",
        "  index_sort = np.argsort(-y,axis=-1)\n",
        "  for i in range(y.shape[0]):\n",
        "    sum = 0\n",
        "    y_sort_row = y[i][index_sort[i]]\n",
        "    for j in range(y.shape[1]):\n",
        "      sum += y_sort_row[j]\n",
        "      if sum>=set_threshold:\n",
        "        ls.append(y_sort_row[j])\n",
        "        break\n",
        "  result = [np.where(y[i] >= ls[i],1,0) for i in range(len(y))]\n",
        "  multi_label = np.array(result)\n",
        "  return multi_label"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CanQJ1Ybho7J",
        "outputId": "b082239c-b53f-476d-8dfd-10734d2aa619"
      },
      "source": [
        "y_sl_train = get_single_label(y_train)\n",
        "y_sl_test = get_single_label(y_test)\n",
        "\n",
        "y_ml_train = get_multi_label(y_train)\n",
        "y_ml_test = get_multi_label(y_test)\n",
        "print(y_sl_train.shape, y_sl_test.shape)\n",
        "print(y_ml_train.shape, y_ml_test.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(2000, 6) (500, 6)\n",
            "(2000, 6) (500, 6)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xP12CosGjSox"
      },
      "source": [
        "Convert to DataFrame format data, check the statistical characteristics of features and labels."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284
        },
        "id": "j_rJti_Mjf2W",
        "outputId": "2cb290df-4cb2-4de5-e5a8-7ed638968fdd"
      },
      "source": [
        "features = pd.DataFrame(X)\n",
        "labels = pd.DataFrame(get_single_label(y))\n",
        "\n",
        "labels.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.102870</td>\n",
              "      <td>0.126099</td>\n",
              "      <td>0.594888</td>\n",
              "      <td>0.057937</td>\n",
              "      <td>0.020628</td>\n",
              "      <td>0.047803</td>\n",
              "      <td>0.068969</td>\n",
              "      <td>0.081525</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.303803</td>\n",
              "      <td>0.331975</td>\n",
              "      <td>0.490936</td>\n",
              "      <td>0.233635</td>\n",
              "      <td>0.142141</td>\n",
              "      <td>0.213358</td>\n",
              "      <td>0.253412</td>\n",
              "      <td>0.273651</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                  0             1  ...             6             7\n",
              "count  11150.000000  11150.000000  ...  11150.000000  11150.000000\n",
              "mean       0.102870      0.126099  ...      0.068969      0.081525\n",
              "std        0.303803      0.331975  ...      0.253412      0.273651\n",
              "min        0.000000      0.000000  ...      0.000000      0.000000\n",
              "25%        0.000000      0.000000  ...      0.000000      0.000000\n",
              "50%        0.000000      0.000000  ...      0.000000      0.000000\n",
              "75%        0.000000      0.000000  ...      0.000000      0.000000\n",
              "max        1.000000      1.000000  ...      1.000000      1.000000\n",
              "\n",
              "[8 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wNx6wo6DGxx3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284
        },
        "outputId": "11c7dfa4-028e-4768-94ed-69b529f8490f"
      },
      "source": [
        "# Special for emotion datasets: \"SJAFFE\" & \"SBU_3DFE\"\n",
        "if dataset_name in ['SJAFFE','SBU_3DFE']:\n",
        "  labels = pd.DataFrame(y,columns=['Happy', 'Sad', 'Surprise', 'Anger', 'Disgust', 'Fear'])\n",
        "elif dataset_name in ['Flickr','Twitter']:\n",
        "  labels = pd.DataFrame(y,columns=['Amusement', 'Awe', 'Contentment', 'Excitement', 'Anger', 'Disgust','Fear','Sadness'])\n",
        "\n",
        "labels.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Amusement</th>\n",
              "      <th>Awe</th>\n",
              "      <th>Contentment</th>\n",
              "      <th>Excitement</th>\n",
              "      <th>Anger</th>\n",
              "      <th>Disgust</th>\n",
              "      <th>Fear</th>\n",
              "      <th>Sadness</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.111570</td>\n",
              "      <td>0.115793</td>\n",
              "      <td>0.429735</td>\n",
              "      <td>0.091431</td>\n",
              "      <td>0.024688</td>\n",
              "      <td>0.063041</td>\n",
              "      <td>0.077350</td>\n",
              "      <td>0.086392</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.156394</td>\n",
              "      <td>0.187092</td>\n",
              "      <td>0.245724</td>\n",
              "      <td>0.138768</td>\n",
              "      <td>0.076649</td>\n",
              "      <td>0.127267</td>\n",
              "      <td>0.138386</td>\n",
              "      <td>0.166581</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.272727</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.090909</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.454545</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.181818</td>\n",
              "      <td>0.181818</td>\n",
              "      <td>0.636364</td>\n",
              "      <td>0.090909</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.090909</td>\n",
              "      <td>0.090909</td>\n",
              "      <td>0.090909</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>0.909091</td>\n",
              "      <td>0.909091</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.909091</td>\n",
              "      <td>0.909091</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          Amusement           Awe  ...          Fear       Sadness\n",
              "count  11150.000000  11150.000000  ...  11150.000000  11150.000000\n",
              "mean       0.111570      0.115793  ...      0.077350      0.086392\n",
              "std        0.156394      0.187092  ...      0.138386      0.166581\n",
              "min        0.000000      0.000000  ...      0.000000      0.000000\n",
              "25%        0.000000      0.000000  ...      0.000000      0.000000\n",
              "50%        0.090909      0.000000  ...      0.000000      0.000000\n",
              "75%        0.181818      0.181818  ...      0.090909      0.090909\n",
              "max        0.909091      0.909091  ...      1.000000      1.000000\n",
              "\n",
              "[8 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284
        },
        "id": "T3nd7JgqMJt2",
        "outputId": "7647e672-e728-48d8-cbf2-efdf74a0a16a"
      },
      "source": [
        "labels_T = labels.transpose()\n",
        "labels_T.std()\n",
        "label_normalization =  (labels_T-labels_T.min())/(labels_T.max()-labels_T.min())\n",
        "label_normalization.transpose().describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Amusement</th>\n",
              "      <th>Awe</th>\n",
              "      <th>Contentment</th>\n",
              "      <th>Excitement</th>\n",
              "      <th>Anger</th>\n",
              "      <th>Disgust</th>\n",
              "      <th>Fear</th>\n",
              "      <th>Sadness</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "      <td>11150.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.229528</td>\n",
              "      <td>0.219736</td>\n",
              "      <td>0.755411</td>\n",
              "      <td>0.177816</td>\n",
              "      <td>0.056368</td>\n",
              "      <td>0.125741</td>\n",
              "      <td>0.162405</td>\n",
              "      <td>0.167480</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.321093</td>\n",
              "      <td>0.344294</td>\n",
              "      <td>0.341291</td>\n",
              "      <td>0.268416</td>\n",
              "      <td>0.176366</td>\n",
              "      <td>0.244504</td>\n",
              "      <td>0.282105</td>\n",
              "      <td>0.297326</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.111111</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.250000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.250000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.166667</td>\n",
              "      <td>0.200000</td>\n",
              "      <td>0.200000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          Amusement           Awe  ...          Fear       Sadness\n",
              "count  11150.000000  11150.000000  ...  11150.000000  11150.000000\n",
              "mean       0.229528      0.219736  ...      0.162405      0.167480\n",
              "std        0.321093      0.344294  ...      0.282105      0.297326\n",
              "min        0.000000      0.000000  ...      0.000000      0.000000\n",
              "25%        0.000000      0.000000  ...      0.000000      0.000000\n",
              "50%        0.111111      0.000000  ...      0.000000      0.000000\n",
              "75%        0.333333      0.250000  ...      0.200000      0.200000\n",
              "max        1.000000      1.000000  ...      1.000000      1.000000\n",
              "\n",
              "[8 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284
        },
        "id": "_UyUXMPlPCuF",
        "outputId": "6ed54d8d-064e-4a23-d0c5-d9f5981ee388"
      },
      "source": [
        "labels_T.std().describe()\n",
        "labels_z = (labels_T-labels_T.mean())/labels_T.std()\n",
        "labels_z.transpose().describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Happy</th>\n",
              "      <th>Sad</th>\n",
              "      <th>Surprise</th>\n",
              "      <th>Anger</th>\n",
              "      <th>Disgust</th>\n",
              "      <th>Fear</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>2500.000000</td>\n",
              "      <td>2500.000000</td>\n",
              "      <td>2500.000000</td>\n",
              "      <td>2500.000000</td>\n",
              "      <td>2500.000000</td>\n",
              "      <td>2500.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>-0.022777</td>\n",
              "      <td>-0.024897</td>\n",
              "      <td>0.349866</td>\n",
              "      <td>0.021246</td>\n",
              "      <td>-0.235502</td>\n",
              "      <td>-0.087935</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>1.133335</td>\n",
              "      <td>0.793839</td>\n",
              "      <td>1.009938</td>\n",
              "      <td>0.664282</td>\n",
              "      <td>0.598188</td>\n",
              "      <td>1.039727</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>-1.932503</td>\n",
              "      <td>-1.488131</td>\n",
              "      <td>-1.651388</td>\n",
              "      <td>-1.459804</td>\n",
              "      <td>-1.936583</td>\n",
              "      <td>-1.852939</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>-0.770892</td>\n",
              "      <td>-0.534903</td>\n",
              "      <td>-0.463033</td>\n",
              "      <td>-0.430860</td>\n",
              "      <td>-0.586595</td>\n",
              "      <td>-0.745163</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>-0.560248</td>\n",
              "      <td>-0.408248</td>\n",
              "      <td>-0.193782</td>\n",
              "      <td>-0.242527</td>\n",
              "      <td>-0.408248</td>\n",
              "      <td>-0.529301</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.550482</td>\n",
              "      <td>0.346046</td>\n",
              "      <td>1.447817</td>\n",
              "      <td>0.333323</td>\n",
              "      <td>-0.098439</td>\n",
              "      <td>0.000200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>2.041241</td>\n",
              "      <td>2.036787</td>\n",
              "      <td>2.037014</td>\n",
              "      <td>2.026201</td>\n",
              "      <td>1.999115</td>\n",
              "      <td>2.041241</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             Happy          Sad  ...      Disgust         Fear\n",
              "count  2500.000000  2500.000000  ...  2500.000000  2500.000000\n",
              "mean     -0.022777    -0.024897  ...    -0.235502    -0.087935\n",
              "std       1.133335     0.793839  ...     0.598188     1.039727\n",
              "min      -1.932503    -1.488131  ...    -1.936583    -1.852939\n",
              "25%      -0.770892    -0.534903  ...    -0.586595    -0.745163\n",
              "50%      -0.560248    -0.408248  ...    -0.408248    -0.529301\n",
              "75%       0.550482     0.346046  ...    -0.098439     0.000200\n",
              "max       2.041241     2.036787  ...     1.999115     2.041241\n",
              "\n",
              "[8 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284
        },
        "id": "cBwYAagbnDIL",
        "outputId": "d5c396bd-44f0-4158-bfde-acf1974f096b"
      },
      "source": [
        "# Special for emotion datasets: \"SJAFFE\" & \"SBU_3DFE\"\n",
        "if dataset_name in ['SJAFFE','SBU_3DFE']:\n",
        "  label_train = pd.DataFrame(y_train,columns=['Happy', 'Sad', 'Surprise', 'Anger', 'Disgust', 'Fear'])\n",
        "else:\n",
        "  label_train = pd.DataFrame(y_train)\n",
        "\n",
        "label_train.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Happy</th>\n",
              "      <th>Sad</th>\n",
              "      <th>Surprise</th>\n",
              "      <th>Anger</th>\n",
              "      <th>Disgust</th>\n",
              "      <th>Fear</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1750.000000</td>\n",
              "      <td>1750.000000</td>\n",
              "      <td>1750.000000</td>\n",
              "      <td>1750.000000</td>\n",
              "      <td>1750.000000</td>\n",
              "      <td>1750.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.178437</td>\n",
              "      <td>0.156376</td>\n",
              "      <td>0.180185</td>\n",
              "      <td>0.159376</td>\n",
              "      <td>0.151039</td>\n",
              "      <td>0.174587</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.109652</td>\n",
              "      <td>0.056219</td>\n",
              "      <td>0.075269</td>\n",
              "      <td>0.044231</td>\n",
              "      <td>0.041548</td>\n",
              "      <td>0.096124</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.083333</td>\n",
              "      <td>0.085714</td>\n",
              "      <td>0.085714</td>\n",
              "      <td>0.085714</td>\n",
              "      <td>0.085714</td>\n",
              "      <td>0.090452</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.111111</td>\n",
              "      <td>0.115699</td>\n",
              "      <td>0.118422</td>\n",
              "      <td>0.122179</td>\n",
              "      <td>0.125002</td>\n",
              "      <td>0.116883</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.126866</td>\n",
              "      <td>0.137808</td>\n",
              "      <td>0.155708</td>\n",
              "      <td>0.154103</td>\n",
              "      <td>0.140422</td>\n",
              "      <td>0.132206</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.181101</td>\n",
              "      <td>0.180440</td>\n",
              "      <td>0.223698</td>\n",
              "      <td>0.181822</td>\n",
              "      <td>0.161022</td>\n",
              "      <td>0.167802</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>0.488638</td>\n",
              "      <td>0.414014</td>\n",
              "      <td>0.416666</td>\n",
              "      <td>0.341931</td>\n",
              "      <td>0.391959</td>\n",
              "      <td>0.459456</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             Happy          Sad  ...      Disgust         Fear\n",
              "count  1750.000000  1750.000000  ...  1750.000000  1750.000000\n",
              "mean      0.178437     0.156376  ...     0.151039     0.174587\n",
              "std       0.109652     0.056219  ...     0.041548     0.096124\n",
              "min       0.083333     0.085714  ...     0.085714     0.090452\n",
              "25%       0.111111     0.115699  ...     0.125002     0.116883\n",
              "50%       0.126866     0.137808  ...     0.140422     0.132206\n",
              "75%       0.181101     0.180440  ...     0.161022     0.167802\n",
              "max       0.488638     0.414014  ...     0.391959     0.459456\n",
              "\n",
              "[8 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284
        },
        "id": "uLwqT54FnKe4",
        "outputId": "1c9c3828-3220-42fa-ab43-4da0d00a0c70"
      },
      "source": [
        "# Special for emotion datasets: \"SJAFFE\" & \"SBU_3DFE\"\n",
        "if dataset_name in ['SJAFFE','SBU_3DFE']:\n",
        "  label_test = pd.DataFrame(y_test,columns=['Happy', 'Sad', 'Surprise', 'Anger', 'Disgust', 'Fear'])\n",
        "else:\n",
        "  label_test = pd.DataFrame(y_test)\n",
        "  \n",
        "label_test.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Happy</th>\n",
              "      <th>Sad</th>\n",
              "      <th>Surprise</th>\n",
              "      <th>Anger</th>\n",
              "      <th>Disgust</th>\n",
              "      <th>Fear</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>750.000000</td>\n",
              "      <td>750.000000</td>\n",
              "      <td>750.000000</td>\n",
              "      <td>750.000000</td>\n",
              "      <td>750.000000</td>\n",
              "      <td>750.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.180769</td>\n",
              "      <td>0.156049</td>\n",
              "      <td>0.180289</td>\n",
              "      <td>0.160913</td>\n",
              "      <td>0.148441</td>\n",
              "      <td>0.173539</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.107860</td>\n",
              "      <td>0.053896</td>\n",
              "      <td>0.073252</td>\n",
              "      <td>0.046159</td>\n",
              "      <td>0.036113</td>\n",
              "      <td>0.093595</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.078602</td>\n",
              "      <td>0.090000</td>\n",
              "      <td>0.090000</td>\n",
              "      <td>0.091837</td>\n",
              "      <td>0.093264</td>\n",
              "      <td>0.091892</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.113334</td>\n",
              "      <td>0.117075</td>\n",
              "      <td>0.121023</td>\n",
              "      <td>0.124391</td>\n",
              "      <td>0.125000</td>\n",
              "      <td>0.117647</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.130601</td>\n",
              "      <td>0.138071</td>\n",
              "      <td>0.160005</td>\n",
              "      <td>0.155050</td>\n",
              "      <td>0.141628</td>\n",
              "      <td>0.132231</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.192511</td>\n",
              "      <td>0.179963</td>\n",
              "      <td>0.217211</td>\n",
              "      <td>0.181820</td>\n",
              "      <td>0.159420</td>\n",
              "      <td>0.162538</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>0.494050</td>\n",
              "      <td>0.422624</td>\n",
              "      <td>0.437498</td>\n",
              "      <td>0.325926</td>\n",
              "      <td>0.335050</td>\n",
              "      <td>0.442107</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            Happy         Sad    Surprise       Anger     Disgust        Fear\n",
              "count  750.000000  750.000000  750.000000  750.000000  750.000000  750.000000\n",
              "mean     0.180769    0.156049    0.180289    0.160913    0.148441    0.173539\n",
              "std      0.107860    0.053896    0.073252    0.046159    0.036113    0.093595\n",
              "min      0.078602    0.090000    0.090000    0.091837    0.093264    0.091892\n",
              "25%      0.113334    0.117075    0.121023    0.124391    0.125000    0.117647\n",
              "50%      0.130601    0.138071    0.160005    0.155050    0.141628    0.132231\n",
              "75%      0.192511    0.179963    0.217211    0.181820    0.159420    0.162538\n",
              "max      0.494050    0.422624    0.437498    0.325926    0.335050    0.442107"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_lxNj1cPC-1Z"
      },
      "source": [
        "Distribution to logic label"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vLCTGQmdIXcU"
      },
      "source": [
        "single_label = get_single_label(y)\n",
        "multi_label = get_multi_label(y)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "asX1iGPIqsgP"
      },
      "source": [
        "def get_label_groups(X=X,y=y):\n",
        "  logic_label = get_single_label(y)\n",
        "  X_ls = [[]*j for j in range(logic_label.shape[1])]\n",
        "  y_ls = [[]*j for j in range(logic_label.shape[1])]\n",
        "  for i in range(logic_label.shape[0]):\n",
        "    for j in range(logic_label.shape[1]):\n",
        "      if logic_label[i][j] == 1:\n",
        "        X_ls[j].append(X[i])\n",
        "        y_ls[j].append(y[i])\n",
        "  for j in range(y.shape[1]):\n",
        "    X_ls[j]=np.asarray(X_ls[j])\n",
        "    y_ls[j]=np.asarray(y_ls[j])\n",
        "  return X_ls,y_ls"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1BneW9jfvI7w"
      },
      "source": [
        "(X_ls,y_ls) = get_label_groups(X_train,y_train)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0sQp-meav3zq"
      },
      "source": [
        "import copy\n",
        "\n",
        "def cal_kurt_skew_entropy(labels):\n",
        "  label_kurtosis = scipy.stats.kurtosis(labels, axis=1, fisher=False, bias=True)\n",
        "  label_skewness = scipy.stats.skew(labels, axis=1, bias=True)\n",
        "  label_entropy = scipy.stats.entropy(labels, axis=1)\n",
        "  label_kurt_skew_entropy = np.c_[labels,label_kurtosis,label_skewness,label_entropy]\n",
        "  return label_kurt_skew_entropy\n",
        "\n",
        "def label_enhancement(label_groups):\n",
        "  label_enhanced = copy.deepcopy(label_groups)\n",
        "  num_label = label_groups[0].shape[1]\n",
        "  ls = []\n",
        "  for j in range(num_label):\n",
        "    ls.append(y_ls[j].mean(axis=0))\n",
        "  group_mean = np.asarray(ls) \n",
        "  mean_arr = np.asarray([group_mean[j,j] for j in range(num_label)])\n",
        "  # dif = mean_arr - mean_arr.mean()\n",
        "  dif = mean_arr - 0.6\n",
        "  for j in range(num_label):\n",
        "    labels = label_enhanced[j]\n",
        "    labels += dif[j]/(num_label-1)\n",
        "    labels[:,j] -= dif[j]/(num_label-1)+dif[j]\n",
        "    label_enhanced[j] = labels\n",
        "  return label_enhanced"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GakcDbJ_Mp0Q"
      },
      "source": [
        "label_enhanced = label_enhancement(y_ls)\n",
        "X_train_new = np.vstack( [X_ls[j] for j in range(y.shape[1])])\n",
        "y_train_new = np.vstack( [label_enhanced[j] for j in range(y.shape[1])])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284
        },
        "id": "wtLc7VwaaZFL",
        "outputId": "2562a4d0-75c3-43c9-a83e-2a5fa2bd9ab0"
      },
      "source": [
        "# Special for emotion datasets: \"SJAFFE\" & \"SBU_3DFE\"\n",
        "if dataset_name in ['SBU_3DFE','SJAFFE']:\n",
        "  y_train_new_pd = pd.DataFrame(y_train_new,columns=['Happy', 'Sad', 'Surprise', 'Anger', 'Disgust', 'Fear'])\n",
        "else:\n",
        "  y_train_new_pd = pd.DataFrame(y_train_new)\n",
        "y_train_new_pd.describe()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Happy</th>\n",
              "      <th>Sad</th>\n",
              "      <th>Surprise</th>\n",
              "      <th>Anger</th>\n",
              "      <th>Disgust</th>\n",
              "      <th>Fear</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1802.000000</td>\n",
              "      <td>1802.000000</td>\n",
              "      <td>1802.000000</td>\n",
              "      <td>1802.000000</td>\n",
              "      <td>1802.000000</td>\n",
              "      <td>1802.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.181885</td>\n",
              "      <td>0.155310</td>\n",
              "      <td>0.233079</td>\n",
              "      <td>0.142013</td>\n",
              "      <td>0.116660</td>\n",
              "      <td>0.171053</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.228080</td>\n",
              "      <td>0.183305</td>\n",
              "      <td>0.237202</td>\n",
              "      <td>0.152323</td>\n",
              "      <td>0.127316</td>\n",
              "      <td>0.212022</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.014155</td>\n",
              "      <td>0.020597</td>\n",
              "      <td>0.020597</td>\n",
              "      <td>0.022967</td>\n",
              "      <td>0.022284</td>\n",
              "      <td>0.025399</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.048208</td>\n",
              "      <td>0.063130</td>\n",
              "      <td>0.067000</td>\n",
              "      <td>0.071811</td>\n",
              "      <td>0.065468</td>\n",
              "      <td>0.054701</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.065469</td>\n",
              "      <td>0.080808</td>\n",
              "      <td>0.096594</td>\n",
              "      <td>0.092747</td>\n",
              "      <td>0.079914</td>\n",
              "      <td>0.071779</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.117889</td>\n",
              "      <td>0.119894</td>\n",
              "      <td>0.537402</td>\n",
              "      <td>0.121795</td>\n",
              "      <td>0.105361</td>\n",
              "      <td>0.106519</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>0.728537</td>\n",
              "      <td>0.753334</td>\n",
              "      <td>0.739207</td>\n",
              "      <td>0.697998</td>\n",
              "      <td>0.729385</td>\n",
              "      <td>0.705289</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             Happy          Sad  ...      Disgust         Fear\n",
              "count  1802.000000  1802.000000  ...  1802.000000  1802.000000\n",
              "mean      0.181885     0.155310  ...     0.116660     0.171053\n",
              "std       0.228080     0.183305  ...     0.127316     0.212022\n",
              "min       0.014155     0.020597  ...     0.022284     0.025399\n",
              "25%       0.048208     0.063130  ...     0.065468     0.054701\n",
              "50%       0.065469     0.080808  ...     0.079914     0.071779\n",
              "75%       0.117889     0.119894  ...     0.105361     0.106519\n",
              "max       0.728537     0.753334  ...     0.729385     0.705289\n",
              "\n",
              "[8 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "geYaEMc6ONVy"
      },
      "source": [
        "# Data analysis\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5LLTBDU7Nuwm"
      },
      "source": [
        "## Pandas_profiling\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N511BnAJB3jc"
      },
      "source": [
        "If you need further analysis, you can form an HTML report by pandas_profiling.\n",
        "\n",
        "*Note that in the colab environment, the pandas and pandas_profiling libraries need to be updated before the pandas instance runs, and the data needs to be converted to the DataFrame type in pandas.*\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yGyRqbHzJP7q"
      },
      "source": [
        "import pandas_profiling\n",
        "# profile_feature = pandas_profiling.ProfileReport(features)\n",
        "# profile_feature.to_file(\"feature report.html\") \n",
        "profile_label = pandas_profiling.ProfileReport(labels)\n",
        "profile_label.to_file(\"/content/drive/MyDrive/ProfileReport/label_report.html\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o3fbvcq8KPkh"
      },
      "source": [
        "def changeNumToChar(toSmallChar=None, toBigChar=None):\n",
        "  init_number=0\n",
        "  increment=0\n",
        "  res_char = ''\n",
        "  if toSmallChar!=None:\n",
        "    init_number = toSmallChar\n",
        "    increment = ord('a')\n",
        "  else:\n",
        "\t  init_number = toBigChar\n",
        "\t  increment = ord('A')\n",
        "  shang,yu = divmod(init_number, 26)\n",
        "  char = chr(yu + increment)\n",
        "  res_char = char * (shang + 1)\n",
        "  return res_char"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j9z9ZC_jS4W4"
      },
      "source": [
        "analysis_dict = dict( \n",
        "    SBU_3DFE='/content/data_mat/SBU_3DFE.mat',\n",
        "    SJAFFE='/content/data_mat/SJAFFE.mat',\n",
        "    Flickr='/content/data_mat/Flickr.mat',\n",
        "    Twitter='/content/data_mat/Twitter.mat',)\n",
        "\n",
        "label_list_1 = ['Happy', 'Sad', 'Surprise', 'Anger', 'Disgust', 'Fear', 'Kurtosis', 'Skewness', 'Entropy']\n",
        "label_list_2 = ['Amusement', 'Awe', 'Contentment', 'Excitement', 'Anger', 'Disgust','Fear', 'Sadness', 'Kurtosis', 'Skewness', 'Entropy']\n",
        "\n",
        "import pandas_profiling\n",
        "for name in analysis_dict:\n",
        "  data = spio.loadmat(data_path[name])\n",
        "  X = data['features']\n",
        "  y = data['labels']\n",
        "  \n",
        "  # features = pd.DataFrame(X)\n",
        "  # profile_feature = pandas_profiling.ProfileReport(features)\n",
        "  # profile_feature.to_file(name+\"_feature_report.html\")\n",
        "  if name!='Human_Gene':\n",
        "    labels = pd.DataFrame(y)\n",
        "    label_kurtosis = scipy.stats.kurtosis(labels, axis=1, fisher=False, bias=True)\n",
        "    label_skewness = scipy.stats.skew(labels, axis=1, bias=True)\n",
        "    label_entropy = scipy.stats.entropy(labels, axis=1)\n",
        "    label_kurt_skew_entropy = np.c_[y,label_kurtosis,label_skewness,label_entropy]\n",
        "    if name in ['SBU_3DFE','SJAFFE']:\n",
        "      label_kurt_skew_entropy = pd.DataFrame(label_kurt_skew_entropy,columns=label_list_1)\n",
        "    elif name in ['Flickr','Twitter']:\n",
        "      label_kurt_skew_entropy = pd.DataFrame(label_kurt_skew_entropy,columns=label_list_2)\n",
        "    else:\n",
        "      num_label = label_kurt_skew_entropy.shape[1]\n",
        "      label_kurt_skew_entropy = pd.DataFrame(label_kurt_skew_entropy,columns=[\"label(\"+changeNumToChar(i)+')' for i in range(num_label)])\n",
        "      label_kurt_skew_entropy.rename(columns={'label('+changeNumToChar(num_label-3)+')':'kurtosis',\n",
        "                          'label('+changeNumToChar(num_label-2)+')':'skewness',\n",
        "                          'label('+changeNumToChar(num_label-1)+')':'entropy'},inplace=True)\n",
        "    profile_label = pandas_profiling.ProfileReport(label_kurt_skew_entropy)\n",
        "    profile_label.to_file(\"/content/drive/MyDrive/ProfileReport/LDL/\"+name+\"_label_report.html\")\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b_ACQUlCBlTc"
      },
      "source": [
        "## Kurtosis and skewness analysis"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zn17MlsqdRtW"
      },
      "source": [
        "Calculate Kurtosis of Every Annotated Label"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RRveay39eaN6"
      },
      "source": [
        "label_kurtosis = scipy.stats.kurtosis(labels, axis=1, fisher=False, bias=True)\n",
        "label_kurtosis = np.c_[y,label_kurtosis]\n",
        "label_kurtosis = pd.DataFrame(label_kurtosis)\n",
        "kurtosis_sort = label_kurtosis.sort_values(labels.shape[1]-1,axis=0)\n",
        "\n",
        "# kurtosis_sort.columns=['Happy', 'Sad', 'Surprise', 'Anger', 'Disgust', 'Fear','Kurtosis']\n",
        "kurtosis_sort.describe()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VV8D0tmsljyC"
      },
      "source": [
        "kurtosis_sort[-30:]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ePUpAGEvgmNe"
      },
      "source": [
        "label_skewness = scipy.stats.skew(labels, axis=1, bias=True)\n",
        "label_skewness = np.c_[y,label_skewness]\n",
        "label_skewness = pd.DataFrame(label_skewness)\n",
        "skewness_sort = label_skewness.sort_values(labels.shape[1]-1,axis=0)\n",
        "\n",
        "# skewness_sort.columns=['Happy', 'Sad', 'Surprise', 'Anger', 'Disgust', 'Fear','Skewness']\n",
        "skewness_sort.describe()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "abmSz-SkI-ot"
      },
      "source": [
        "label_entropy = scipy.stats.entropy(labels, axis=1)\n",
        "label_entropy = np.c_[y,label_entropy]\n",
        "label_entropy = pd.DataFrame(label_entropy)\n",
        "entropy_sort = label_entropy.sort_values(labels.shape[1]-1,axis=0)\n",
        "\n",
        "# entropy_sort.columns=['Happy', 'Sad', 'Surprise', 'Anger', 'Disgust', 'Fear','Entropy']\n",
        "entropy_sort.describe()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yCmYn3riunMF"
      },
      "source": [
        "## Preprocess features"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qudf6M7lkfIM"
      },
      "source": [
        "Not all algorithms need to pre-process the features in advance. If confirmed, you can normalize or reduce the dimensions of the features."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HHscEDFNA5nx"
      },
      "source": [
        "scalerstd = preprocessing.StandardScaler().fit(X)\n",
        "X = scalerstd.transform(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0z4kPDWmIEM1"
      },
      "source": [
        "scalermm = preprocessing.MinMaxScaler().fit(X)\n",
        "X = scalermm.transform(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OC76Siock5lJ"
      },
      "source": [
        "Check feature statistics after preprocessing."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gjmsTS7tk60I"
      },
      "source": [
        "features_processed = pd.DataFrame(X)\n",
        "features_processed.describe()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-61wpUQbFTom"
      },
      "source": [
        "# Define metrics"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kfkwW0Z8FqJY"
      },
      "source": [
        "## Multi-label learning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RyHCaHNYrzCK"
      },
      "source": [
        "Define multi-label problem metrics\n",
        "\n",
        "\n",
        "> Since some instances of missing labels may appear in the test set during random division, it is more appropriate to use the micro method to evaluate the classification model.\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "du9YUj6bF0Gk"
      },
      "source": [
        "from sklearn.metrics import *\n",
        "\n",
        "def clf_eval(y_test, y_pred, y_scores):\n",
        "  acc = accuracy_score(y_test,y_pred)\n",
        "  print(\"acc:\",acc)\n",
        "\n",
        "  f1_micro = f1_score(y_test, y_pred,average=\"micro\") \n",
        "  print(\"f1 score(micro):\",f1_micro)\n",
        "\n",
        "  precision_micro = precision_score(y_test, y_pred,average=\"micro\") \n",
        "  print(\"precision score(micro):\",precision_micro)\n",
        "\n",
        "  recall_micro = recall_score(y_test, y_pred,average=\"micro\") \n",
        "  print(\"recall score(micro):\",recall_micro)\n",
        "\n",
        "  AUC_micro = roc_auc_score(y_test, y_pred, average='micro')\n",
        "  print(\"AUC score(micro):\",AUC_micro)\n",
        "\n",
        "  loss_hamming = hamming_loss(y_test, y_pred)  \n",
        "  print(\"hamming loss:\",loss_hamming)\n",
        "\n",
        "  loss_ranking = label_ranking_loss(y_test, y_scores)\n",
        "  print(\"label ranking loss:\",loss_ranking)\n",
        "  \n",
        "  confusion_matrix = multilabel_confusion_matrix(y_test, y_pred)\n",
        "  print(\"confusion matrix:\",confusion_matrix)\n",
        "  return acc, f1_micro, precision_micro, recall_micro, loss_hamming, loss_ranking, confusion_matrix\n",
        "  \n",
        "  #l = label_ranking_average_precision_score(y_test, y_pred)\n",
        "  #print(\"label ranking average precision score：\",l)\n",
        "\n",
        "  #c = coverage_error(y_test, y_scores)\n",
        "  #print(\"coverage error:\",c)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5jOswDf3-884"
      },
      "source": [
        "## Label distribution learning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wMV0bOzn_FZx"
      },
      "source": [
        "Define LDL metrics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_I8k0u4n_hY2"
      },
      "source": [
        "from sklearn.metrics import *\n",
        "import scipy\n",
        "from scipy.spatial import distance\n",
        "import numpy as np\n",
        "\n",
        "def ldl_eval(y_test, y_pred):\n",
        "  eps = np.finfo(np.float64).eps\n",
        "  for i in range(len(y_test)):\n",
        "    y_test[i] = y_test[i]/np.sum(y_test[i])\n",
        "    y_pred[i] = y_pred[i]/np.sum(y_pred[i])\n",
        "  \n",
        "  # distance\n",
        "  kl = np.mean(scipy.stats.entropy(y_test+eps,y_pred+eps,axis=1))\n",
        "  print(\"Kullback–Leibler divergence: \",kl)\n",
        "\n",
        "  euclidean_distance = np.mean(np.linalg.norm(y_test-y_pred,axis=1))\n",
        "  print(\"Euclidean distance: \",euclidean_distance)\n",
        "\n",
        "  mse = np.mean(np.power(y_test-y_pred,2))\n",
        "  print(\"MSE: \",mse)\n",
        "\n",
        "  chebyshev = np.mean(np.max(np.abs(y_test-y_pred),axis=1))\n",
        "  print(\"Chebyshev distance: \",chebyshev)\n",
        "\n",
        "  clark = np.mean(np.sqrt(np.sum(np.power(y_test-y_pred,2)/np.power(y_test+y_pred,2),axis=1)))\n",
        "  print(\"Clark distance: \",clark)\n",
        "\n",
        "  canberra = np.mean(np.sum(np.abs(y_test-y_pred)/(y_test+y_pred),axis=1))\n",
        "  print(\"Canberra distance: \",canberra)\n",
        "\n",
        "  # similarity\n",
        "  cosine = np.mean( [1.0-scipy.spatial.distance.cosine(y_test[i],y_pred[i]) for i in range(len(y_test))])\n",
        "  print(\"Cosine similarity: \",cosine)\n",
        "\n",
        "  intersection = np.mean(np.sum(np.min([y_test,y_pred],axis=0),axis=1))\n",
        "  print(\"Intersection similarity: \",intersection)\n",
        "\n",
        "  # ambiguity \n",
        "  kurtosis_pred = scipy.stats.kurtosis(y_pred, axis=1, fisher=False, bias=True)\n",
        "  kurtosis_test = scipy.stats.kurtosis(y_test, axis=1, fisher=False, bias=True)\n",
        "  kurtosis_dif = kurtosis_pred - kurtosis_test\n",
        "  kurtosis_signed_offset = np.mean(kurtosis_dif)\n",
        "  kurtosis_abs_offset = np.mean(np.abs(kurtosis_dif))\n",
        "  laloss = np.mean([ -x if x < 0 else 0 for x in kurtosis_dif])\n",
        "  print(\"Kurtosis_signed_offset: \",kurtosis_signed_offset)\n",
        "  print(\"Kurtosis_abs_offset: \",kurtosis_abs_offset)\n",
        "  print(\"Laloss: \",laloss)\n",
        "\n",
        "\n",
        "\n",
        "  return kl, euclidean_distance, mse, chebyshev,\n",
        "  clark, canberra, cosine, intersection,\n",
        "  kurtosis_signed_offset, kurtosis_abs_offset, laloss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FKj8sQVLnUml"
      },
      "source": [
        "# Sklearn"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "buwCtht5GUb9"
      },
      "source": [
        "## Classification model (special for multi-label problems)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JYmnytjB3Aa8"
      },
      "source": [
        "Define multi-label specific prediction function to get the prediction labels and scores."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aRt4Lo5BjCSx"
      },
      "source": [
        "def clf_pre(clf, X_test):\n",
        "  y_pred = clf.predict(X_test)\n",
        "  y_proba = clf.predict_proba(X_test)\n",
        "  y_scores = np.zeros(y_pred.shape)\n",
        "  y_scores_t = y_scores.T\n",
        "  for i in range(len(y_scores_t)):\n",
        "    for j in range(len(y_scores_t[0])):\n",
        "      y_scores_t[i][j] = 1 - y_proba[i][j,0]\n",
        "  y_scores = y_scores_t.T\n",
        "  return y_pred, y_scores\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YYemVM4PyI2p"
      },
      "source": [
        "### KNN classifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F6P_KH8pyWbg"
      },
      "source": [
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "clf_KNN = KNeighborsClassifier()\n",
        "clf_KNN.fit(X_train, y_ml_train)\n",
        "y_pred, y_scores = clf_pre(clf_KNN,X_test)\n",
        "print('KNN Eval')\n",
        "eval_KNN = clf_eval(y_ml_test,y_pred,y_scores)\n",
        "eval_KNN2 = ldl_eval(y_test,y_scores)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vb8lUToAx_jE"
      },
      "source": [
        "### Random forest classifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aZEQibGqyAKs"
      },
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "clf_RandomForest = RandomForestClassifier(n_estimators=100)\n",
        "clf_RandomForest.fit(X_train, y_ml_train)\n",
        "y_pred, y_scores = clf_pre(clf_RandomForest,X_test)\n",
        "print('RandomForest Eval')\n",
        "eval_RF = clf_eval(y_ml_test,y_pred,y_scores)\n",
        "eval_RF2 = ldl_eval(y_test,y_scores)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ycmZnbGBINF8"
      },
      "source": [
        "On training set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ShqEhnpIPmt"
      },
      "source": [
        "y_pred, y_scores = clf_pre(clf_RandomForest,X_train)\n",
        "print('RandomForest Eval')\n",
        "print('On training Set')\n",
        "eval_RF = clf_eval(y_ml_train,y_pred,y_scores)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Sy6qaOoUGjcH"
      },
      "source": [
        "### Feature engineering attempt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "904qIE5jMahG"
      },
      "source": [
        "Get the top ten features with the largest information gain."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AKE9ebBK1Wzj"
      },
      "source": [
        "importances = clf_RandomForest.feature_importances_\n",
        "indices = np.argsort(importances)[::-1]\n",
        "print(\"Importance of the top 10 features\")\n",
        "for f in range(min(20,X_train.shape[1])):\n",
        "    print (\"%2d) %-*s %f\" % (f+1,30,indices[f],importances[indices[f]]) )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-qpZqLK98Fq1"
      },
      "source": [
        "Try: combine the top 10 features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tgVf61tJ75nd"
      },
      "source": [
        "X_train_processed = X_train.copy()\n",
        "X_test_processed = X_test.copy()\n",
        "for i in range(min(10,X_train.shape[1])):\n",
        "  for j in range(min(10,X_train.shape[1])):\n",
        "    if i>=j:\n",
        "      features_combined_train = X_train[:,indices[i]] * X_train[:,indices[j]]\n",
        "      X_train_processed = np.column_stack((X_train,features_combined_train.T))\n",
        "      features_combined_test = X_test[:,indices[i]] * X_test[:,indices[j]]\n",
        "      X_test_processed = np.column_stack((X_test,features_combined_test.T))\n",
        "\n",
        "print(X_train_processed.shape)\n",
        "print(X_test_processed.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G5LDvkggHajm"
      },
      "source": [
        "Evaluated by RF classifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1EKIZ_A-r6jZ"
      },
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "clf_RandomForest = RandomForestClassifier(n_estimators=100)\n",
        "clf_RandomForest.fit(X_train_processed, y_train)\n",
        "y_pred, y_scores = clf_pre(clf_RandomForest,X_test_processed)\n",
        "print('RandomForest Eval')\n",
        "eval_RF = clf_eval(y_test,y_pred,y_scores)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zLRw2iNKGTJN"
      },
      "source": [
        "## Regression model (special for label distribution problems)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "guDY76yl_mWA"
      },
      "source": [
        "### SVR"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pOfvNdGuAAuT",
        "outputId": "a213eba6-bcfc-4165-c131-3f928b9adb3c"
      },
      "source": [
        "from sklearn import svm\n",
        "from sklearn.multioutput import MultiOutputRegressor\n",
        "regressor_svr = MultiOutputRegressor(svm.SVR())\n",
        "regressor_svr.fit(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MultiOutputRegressor(estimator=SVR(C=1.0, cache_size=200, coef0=0.0, degree=3,\n",
              "                                   epsilon=0.1, gamma='scale', kernel='rbf',\n",
              "                                   max_iter=-1, shrinking=True, tol=0.001,\n",
              "                                   verbose=False),\n",
              "                     n_jobs=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gREt5xQmAV-x",
        "outputId": "3201c696-a65d-424e-c726-1532c3796cfb"
      },
      "source": [
        "y_pred = regressor_svr.predict(X_test)\n",
        "for i in range(len(y_pred)):\n",
        "  y_pred[i] = y_pred[i]/np.sum(y_pred[i])\n",
        "\n",
        "print(\"Testing set\")\n",
        "\n",
        "sl_eval = clf_eval(get_single_label(y_test),get_single_label(y_pred),y_pred)\n",
        "ml_eval = clf_eval(get_multi_label(y_test),get_multi_label(y_pred),y_pred)\n",
        "loss_test_mean = ldl_eval(y_test,y_pred)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Testing set\n",
            "acc: 0.37333333333333335\n",
            "f1 score(micro): 0.37319316688567666\n",
            "precision score(micro): 0.37866666666666665\n",
            "recall score(micro): 0.36787564766839376\n",
            "AUC score(micro): 0.6214378238341969\n",
            "hamming loss: 0.212\n",
            "label ranking loss: 0.3287888888888889\n",
            "confusion matrix: [[[501  76]\n",
            "  [112  61]]\n",
            "\n",
            " [[650   0]\n",
            "  [100   0]]\n",
            "\n",
            " [[327 193]\n",
            "  [106 124]]\n",
            "\n",
            " [[662   0]\n",
            "  [ 88   0]]\n",
            "\n",
            " [[713   0]\n",
            "  [ 37   0]]\n",
            "\n",
            " [[409 197]\n",
            "  [ 45  99]]]\n",
            "acc: 0.04533333333333334\n",
            "f1 score(micro): 0.425359380759307\n",
            "precision score(micro): 0.38466666666666666\n",
            "recall score(micro): 0.4756801319043693\n",
            "AUC score(micro): 0.5974384839625284\n",
            "hamming loss: 0.34644444444444444\n",
            "label ranking loss: 0.3589259259259259\n",
            "confusion matrix: [[[221 317]\n",
            "  [ 44 168]]\n",
            "\n",
            " [[507  33]\n",
            "  [182  28]]\n",
            "\n",
            " [[145 285]\n",
            "  [ 59 261]]\n",
            "\n",
            " [[534   0]\n",
            "  [216   0]]\n",
            "\n",
            " [[606  58]\n",
            "  [ 84   2]]\n",
            "\n",
            " [[351 230]\n",
            "  [ 51 118]]]\n",
            "Kullback–Leibler divergence:  0.0763776037678445\n",
            "Euclidean distance:  0.15873612697394243\n",
            "MSE:  0.005132965883368118\n",
            "Chebyshev distance:  0.1308476705422391\n",
            "Clark distance:  0.3948085105858768\n",
            "Canberra distance:  0.8629585023464994\n",
            "Cosine similarity:  0.925627471871945\n",
            "Intersection similarity:  0.8459884504808337\n",
            "Kurtosis_signed_offset:  -0.19410576477080518\n",
            "Kurtosis_abs_offset:  1.0550180257408395\n",
            "Laloss:  0.6245618952558223\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R70hQwPwirEM"
      },
      "source": [
        "### Random Forest Regressor"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8WtolTHMckXV",
        "outputId": "743bd28c-b5bb-4f5d-9ee6-b9b64f45d530"
      },
      "source": [
        "from sklearn.ensemble import RandomForestRegressor\n",
        "\n",
        "regressor = RandomForestRegressor(n_estimators=100)\n",
        "regressor.fit(X_train, y_train)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomForestRegressor(bootstrap=True, ccp_alpha=0.0, criterion='mse',\n",
              "                      max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
              "                      max_samples=None, min_impurity_decrease=0.0,\n",
              "                      min_impurity_split=None, min_samples_leaf=1,\n",
              "                      min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
              "                      n_estimators=100, n_jobs=None, oob_score=False,\n",
              "                      random_state=None, verbose=0, warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "niNDKeJQHLvD"
      },
      "source": [
        "Evaluate the kl loss on the training set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oRQj37A4j__5",
        "outputId": "e25651f5-d050-42a3-8341-67cecf504886"
      },
      "source": [
        "y_pred_train = regressor.predict(X_train)\n",
        "\n",
        "for i in range(len(y_pred_train)):\n",
        "  y_pred_train[i] = y_pred_train[i]/sum(y_pred_train[i])\n",
        "  \n",
        "print(\"Training set\")\n",
        "loss_train_mean = ldl_eval(y_train,y_pred_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training set\n",
            "Kullback–Leibler divergence:  0.14403391478037555\n",
            "Euclidean distance:  0.14535064036678005\n",
            "MSE:  0.0031256182596449654\n",
            "Chebyshev distance:  0.11341969017529563\n",
            "Clark distance:  nan\n",
            "Canberra distance:  nan\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:25: RuntimeWarning: invalid value encountered in true_divide\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:28: RuntimeWarning: invalid value encountered in true_divide\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Cosine similarity:  0.9780090306270477\n",
            "Intersection similarity:  0.8488760701182225\n",
            "Kurtosis_signed_offset:  -0.015575000196273248\n",
            "Kurtosis_abs_offset:  0.6084820865960565\n",
            "Laloss:  0.312028543396165\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rmtYVnewHYiv"
      },
      "source": [
        "Evaluate the kl loss on the testing set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QUohepez7hod",
        "outputId": "0e39625b-1e4a-4150-d27c-69ae2e2e1ae9"
      },
      "source": [
        "y_pred = regressor.predict(X_test)\n",
        "for i in range(len(y_pred)):\n",
        "  y_pred[i] = y_pred[i]/np.sum(y_pred[i])\n",
        "\n",
        "print(\"Testing set\")\n",
        "\n",
        "sl_eval = clf_eval(get_single_label(y_test),get_single_label(y_pred),y_pred)\n",
        "ml_eval = clf_eval(get_multi_label(y_test),get_multi_label(y_pred),y_pred)\n",
        "loss_test_mean = ldl_eval(y_test,y_pred)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Testing set\n",
            "acc: 0.48933333333333334\n",
            "f1 score(micro): 0.49408672798948744\n",
            "precision score(micro): 0.5013333333333333\n",
            "recall score(micro): 0.48704663212435234\n",
            "AUC score(micro): 0.6933623718561676\n",
            "hamming loss: 0.1711111111111111\n",
            "label ranking loss: 0.23394814814814816\n",
            "confusion matrix: [[[472 105]\n",
            "  [ 64 109]]\n",
            "\n",
            " [[640  10]\n",
            "  [ 90  10]]\n",
            "\n",
            " [[329 191]\n",
            "  [ 68 162]]\n",
            "\n",
            " [[659   3]\n",
            "  [ 85   3]]\n",
            "\n",
            " [[710   3]\n",
            "  [ 37   0]]\n",
            "\n",
            " [[544  62]\n",
            "  [ 52  92]]]\n",
            "acc: 0.156\n",
            "f1 score(micro): 0.5132275132275133\n",
            "precision score(micro): 0.473831123517097\n",
            "recall score(micro): 0.5597691673536686\n",
            "AUC score(micro): 0.665190333600777\n",
            "hamming loss: 0.2862222222222222\n",
            "label ranking loss: 0.24960370370370372\n",
            "confusion matrix: [[[342 196]\n",
            "  [ 56 156]]\n",
            "\n",
            " [[483  57]\n",
            "  [138  72]]\n",
            "\n",
            " [[161 269]\n",
            "  [ 36 284]]\n",
            "\n",
            " [[442  92]\n",
            "  [161  55]]\n",
            "\n",
            " [[635  29]\n",
            "  [ 83   3]]\n",
            "\n",
            " [[470 111]\n",
            "  [ 60 109]]]\n",
            "Kullback–Leibler divergence:  0.055973061029926004\n",
            "Euclidean distance:  0.13400012910974346\n",
            "MSE:  0.0037962159548448468\n",
            "Chebyshev distance:  0.10734642399549746\n",
            "Clark distance:  0.336009419557134\n",
            "Canberra distance:  0.7157363323423451\n",
            "Cosine similarity:  0.9445842628853243\n",
            "Intersection similarity:  0.8715007472424151\n",
            "Kurtosis_signed_offset:  -0.3091121267644367\n",
            "Kurtosis_abs_offset:  0.9557403809129121\n",
            "Laloss:  0.6324262538386745\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oRkEj52SRCsX",
        "outputId": "47f29b13-7d11-46ae-fab4-205dd2cbb043"
      },
      "source": [
        "y_pred = regressor2.predict(X_test)\n",
        "for i in range(len(y_pred)):\n",
        "  y_pred[i] = y_pred[i]/np.sum(y_pred[i])\n",
        "\n",
        "print(\"Testing set\")\n",
        "\n",
        "sl_eval = clf_eval(get_single_label(y_test),get_single_label(y_pred),y_pred)\n",
        "ml_eval = clf_eval(get_multi_label(y_test),get_multi_label(y_pred),y_pred)\n",
        "loss_test_mean = ldl_eval(y_test,y_pred)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Testing set\n",
            "acc: 0.48533333333333334\n",
            "f1 score(micro): 0.4907407407407407\n",
            "precision score(micro): 0.49466666666666664\n",
            "recall score(micro): 0.4868766404199475\n",
            "AUC score(micro): 0.6927427610874483\n",
            "hamming loss: 0.1711111111111111\n",
            "label ranking loss: 0.23013333333333333\n",
            "confusion matrix: [[[512  69]\n",
            "  [ 73  96]]\n",
            "\n",
            " [[622  22]\n",
            "  [ 91  15]]\n",
            "\n",
            " [[298 241]\n",
            "  [ 40 171]]\n",
            "\n",
            " [[652  18]\n",
            "  [ 70  10]]\n",
            "\n",
            " [[702   3]\n",
            "  [ 44   1]]\n",
            "\n",
            " [[573  26]\n",
            "  [ 73  78]]]\n",
            "acc: 0.204\n",
            "f1 score(micro): 0.5363601385148133\n",
            "precision score(micro): 0.49750178443968596\n",
            "recall score(micro): 0.5818030050083473\n",
            "AUC score(micro): 0.6842994431462088\n",
            "hamming loss: 0.2677777777777778\n",
            "label ranking loss: 0.23342962962962963\n",
            "confusion matrix: [[[433 117]\n",
            "  [ 73 127]]\n",
            "\n",
            " [[408 139]\n",
            "  [ 97 106]]\n",
            "\n",
            " [[165 272]\n",
            "  [ 24 289]]\n",
            "\n",
            " [[446  99]\n",
            "  [141  64]]\n",
            "\n",
            " [[633  18]\n",
            "  [ 87  12]]\n",
            "\n",
            " [[513  59]\n",
            "  [ 79  99]]]\n",
            "Kullback–Leibler divergence:  0.05999203885618337\n",
            "Euclidean distance:  0.1386967649513472\n",
            "MSE:  0.004099387457888263\n",
            "Chebyshev distance:  0.1093024047480481\n",
            "Clark distance:  0.3466428013692079\n",
            "Canberra distance:  0.7217363439786784\n",
            "Cosine similarity:  0.9405430925090364\n",
            "Intersection similarity:  0.8687410098563141\n",
            "Kurtosis_signed_offset:  -0.34412789009816536\n",
            "Kurtosis_abs_offset:  1.0364143425694794\n",
            "Laloss:  0.6902711163338223\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rZ9I5t7D9Qu7"
      },
      "source": [
        "Calculate Kurtosis of Annotated Labels on Test Set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IcMVRXep9Z7K",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284
        },
        "outputId": "43baa7e6-e60c-4807-9a52-f196d64c71f5"
      },
      "source": [
        "label_kurtosis = scipy.stats.kurtosis(y_test, axis=1, fisher=False, bias=True)\n",
        "label_kurtosis = np.c_[y_test,label_kurtosis]\n",
        "label_kurtosis = pd.DataFrame(label_kurtosis)\n",
        "kurtosis_sort = label_kurtosis.sort_values(labels.shape[1]-1,axis=0)\n",
        "\n",
        "# kurtosis_sort.columns=['Happy', 'Sad', 'Surprise', 'Anger', 'Disgust', 'Fear','Kurtosis']\n",
        "kurtosis_sort.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>64.000000</td>\n",
              "      <td>64.000000</td>\n",
              "      <td>64.000000</td>\n",
              "      <td>64.000000</td>\n",
              "      <td>64.000000</td>\n",
              "      <td>64.000000</td>\n",
              "      <td>64.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.164181</td>\n",
              "      <td>0.173232</td>\n",
              "      <td>0.163764</td>\n",
              "      <td>0.164016</td>\n",
              "      <td>0.181437</td>\n",
              "      <td>0.153370</td>\n",
              "      <td>2.767488</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.112310</td>\n",
              "      <td>0.052366</td>\n",
              "      <td>0.075027</td>\n",
              "      <td>0.056674</td>\n",
              "      <td>0.057771</td>\n",
              "      <td>0.028992</td>\n",
              "      <td>0.882297</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.067433</td>\n",
              "      <td>0.099451</td>\n",
              "      <td>0.086907</td>\n",
              "      <td>0.096532</td>\n",
              "      <td>0.100281</td>\n",
              "      <td>0.099458</td>\n",
              "      <td>1.357919</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.086074</td>\n",
              "      <td>0.133765</td>\n",
              "      <td>0.114018</td>\n",
              "      <td>0.122685</td>\n",
              "      <td>0.127356</td>\n",
              "      <td>0.131960</td>\n",
              "      <td>2.050240</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.113784</td>\n",
              "      <td>0.162520</td>\n",
              "      <td>0.130567</td>\n",
              "      <td>0.141802</td>\n",
              "      <td>0.173124</td>\n",
              "      <td>0.152279</td>\n",
              "      <td>2.662803</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.184334</td>\n",
              "      <td>0.192755</td>\n",
              "      <td>0.179354</td>\n",
              "      <td>0.197822</td>\n",
              "      <td>0.235069</td>\n",
              "      <td>0.171473</td>\n",
              "      <td>3.425403</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>0.456420</td>\n",
              "      <td>0.330827</td>\n",
              "      <td>0.358845</td>\n",
              "      <td>0.317175</td>\n",
              "      <td>0.306174</td>\n",
              "      <td>0.233681</td>\n",
              "      <td>4.188581</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "               0          1          2  ...          4          5          6\n",
              "count  64.000000  64.000000  64.000000  ...  64.000000  64.000000  64.000000\n",
              "mean    0.164181   0.173232   0.163764  ...   0.181437   0.153370   2.767488\n",
              "std     0.112310   0.052366   0.075027  ...   0.057771   0.028992   0.882297\n",
              "min     0.067433   0.099451   0.086907  ...   0.100281   0.099458   1.357919\n",
              "25%     0.086074   0.133765   0.114018  ...   0.127356   0.131960   2.050240\n",
              "50%     0.113784   0.162520   0.130567  ...   0.173124   0.152279   2.662803\n",
              "75%     0.184334   0.192755   0.179354  ...   0.235069   0.171473   3.425403\n",
              "max     0.456420   0.330827   0.358845  ...   0.306174   0.233681   4.188581\n",
              "\n",
              "[8 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mAqdBkNG8Hc2"
      },
      "source": [
        "Calculate Kurtosis of Predicted Labels on Test Set （need y_pred）"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7e2489oL7iD5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284
        },
        "outputId": "de0823f8-5a62-45eb-fea7-d96fac1b3e32"
      },
      "source": [
        "label_kurtosis = scipy.stats.kurtosis(y_pred, axis=1, fisher=False, bias=True)\n",
        "label_kurtosis = np.c_[y_pred,label_kurtosis]\n",
        "label_kurtosis = pd.DataFrame(label_kurtosis)\n",
        "kurtosis_sort = label_kurtosis.sort_values(labels.shape[1]-1,axis=0)\n",
        "\n",
        "# kurtosis_sort.columns=['Happy', 'Sad', 'Surprise', 'Anger', 'Disgust', 'Fear','Kurtosis']\n",
        "kurtosis_sort.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>64.000000</td>\n",
              "      <td>64.000000</td>\n",
              "      <td>64.000000</td>\n",
              "      <td>64.000000</td>\n",
              "      <td>64.000000</td>\n",
              "      <td>64.000000</td>\n",
              "      <td>64.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.155469</td>\n",
              "      <td>0.175041</td>\n",
              "      <td>0.166498</td>\n",
              "      <td>0.164355</td>\n",
              "      <td>0.182326</td>\n",
              "      <td>0.156312</td>\n",
              "      <td>2.199864</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.035653</td>\n",
              "      <td>0.011751</td>\n",
              "      <td>0.027917</td>\n",
              "      <td>0.016113</td>\n",
              "      <td>0.019079</td>\n",
              "      <td>0.007720</td>\n",
              "      <td>0.823385</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.107750</td>\n",
              "      <td>0.143283</td>\n",
              "      <td>0.124946</td>\n",
              "      <td>0.123986</td>\n",
              "      <td>0.129556</td>\n",
              "      <td>0.129489</td>\n",
              "      <td>1.260523</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.135349</td>\n",
              "      <td>0.166956</td>\n",
              "      <td>0.146199</td>\n",
              "      <td>0.152428</td>\n",
              "      <td>0.171541</td>\n",
              "      <td>0.152960</td>\n",
              "      <td>1.618657</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.145497</td>\n",
              "      <td>0.177620</td>\n",
              "      <td>0.158399</td>\n",
              "      <td>0.163267</td>\n",
              "      <td>0.182870</td>\n",
              "      <td>0.156812</td>\n",
              "      <td>1.906927</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.162892</td>\n",
              "      <td>0.182972</td>\n",
              "      <td>0.182037</td>\n",
              "      <td>0.176285</td>\n",
              "      <td>0.197734</td>\n",
              "      <td>0.161615</td>\n",
              "      <td>2.655288</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>0.303983</td>\n",
              "      <td>0.192161</td>\n",
              "      <td>0.257398</td>\n",
              "      <td>0.211255</td>\n",
              "      <td>0.221861</td>\n",
              "      <td>0.170987</td>\n",
              "      <td>3.994224</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "               0          1          2  ...          4          5          6\n",
              "count  64.000000  64.000000  64.000000  ...  64.000000  64.000000  64.000000\n",
              "mean    0.155469   0.175041   0.166498  ...   0.182326   0.156312   2.199864\n",
              "std     0.035653   0.011751   0.027917  ...   0.019079   0.007720   0.823385\n",
              "min     0.107750   0.143283   0.124946  ...   0.129556   0.129489   1.260523\n",
              "25%     0.135349   0.166956   0.146199  ...   0.171541   0.152960   1.618657\n",
              "50%     0.145497   0.177620   0.158399  ...   0.182870   0.156812   1.906927\n",
              "75%     0.162892   0.182972   0.182037  ...   0.197734   0.161615   2.655288\n",
              "max     0.303983   0.192161   0.257398  ...   0.221861   0.170987   3.994224\n",
              "\n",
              "[8 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AysvNM-F3lb9"
      },
      "source": [
        "# keras"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jxd2-6ZifBne"
      },
      "source": [
        "# Using TensorFlow as keras backend\n",
        "import tensorflow as tf\n",
        "\n",
        "from tensorflow import keras\n",
        "import numpy as np\n",
        "from keras.datasets import mnist\n",
        "from keras.models import Sequential,Model\n",
        "from keras.layers import Dense,Dropout,Input\n",
        "from keras.utils import get_file,to_categorical\n",
        "from keras import regularizers\n",
        "\n",
        "from keras import backend as K"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9i6k_ZgtCSLZ"
      },
      "source": [
        "## Define Loss functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YBAdVKG0Caw5"
      },
      "source": [
        "Define LDL loss functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sjdfg9acCd86"
      },
      "source": [
        "def kurt_cal(y):\n",
        "  mean = tf.reduce_mean(y,axis=-1,keepdims=True)\n",
        "  std = tf.keras.backend.std(y,axis=-1,keepdims=True)\n",
        "  kurt = tf.reduce_mean(tf.pow((y-mean)/std,4),axis=-1)\n",
        "  return kurt\n",
        "\n",
        "def new_kurt_cal(y):\n",
        "  mean = tf.reduce_mean(y,axis=-1,keepdims=True)\n",
        "  std = tf.keras.backend.std(y,axis=-1,keepdims=True)\n",
        "  dif = y-mean\n",
        "  dif_above_zero = tf.reduce_max([dif,tf.zeros_like(dif)],axis=0)\n",
        "  new_kurt = tf.reduce_mean(tf.pow((dif_above_zero)/std,4),axis=-1)\n",
        "  return new_kurt\n",
        "  \n",
        "def label_ambiguity_loss(y_true, y_pred):\n",
        "  kurt_dif = kurt_cal(y_true)-kurt_cal(y_pred)\n",
        "  # la_loss = tf.reduce_max([kurt_dif,tf.zeros_like(kurt_dif)])\n",
        "  laloss_abs = tf.abs(kurt_cal(y_true)-kurt_cal(y_pred))\n",
        "  # mse = tf.reduce_mean(tf.pow((y_true-y_pred),2))\n",
        "  kl = tf.keras.losses.KLDivergence()\n",
        "  kl_loss = kl(y_true, y_pred)\n",
        "  loss = kl_loss * laloss_abs\n",
        "  return loss\n",
        "\n",
        "def new_label_ambiguity_loss(y_true, y_pred):\n",
        "  kurt_dif = new_kurt_cal(y_true)-new_kurt_cal(y_pred)\n",
        "  la_loss = tf.reduce_max([kurt_dif,tf.zeros_like(kurt_dif)])\n",
        "  # laloss_abs = tf.abs(kurt_cal(y_true)-kurt_cal(y_pred))\n",
        "  # mse = tf.reduce_mean(tf.pow((y_true-y_pred),2))\n",
        "  kl = tf.keras.losses.KLDivergence()\n",
        "  kl_loss = kl(y_true, y_pred)\n",
        "  loss = kl_loss + 0.001 * la_loss\n",
        "  return loss\n",
        "\n",
        "def weighted_kl(y_true, y_pred):\n",
        "  kl = tf.keras.losses.KLDivergence()\n",
        "  weight = kurt_cal(y_true)\n",
        "  weighted_kl = kl(y_true, y_pred, sample_weight= tf.reduce_min([weight,tf.ones_like(weight)*3],axis=0))\n",
        "  return weighted_kl"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RenTcYfQrD1N",
        "outputId": "c14b3a65-4b8e-48ef-9e76-f6f8a9f37461"
      },
      "source": [
        "\"\"\"\n",
        "y = tf.constant([[0.1,0.3,0.4,0.2],[0.2,0.6,0.1,0.3]])\n",
        "mean = tf.reduce_mean(y,axis=-1,keepdims=True)\n",
        "std = tf.keras.backend.std(y,axis=-1,keepdims=True)\n",
        "kurt = tf.reduce_mean(tf.pow((y-mean)/std,4),axis=-1)\n",
        "kurt.numpy()\n",
        "kurtosis_scipy = scipy.stats.kurtosis(y.numpy(), axis=1, fisher=False, bias=True)\n",
        "kurtosis_scipy-kurt.numpy()\n",
        "\n",
        "y_true = y\n",
        "y_pred = y/std\n",
        "kurt_dif = kurt_cal(y_true)-kurt_cal(y_pred)\n",
        "laloss = tf.reduce_max([-kurt_dif,tf.zeros(kurt_dif.shape)],axis=0)\n",
        "kl = tf.keras.losses.KLDivergence()\n",
        "weight = kurt_cal(y_true)\n",
        "tf.ones_like(weight)*3\n",
        "weighted_kl = kl(y_true, y_pred, sample_weight= tf.reduce_min([weight,tf.ones_like(weight)*3],axis=0))\n",
        "mean = tf.reduce_mean(y,axis=-1,keepdims=True)\n",
        "std = tf.keras.backend.std(y,axis=-1,keepdims=True)\n",
        "dif = y-mean\n",
        "dif_above_zero = tf.reduce_max([dif,tf.zeros_like(dif)],axis=0)\n",
        "new_kurt = tf.reduce_mean(tf.pow((dif_above_zero)/std,4),axis=-1)\n",
        "new_kurt\n",
        "\"\"\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'\\ny = tf.constant([[0.1,0.3,0.4,0.2],[0.2,0.6,0.1,0.3]])\\nmean = tf.reduce_mean(y,axis=-1,keepdims=True)\\nstd = tf.keras.backend.std(y,axis=-1,keepdims=True)\\nkurt = tf.reduce_mean(tf.pow((y-mean)/std,4),axis=-1)\\nkurt.numpy()\\nkurtosis_scipy = scipy.stats.kurtosis(y.numpy(), axis=1, fisher=False, bias=True)\\nkurtosis_scipy-kurt.numpy()\\n\\ny_true = y\\ny_pred = y/std\\nkurt_dif = kurt_cal(y_true)-kurt_cal(y_pred)\\nlaloss = tf.reduce_max([-kurt_dif,tf.zeros(kurt_dif.shape)],axis=0)\\nkl = tf.keras.losses.KLDivergence()\\nweight = kurt_cal(y_true)\\ntf.ones_like(weight)*3\\nweighted_kl = kl(y_true, y_pred, sample_weight= tf.reduce_min([weight,tf.ones_like(weight)*3],axis=0))\\nmean = tf.reduce_mean(y,axis=-1,keepdims=True)\\nstd = tf.keras.backend.std(y,axis=-1,keepdims=True)\\ndif = y-mean\\ndif_above_zero = tf.reduce_max([dif,tf.zeros_like(dif)],axis=0)\\nnew_kurt = tf.reduce_mean(tf.pow((dif_above_zero)/std,4),axis=-1)\\nnew_kurt\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EaSMKNxgumaZ"
      },
      "source": [
        "## Define Metrics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HCXsoTIRurhj"
      },
      "source": [
        "def laloss(y_true, y_pred):\n",
        "  kurt_dif = kurt_cal(y_true)-kurt_cal(y_pred)\n",
        "  laloss = tf.reduce_max([kurt_dif,tf.zeros_like(kurt_dif)])\n",
        "  return laloss\n",
        "\n",
        "def new_laloss(y_true, y_pred):\n",
        "  kurt_dif = new_kurt_cal(y_true)-new_kurt_cal(y_pred)\n",
        "  laloss = tf.reduce_max([kurt_dif,tf.zeros_like(kurt_dif)])\n",
        "  return laloss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8M15ksnvFa3h"
      },
      "source": [
        "## LDL net"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QVf82TQ5FgKP"
      },
      "source": [
        "feature_dim = X.shape[1]\n",
        "label_dim = y.shape[1]\n",
        "\n",
        "def get_model():\n",
        "  model = Sequential()\n",
        "  model.add(Dense(64,activation='relu',input_shape=(feature_dim,)))\n",
        "  model.add(Dropout(0.4))\n",
        "  model.add(Dense(64,activation='relu'))\n",
        "  model.add(Dense(label_dim, activation='softmax'))\n",
        "  return model\n",
        "\n",
        "def get_model_baseline():\n",
        "  model = Sequential()\n",
        "  model.add(Dense(64,activation='relu',input_shape=(feature_dim,)))\n",
        "  model.add(Dropout(0.4))\n",
        "  model.add(Dense(64,activation='relu'))\n",
        "  model.add(Dense(label_dim, activation='softmax'))\n",
        "  model.compile(optimizer=keras.optimizers.RMSprop(lr=1e-3),loss= 'kullback_leibler_divergence', metrics=['kullback_leibler_divergence',laloss,new_laloss,weighted_kl])\n",
        "  return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HcWfq4bLDUs2"
      },
      "source": [
        "### Validate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3yWW9wlvJu2o",
        "outputId": "419dc0f3-74fd-49d1-f455-604b7ee2694d"
      },
      "source": [
        "val_num = np.int32(len(X_train)/10)\n",
        "X_val = X_train[:val_num]\n",
        "partial_X_train = X_train[val_num:]\n",
        "y_val = y_train[:val_num]\n",
        "partial_y_train = y_train[val_num:]\n",
        "val_num"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "200"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dIsyiz2IJs0E"
      },
      "source": [
        "model = get_model()\n",
        "model.compile(optimizer=keras.optimizers.RMSprop(),loss= label_ambiguity_loss, metrics=['acc','kullback_leibler_divergence',laloss])\n",
        "history = model.fit(partial_X_train,partial_y_train,batch_size=256,epochs=200,validation_data=(X_val,y_val),verbose=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JBaJyCVx8zhR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "92c778b1-eb2b-47dc-e4b0-4acdfcdc2bb6"
      },
      "source": [
        "model_baseline = get_model()\n",
        "model_baseline.compile(optimizer=keras.optimizers.RMSprop(),loss= 'kullback_leibler_divergence', metrics=['acc','kullback_leibler_divergence',laloss])\n",
        "history_baseline = model_baseline.fit(partial_X_train,partial_y_train,batch_size=256,epochs=200,validation_data=(X_val,y_val),verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/200\n",
            "8/8 [==============================] - 1s 31ms/step - loss: 0.0841 - acc: 0.2023 - kullback_leibler_divergence: 0.0841 - laloss: 2.8734 - val_loss: 0.0841 - val_acc: 0.2750 - val_kullback_leibler_divergence: 0.0841 - val_laloss: 2.2106\n",
            "Epoch 2/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0838 - acc: 0.2990 - kullback_leibler_divergence: 0.0838 - laloss: 2.8964 - val_loss: 0.0836 - val_acc: 0.2750 - val_kullback_leibler_divergence: 0.0836 - val_laloss: 2.5828\n",
            "Epoch 3/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0836 - acc: 0.2854 - kullback_leibler_divergence: 0.0836 - laloss: 2.9430 - val_loss: 0.0834 - val_acc: 0.2750 - val_kullback_leibler_divergence: 0.0834 - val_laloss: 2.3658\n",
            "Epoch 4/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0814 - acc: 0.2639 - kullback_leibler_divergence: 0.0814 - laloss: 2.8974 - val_loss: 0.0834 - val_acc: 0.2300 - val_kullback_leibler_divergence: 0.0834 - val_laloss: 2.7850\n",
            "Epoch 5/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0829 - acc: 0.2543 - kullback_leibler_divergence: 0.0829 - laloss: 2.9940 - val_loss: 0.0835 - val_acc: 0.2750 - val_kullback_leibler_divergence: 0.0835 - val_laloss: 2.9144\n",
            "Epoch 6/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0810 - acc: 0.2885 - kullback_leibler_divergence: 0.0810 - laloss: 3.0718 - val_loss: 0.0837 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0837 - val_laloss: 3.0201\n",
            "Epoch 7/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0818 - acc: 0.2936 - kullback_leibler_divergence: 0.0818 - laloss: 3.1412 - val_loss: 0.0836 - val_acc: 0.2750 - val_kullback_leibler_divergence: 0.0836 - val_laloss: 3.0167\n",
            "Epoch 8/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0834 - acc: 0.3030 - kullback_leibler_divergence: 0.0834 - laloss: 3.1522 - val_loss: 0.0837 - val_acc: 0.2100 - val_kullback_leibler_divergence: 0.0837 - val_laloss: 3.0533\n",
            "Epoch 9/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0820 - acc: 0.2586 - kullback_leibler_divergence: 0.0820 - laloss: 3.1183 - val_loss: 0.0835 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0835 - val_laloss: 3.1374\n",
            "Epoch 10/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0831 - acc: 0.2986 - kullback_leibler_divergence: 0.0831 - laloss: 3.1266 - val_loss: 0.0833 - val_acc: 0.2750 - val_kullback_leibler_divergence: 0.0833 - val_laloss: 2.8852\n",
            "Epoch 11/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0801 - acc: 0.3085 - kullback_leibler_divergence: 0.0801 - laloss: 3.0737 - val_loss: 0.0838 - val_acc: 0.1700 - val_kullback_leibler_divergence: 0.0838 - val_laloss: 3.1037\n",
            "Epoch 12/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0833 - acc: 0.2333 - kullback_leibler_divergence: 0.0833 - laloss: 3.1214 - val_loss: 0.0832 - val_acc: 0.3000 - val_kullback_leibler_divergence: 0.0832 - val_laloss: 3.0128\n",
            "Epoch 13/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0829 - acc: 0.2801 - kullback_leibler_divergence: 0.0829 - laloss: 3.1391 - val_loss: 0.0832 - val_acc: 0.2300 - val_kullback_leibler_divergence: 0.0832 - val_laloss: 2.9152\n",
            "Epoch 14/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0828 - acc: 0.2425 - kullback_leibler_divergence: 0.0828 - laloss: 3.0833 - val_loss: 0.0830 - val_acc: 0.2500 - val_kullback_leibler_divergence: 0.0830 - val_laloss: 2.8204\n",
            "Epoch 15/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0813 - acc: 0.2846 - kullback_leibler_divergence: 0.0813 - laloss: 2.9989 - val_loss: 0.0832 - val_acc: 0.2750 - val_kullback_leibler_divergence: 0.0832 - val_laloss: 2.8518\n",
            "Epoch 16/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0805 - acc: 0.3113 - kullback_leibler_divergence: 0.0805 - laloss: 3.0673 - val_loss: 0.0829 - val_acc: 0.2750 - val_kullback_leibler_divergence: 0.0829 - val_laloss: 2.8345\n",
            "Epoch 17/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0820 - acc: 0.2928 - kullback_leibler_divergence: 0.0820 - laloss: 3.0183 - val_loss: 0.0830 - val_acc: 0.2550 - val_kullback_leibler_divergence: 0.0830 - val_laloss: 2.9766\n",
            "Epoch 18/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0813 - acc: 0.2818 - kullback_leibler_divergence: 0.0813 - laloss: 3.0814 - val_loss: 0.0830 - val_acc: 0.2050 - val_kullback_leibler_divergence: 0.0830 - val_laloss: 3.1198\n",
            "Epoch 19/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0825 - acc: 0.2763 - kullback_leibler_divergence: 0.0825 - laloss: 3.1548 - val_loss: 0.0828 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0828 - val_laloss: 3.0170\n",
            "Epoch 20/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0816 - acc: 0.2860 - kullback_leibler_divergence: 0.0816 - laloss: 3.1016 - val_loss: 0.0826 - val_acc: 0.2500 - val_kullback_leibler_divergence: 0.0826 - val_laloss: 2.9062\n",
            "Epoch 21/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0817 - acc: 0.2924 - kullback_leibler_divergence: 0.0817 - laloss: 3.0168 - val_loss: 0.0825 - val_acc: 0.2850 - val_kullback_leibler_divergence: 0.0825 - val_laloss: 2.9857\n",
            "Epoch 22/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0808 - acc: 0.3169 - kullback_leibler_divergence: 0.0808 - laloss: 3.0876 - val_loss: 0.0825 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0825 - val_laloss: 2.9925\n",
            "Epoch 23/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0801 - acc: 0.3254 - kullback_leibler_divergence: 0.0801 - laloss: 3.0822 - val_loss: 0.0825 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0825 - val_laloss: 3.1169\n",
            "Epoch 24/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0800 - acc: 0.3062 - kullback_leibler_divergence: 0.0800 - laloss: 3.1378 - val_loss: 0.0825 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0825 - val_laloss: 3.0419\n",
            "Epoch 25/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0810 - acc: 0.3251 - kullback_leibler_divergence: 0.0810 - laloss: 3.1213 - val_loss: 0.0823 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0823 - val_laloss: 2.9595\n",
            "Epoch 26/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0811 - acc: 0.2976 - kullback_leibler_divergence: 0.0811 - laloss: 3.0649 - val_loss: 0.0818 - val_acc: 0.2850 - val_kullback_leibler_divergence: 0.0818 - val_laloss: 2.8226\n",
            "Epoch 27/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0813 - acc: 0.3160 - kullback_leibler_divergence: 0.0813 - laloss: 2.9624 - val_loss: 0.0819 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0819 - val_laloss: 3.0454\n",
            "Epoch 28/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0828 - acc: 0.3320 - kullback_leibler_divergence: 0.0828 - laloss: 3.1205 - val_loss: 0.0819 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0819 - val_laloss: 2.7295\n",
            "Epoch 29/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0817 - acc: 0.3286 - kullback_leibler_divergence: 0.0817 - laloss: 3.0006 - val_loss: 0.0816 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0816 - val_laloss: 3.0086\n",
            "Epoch 30/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0817 - acc: 0.3212 - kullback_leibler_divergence: 0.0817 - laloss: 3.0831 - val_loss: 0.0817 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0817 - val_laloss: 2.8466\n",
            "Epoch 31/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0809 - acc: 0.3321 - kullback_leibler_divergence: 0.0809 - laloss: 2.9695 - val_loss: 0.0813 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0813 - val_laloss: 2.9189\n",
            "Epoch 32/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0794 - acc: 0.3289 - kullback_leibler_divergence: 0.0794 - laloss: 3.0053 - val_loss: 0.0816 - val_acc: 0.2600 - val_kullback_leibler_divergence: 0.0816 - val_laloss: 3.0923\n",
            "Epoch 33/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0797 - acc: 0.3321 - kullback_leibler_divergence: 0.0797 - laloss: 3.0731 - val_loss: 0.0813 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0813 - val_laloss: 3.0975\n",
            "Epoch 34/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0808 - acc: 0.3409 - kullback_leibler_divergence: 0.0808 - laloss: 3.1159 - val_loss: 0.0819 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0819 - val_laloss: 3.0131\n",
            "Epoch 35/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0805 - acc: 0.3467 - kullback_leibler_divergence: 0.0805 - laloss: 3.0571 - val_loss: 0.0812 - val_acc: 0.2700 - val_kullback_leibler_divergence: 0.0812 - val_laloss: 3.0391\n",
            "Epoch 36/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0796 - acc: 0.3225 - kullback_leibler_divergence: 0.0796 - laloss: 3.0615 - val_loss: 0.0816 - val_acc: 0.2550 - val_kullback_leibler_divergence: 0.0816 - val_laloss: 3.0783\n",
            "Epoch 37/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0796 - acc: 0.3223 - kullback_leibler_divergence: 0.0796 - laloss: 3.0757 - val_loss: 0.0806 - val_acc: 0.2950 - val_kullback_leibler_divergence: 0.0806 - val_laloss: 2.9592\n",
            "Epoch 38/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0787 - acc: 0.3313 - kullback_leibler_divergence: 0.0787 - laloss: 2.9676 - val_loss: 0.0808 - val_acc: 0.2550 - val_kullback_leibler_divergence: 0.0808 - val_laloss: 2.8180\n",
            "Epoch 39/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0809 - acc: 0.3160 - kullback_leibler_divergence: 0.0809 - laloss: 2.8985 - val_loss: 0.0805 - val_acc: 0.2850 - val_kullback_leibler_divergence: 0.0805 - val_laloss: 2.8079\n",
            "Epoch 40/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0798 - acc: 0.3510 - kullback_leibler_divergence: 0.0798 - laloss: 3.0197 - val_loss: 0.0816 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0816 - val_laloss: 2.9570\n",
            "Epoch 41/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0782 - acc: 0.3328 - kullback_leibler_divergence: 0.0782 - laloss: 3.0123 - val_loss: 0.0805 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0805 - val_laloss: 2.9411\n",
            "Epoch 42/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0771 - acc: 0.3527 - kullback_leibler_divergence: 0.0771 - laloss: 3.0126 - val_loss: 0.0801 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0801 - val_laloss: 3.1046\n",
            "Epoch 43/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0779 - acc: 0.3257 - kullback_leibler_divergence: 0.0779 - laloss: 3.0705 - val_loss: 0.0799 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.8830\n",
            "Epoch 44/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0777 - acc: 0.3510 - kullback_leibler_divergence: 0.0777 - laloss: 2.9410 - val_loss: 0.0819 - val_acc: 0.2300 - val_kullback_leibler_divergence: 0.0819 - val_laloss: 2.9672\n",
            "Epoch 45/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0778 - acc: 0.3249 - kullback_leibler_divergence: 0.0778 - laloss: 2.9371 - val_loss: 0.0803 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0803 - val_laloss: 2.9019\n",
            "Epoch 46/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0764 - acc: 0.3482 - kullback_leibler_divergence: 0.0764 - laloss: 2.9413 - val_loss: 0.0803 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0803 - val_laloss: 2.9568\n",
            "Epoch 47/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0781 - acc: 0.3506 - kullback_leibler_divergence: 0.0781 - laloss: 3.0660 - val_loss: 0.0794 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0794 - val_laloss: 3.0018\n",
            "Epoch 48/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0777 - acc: 0.3432 - kullback_leibler_divergence: 0.0777 - laloss: 3.0558 - val_loss: 0.0796 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0796 - val_laloss: 3.0521\n",
            "Epoch 49/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0766 - acc: 0.3451 - kullback_leibler_divergence: 0.0766 - laloss: 3.0675 - val_loss: 0.0794 - val_acc: 0.2750 - val_kullback_leibler_divergence: 0.0794 - val_laloss: 2.9180\n",
            "Epoch 50/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0783 - acc: 0.3517 - kullback_leibler_divergence: 0.0783 - laloss: 2.9706 - val_loss: 0.0790 - val_acc: 0.2850 - val_kullback_leibler_divergence: 0.0790 - val_laloss: 2.9216\n",
            "Epoch 51/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0772 - acc: 0.3522 - kullback_leibler_divergence: 0.0772 - laloss: 3.0034 - val_loss: 0.0801 - val_acc: 0.2500 - val_kullback_leibler_divergence: 0.0801 - val_laloss: 2.9128\n",
            "Epoch 52/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0786 - acc: 0.3230 - kullback_leibler_divergence: 0.0786 - laloss: 2.9754 - val_loss: 0.0826 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0826 - val_laloss: 2.8158\n",
            "Epoch 53/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0794 - acc: 0.3569 - kullback_leibler_divergence: 0.0794 - laloss: 2.9536 - val_loss: 0.0800 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.8706\n",
            "Epoch 54/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0758 - acc: 0.3636 - kullback_leibler_divergence: 0.0758 - laloss: 2.9317 - val_loss: 0.0820 - val_acc: 0.2450 - val_kullback_leibler_divergence: 0.0820 - val_laloss: 3.0061\n",
            "Epoch 55/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0775 - acc: 0.3203 - kullback_leibler_divergence: 0.0775 - laloss: 3.0280 - val_loss: 0.0810 - val_acc: 0.2700 - val_kullback_leibler_divergence: 0.0810 - val_laloss: 3.0678\n",
            "Epoch 56/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0757 - acc: 0.3218 - kullback_leibler_divergence: 0.0757 - laloss: 3.0574 - val_loss: 0.0807 - val_acc: 0.2950 - val_kullback_leibler_divergence: 0.0807 - val_laloss: 2.9926\n",
            "Epoch 57/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0766 - acc: 0.3201 - kullback_leibler_divergence: 0.0766 - laloss: 3.0427 - val_loss: 0.0797 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.9785\n",
            "Epoch 58/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0782 - acc: 0.3403 - kullback_leibler_divergence: 0.0782 - laloss: 2.9557 - val_loss: 0.0788 - val_acc: 0.2800 - val_kullback_leibler_divergence: 0.0788 - val_laloss: 2.9733\n",
            "Epoch 59/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0769 - acc: 0.3370 - kullback_leibler_divergence: 0.0769 - laloss: 3.0499 - val_loss: 0.0786 - val_acc: 0.2950 - val_kullback_leibler_divergence: 0.0786 - val_laloss: 2.9609\n",
            "Epoch 60/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0768 - acc: 0.3529 - kullback_leibler_divergence: 0.0768 - laloss: 3.0004 - val_loss: 0.0820 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0820 - val_laloss: 3.0463\n",
            "Epoch 61/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0776 - acc: 0.3622 - kullback_leibler_divergence: 0.0776 - laloss: 3.0055 - val_loss: 0.0803 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0803 - val_laloss: 3.0036\n",
            "Epoch 62/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0772 - acc: 0.3520 - kullback_leibler_divergence: 0.0772 - laloss: 3.0157 - val_loss: 0.0788 - val_acc: 0.2750 - val_kullback_leibler_divergence: 0.0788 - val_laloss: 3.0299\n",
            "Epoch 63/200\n",
            "8/8 [==============================] - 0s 27ms/step - loss: 0.0765 - acc: 0.3276 - kullback_leibler_divergence: 0.0765 - laloss: 3.0633 - val_loss: 0.0790 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0790 - val_laloss: 3.0543\n",
            "Epoch 64/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0785 - acc: 0.3516 - kullback_leibler_divergence: 0.0785 - laloss: 3.0131 - val_loss: 0.0787 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0787 - val_laloss: 2.7663\n",
            "Epoch 65/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0753 - acc: 0.3504 - kullback_leibler_divergence: 0.0753 - laloss: 3.0607 - val_loss: 0.0787 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0787 - val_laloss: 2.8764\n",
            "Epoch 66/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0770 - acc: 0.3394 - kullback_leibler_divergence: 0.0770 - laloss: 3.0391 - val_loss: 0.0787 - val_acc: 0.2750 - val_kullback_leibler_divergence: 0.0787 - val_laloss: 3.0066\n",
            "Epoch 67/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0765 - acc: 0.3599 - kullback_leibler_divergence: 0.0765 - laloss: 3.0864 - val_loss: 0.0834 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0834 - val_laloss: 3.0573\n",
            "Epoch 68/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0764 - acc: 0.3583 - kullback_leibler_divergence: 0.0764 - laloss: 3.0781 - val_loss: 0.0806 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0806 - val_laloss: 2.9715\n",
            "Epoch 69/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0775 - acc: 0.3309 - kullback_leibler_divergence: 0.0775 - laloss: 2.9995 - val_loss: 0.0784 - val_acc: 0.2950 - val_kullback_leibler_divergence: 0.0784 - val_laloss: 2.9318\n",
            "Epoch 70/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0743 - acc: 0.3584 - kullback_leibler_divergence: 0.0743 - laloss: 3.0573 - val_loss: 0.0783 - val_acc: 0.2850 - val_kullback_leibler_divergence: 0.0783 - val_laloss: 2.9918\n",
            "Epoch 71/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0749 - acc: 0.3477 - kullback_leibler_divergence: 0.0749 - laloss: 3.0354 - val_loss: 0.0786 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0786 - val_laloss: 3.0596\n",
            "Epoch 72/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0751 - acc: 0.3494 - kullback_leibler_divergence: 0.0751 - laloss: 3.0122 - val_loss: 0.0783 - val_acc: 0.2800 - val_kullback_leibler_divergence: 0.0783 - val_laloss: 2.8715\n",
            "Epoch 73/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0759 - acc: 0.3409 - kullback_leibler_divergence: 0.0759 - laloss: 3.0171 - val_loss: 0.0788 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0788 - val_laloss: 3.0309\n",
            "Epoch 74/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0742 - acc: 0.3329 - kullback_leibler_divergence: 0.0742 - laloss: 3.0570 - val_loss: 0.0812 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0812 - val_laloss: 3.0274\n",
            "Epoch 75/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0766 - acc: 0.3626 - kullback_leibler_divergence: 0.0766 - laloss: 2.9468 - val_loss: 0.0789 - val_acc: 0.2850 - val_kullback_leibler_divergence: 0.0789 - val_laloss: 2.9003\n",
            "Epoch 76/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0741 - acc: 0.3519 - kullback_leibler_divergence: 0.0741 - laloss: 2.9534 - val_loss: 0.0793 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0793 - val_laloss: 2.8202\n",
            "Epoch 77/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0762 - acc: 0.3477 - kullback_leibler_divergence: 0.0762 - laloss: 2.9357 - val_loss: 0.0799 - val_acc: 0.2650 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 3.1029\n",
            "Epoch 78/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0759 - acc: 0.3405 - kullback_leibler_divergence: 0.0759 - laloss: 3.0740 - val_loss: 0.0793 - val_acc: 0.2750 - val_kullback_leibler_divergence: 0.0793 - val_laloss: 2.7772\n",
            "Epoch 79/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0752 - acc: 0.3629 - kullback_leibler_divergence: 0.0752 - laloss: 2.9691 - val_loss: 0.0803 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0803 - val_laloss: 2.8649\n",
            "Epoch 80/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0759 - acc: 0.3551 - kullback_leibler_divergence: 0.0759 - laloss: 2.9078 - val_loss: 0.0819 - val_acc: 0.3000 - val_kullback_leibler_divergence: 0.0819 - val_laloss: 2.9488\n",
            "Epoch 81/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0772 - acc: 0.3494 - kullback_leibler_divergence: 0.0772 - laloss: 2.9816 - val_loss: 0.0795 - val_acc: 0.2950 - val_kullback_leibler_divergence: 0.0795 - val_laloss: 3.0211\n",
            "Epoch 82/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0769 - acc: 0.3476 - kullback_leibler_divergence: 0.0769 - laloss: 2.9853 - val_loss: 0.0788 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0788 - val_laloss: 3.0561\n",
            "Epoch 83/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0757 - acc: 0.3431 - kullback_leibler_divergence: 0.0757 - laloss: 3.0264 - val_loss: 0.0822 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0822 - val_laloss: 2.8645\n",
            "Epoch 84/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0770 - acc: 0.3466 - kullback_leibler_divergence: 0.0770 - laloss: 2.9370 - val_loss: 0.0800 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.9325\n",
            "Epoch 85/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0749 - acc: 0.3509 - kullback_leibler_divergence: 0.0749 - laloss: 2.9557 - val_loss: 0.0782 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0782 - val_laloss: 2.9680\n",
            "Epoch 86/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0747 - acc: 0.3673 - kullback_leibler_divergence: 0.0747 - laloss: 3.0355 - val_loss: 0.0794 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0794 - val_laloss: 2.8002\n",
            "Epoch 87/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0751 - acc: 0.3558 - kullback_leibler_divergence: 0.0751 - laloss: 2.9703 - val_loss: 0.0795 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0795 - val_laloss: 2.8427\n",
            "Epoch 88/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0740 - acc: 0.3658 - kullback_leibler_divergence: 0.0740 - laloss: 2.9733 - val_loss: 0.0783 - val_acc: 0.2950 - val_kullback_leibler_divergence: 0.0783 - val_laloss: 3.0618\n",
            "Epoch 89/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0749 - acc: 0.3611 - kullback_leibler_divergence: 0.0749 - laloss: 2.9864 - val_loss: 0.0795 - val_acc: 0.3000 - val_kullback_leibler_divergence: 0.0795 - val_laloss: 3.0253\n",
            "Epoch 90/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0751 - acc: 0.3462 - kullback_leibler_divergence: 0.0751 - laloss: 2.9800 - val_loss: 0.0810 - val_acc: 0.2800 - val_kullback_leibler_divergence: 0.0810 - val_laloss: 2.8737\n",
            "Epoch 91/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0776 - acc: 0.3349 - kullback_leibler_divergence: 0.0776 - laloss: 2.9099 - val_loss: 0.0812 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0812 - val_laloss: 2.9198\n",
            "Epoch 92/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0781 - acc: 0.3682 - kullback_leibler_divergence: 0.0781 - laloss: 2.8744 - val_loss: 0.0794 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0794 - val_laloss: 2.8907\n",
            "Epoch 93/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0753 - acc: 0.3589 - kullback_leibler_divergence: 0.0753 - laloss: 2.9724 - val_loss: 0.0783 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0783 - val_laloss: 2.9506\n",
            "Epoch 94/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0735 - acc: 0.3755 - kullback_leibler_divergence: 0.0735 - laloss: 2.9583 - val_loss: 0.0805 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0805 - val_laloss: 2.8964\n",
            "Epoch 95/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0760 - acc: 0.3703 - kullback_leibler_divergence: 0.0760 - laloss: 2.8972 - val_loss: 0.0804 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0804 - val_laloss: 2.8970\n",
            "Epoch 96/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0756 - acc: 0.3368 - kullback_leibler_divergence: 0.0756 - laloss: 2.9153 - val_loss: 0.0792 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0792 - val_laloss: 2.7178\n",
            "Epoch 97/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0750 - acc: 0.3583 - kullback_leibler_divergence: 0.0750 - laloss: 2.8645 - val_loss: 0.0782 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0782 - val_laloss: 2.8760\n",
            "Epoch 98/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0736 - acc: 0.3425 - kullback_leibler_divergence: 0.0736 - laloss: 2.9525 - val_loss: 0.0781 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0781 - val_laloss: 2.9422\n",
            "Epoch 99/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0766 - acc: 0.3600 - kullback_leibler_divergence: 0.0766 - laloss: 2.9642 - val_loss: 0.0849 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0849 - val_laloss: 2.8754\n",
            "Epoch 100/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0759 - acc: 0.3575 - kullback_leibler_divergence: 0.0759 - laloss: 2.8646 - val_loss: 0.0797 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.7877\n",
            "Epoch 101/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0750 - acc: 0.3648 - kullback_leibler_divergence: 0.0750 - laloss: 2.9423 - val_loss: 0.0798 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.9472\n",
            "Epoch 102/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0760 - acc: 0.3561 - kullback_leibler_divergence: 0.0760 - laloss: 3.0386 - val_loss: 0.0796 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0796 - val_laloss: 2.9101\n",
            "Epoch 103/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0764 - acc: 0.3593 - kullback_leibler_divergence: 0.0764 - laloss: 2.9711 - val_loss: 0.0819 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0819 - val_laloss: 3.0859\n",
            "Epoch 104/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0738 - acc: 0.3703 - kullback_leibler_divergence: 0.0738 - laloss: 3.0142 - val_loss: 0.0786 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0786 - val_laloss: 2.9980\n",
            "Epoch 105/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0752 - acc: 0.3579 - kullback_leibler_divergence: 0.0752 - laloss: 3.0381 - val_loss: 0.0785 - val_acc: 0.2950 - val_kullback_leibler_divergence: 0.0785 - val_laloss: 2.8429\n",
            "Epoch 106/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0729 - acc: 0.3557 - kullback_leibler_divergence: 0.0729 - laloss: 2.9323 - val_loss: 0.0782 - val_acc: 0.2950 - val_kullback_leibler_divergence: 0.0782 - val_laloss: 2.9449\n",
            "Epoch 107/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0749 - acc: 0.3503 - kullback_leibler_divergence: 0.0749 - laloss: 3.0015 - val_loss: 0.0798 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.9624\n",
            "Epoch 108/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0744 - acc: 0.3678 - kullback_leibler_divergence: 0.0744 - laloss: 2.9566 - val_loss: 0.0799 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.9407\n",
            "Epoch 109/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0736 - acc: 0.3520 - kullback_leibler_divergence: 0.0736 - laloss: 3.0273 - val_loss: 0.0796 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0796 - val_laloss: 2.9737\n",
            "Epoch 110/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0752 - acc: 0.3450 - kullback_leibler_divergence: 0.0752 - laloss: 3.0092 - val_loss: 0.0786 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0786 - val_laloss: 2.9280\n",
            "Epoch 111/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0740 - acc: 0.3524 - kullback_leibler_divergence: 0.0740 - laloss: 2.9594 - val_loss: 0.0884 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0884 - val_laloss: 2.8259\n",
            "Epoch 112/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0776 - acc: 0.3451 - kullback_leibler_divergence: 0.0776 - laloss: 2.9900 - val_loss: 0.0784 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0784 - val_laloss: 3.0427\n",
            "Epoch 113/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0731 - acc: 0.3632 - kullback_leibler_divergence: 0.0731 - laloss: 3.0244 - val_loss: 0.0901 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0901 - val_laloss: 2.9269\n",
            "Epoch 114/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0779 - acc: 0.3487 - kullback_leibler_divergence: 0.0779 - laloss: 2.9637 - val_loss: 0.0791 - val_acc: 0.2800 - val_kullback_leibler_divergence: 0.0791 - val_laloss: 2.9635\n",
            "Epoch 115/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0753 - acc: 0.3575 - kullback_leibler_divergence: 0.0753 - laloss: 2.9839 - val_loss: 0.0795 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0795 - val_laloss: 2.8248\n",
            "Epoch 116/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0744 - acc: 0.3623 - kullback_leibler_divergence: 0.0744 - laloss: 2.9152 - val_loss: 0.0788 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0788 - val_laloss: 2.7793\n",
            "Epoch 117/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0737 - acc: 0.3670 - kullback_leibler_divergence: 0.0737 - laloss: 2.9305 - val_loss: 0.0797 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.8695\n",
            "Epoch 118/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0740 - acc: 0.3776 - kullback_leibler_divergence: 0.0740 - laloss: 2.9230 - val_loss: 0.0788 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0788 - val_laloss: 2.9648\n",
            "Epoch 119/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0748 - acc: 0.3628 - kullback_leibler_divergence: 0.0748 - laloss: 3.0249 - val_loss: 0.0786 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0786 - val_laloss: 2.8657\n",
            "Epoch 120/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0757 - acc: 0.3429 - kullback_leibler_divergence: 0.0757 - laloss: 2.9210 - val_loss: 0.0805 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0805 - val_laloss: 2.9996\n",
            "Epoch 121/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0749 - acc: 0.3551 - kullback_leibler_divergence: 0.0749 - laloss: 2.9913 - val_loss: 0.0821 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0821 - val_laloss: 2.7443\n",
            "Epoch 122/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0772 - acc: 0.3609 - kullback_leibler_divergence: 0.0772 - laloss: 2.9835 - val_loss: 0.0792 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0792 - val_laloss: 2.8388\n",
            "Epoch 123/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0750 - acc: 0.3571 - kullback_leibler_divergence: 0.0750 - laloss: 2.9051 - val_loss: 0.0794 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0794 - val_laloss: 3.0135\n",
            "Epoch 124/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0749 - acc: 0.3598 - kullback_leibler_divergence: 0.0749 - laloss: 2.9529 - val_loss: 0.0885 - val_acc: 0.2700 - val_kullback_leibler_divergence: 0.0885 - val_laloss: 3.0483\n",
            "Epoch 125/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0760 - acc: 0.3539 - kullback_leibler_divergence: 0.0760 - laloss: 2.9952 - val_loss: 0.0799 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.8093\n",
            "Epoch 126/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0757 - acc: 0.3655 - kullback_leibler_divergence: 0.0757 - laloss: 2.9403 - val_loss: 0.0882 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0882 - val_laloss: 2.9304\n",
            "Epoch 127/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0771 - acc: 0.3476 - kullback_leibler_divergence: 0.0771 - laloss: 2.9053 - val_loss: 0.0793 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0793 - val_laloss: 3.0698\n",
            "Epoch 128/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0742 - acc: 0.3631 - kullback_leibler_divergence: 0.0742 - laloss: 2.9424 - val_loss: 0.0810 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0810 - val_laloss: 3.1322\n",
            "Epoch 129/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0743 - acc: 0.3575 - kullback_leibler_divergence: 0.0743 - laloss: 3.0526 - val_loss: 0.0791 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0791 - val_laloss: 2.8594\n",
            "Epoch 130/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0733 - acc: 0.3823 - kullback_leibler_divergence: 0.0733 - laloss: 2.8892 - val_loss: 0.0783 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0783 - val_laloss: 2.8514\n",
            "Epoch 131/200\n",
            "8/8 [==============================] - 0s 24ms/step - loss: 0.0745 - acc: 0.3741 - kullback_leibler_divergence: 0.0745 - laloss: 2.9376 - val_loss: 0.0796 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0796 - val_laloss: 2.9752\n",
            "Epoch 132/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0725 - acc: 0.3754 - kullback_leibler_divergence: 0.0725 - laloss: 2.9449 - val_loss: 0.0786 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0786 - val_laloss: 2.8816\n",
            "Epoch 133/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0737 - acc: 0.3595 - kullback_leibler_divergence: 0.0737 - laloss: 2.8764 - val_loss: 0.0829 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0829 - val_laloss: 2.7832\n",
            "Epoch 134/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0746 - acc: 0.3610 - kullback_leibler_divergence: 0.0746 - laloss: 2.8143 - val_loss: 0.0787 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0787 - val_laloss: 2.7510\n",
            "Epoch 135/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0741 - acc: 0.3656 - kullback_leibler_divergence: 0.0741 - laloss: 2.8897 - val_loss: 0.0786 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0786 - val_laloss: 2.9986\n",
            "Epoch 136/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0740 - acc: 0.3557 - kullback_leibler_divergence: 0.0740 - laloss: 2.9541 - val_loss: 0.0814 - val_acc: 0.2850 - val_kullback_leibler_divergence: 0.0814 - val_laloss: 3.0403\n",
            "Epoch 137/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0757 - acc: 0.3724 - kullback_leibler_divergence: 0.0757 - laloss: 2.9689 - val_loss: 0.0797 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.8371\n",
            "Epoch 138/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0725 - acc: 0.3760 - kullback_leibler_divergence: 0.0725 - laloss: 2.9951 - val_loss: 0.0786 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0786 - val_laloss: 2.9284\n",
            "Epoch 139/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0738 - acc: 0.3695 - kullback_leibler_divergence: 0.0738 - laloss: 2.9011 - val_loss: 0.0787 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0787 - val_laloss: 2.7535\n",
            "Epoch 140/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0752 - acc: 0.3499 - kullback_leibler_divergence: 0.0752 - laloss: 2.9953 - val_loss: 0.0806 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0806 - val_laloss: 2.7013\n",
            "Epoch 141/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0752 - acc: 0.3415 - kullback_leibler_divergence: 0.0752 - laloss: 2.8257 - val_loss: 0.0784 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0784 - val_laloss: 2.9310\n",
            "Epoch 142/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0735 - acc: 0.3724 - kullback_leibler_divergence: 0.0735 - laloss: 2.9257 - val_loss: 0.0784 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0784 - val_laloss: 2.7611\n",
            "Epoch 143/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0741 - acc: 0.3744 - kullback_leibler_divergence: 0.0741 - laloss: 2.8689 - val_loss: 0.0798 - val_acc: 0.2950 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.9077\n",
            "Epoch 144/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0738 - acc: 0.3766 - kullback_leibler_divergence: 0.0738 - laloss: 2.9190 - val_loss: 0.0793 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0793 - val_laloss: 2.7787\n",
            "Epoch 145/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0742 - acc: 0.3556 - kullback_leibler_divergence: 0.0742 - laloss: 2.7874 - val_loss: 0.0785 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0785 - val_laloss: 2.7754\n",
            "Epoch 146/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0738 - acc: 0.3696 - kullback_leibler_divergence: 0.0738 - laloss: 2.8989 - val_loss: 0.0825 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0825 - val_laloss: 2.7511\n",
            "Epoch 147/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0753 - acc: 0.3599 - kullback_leibler_divergence: 0.0753 - laloss: 2.8817 - val_loss: 0.0782 - val_acc: 0.3500 - val_kullback_leibler_divergence: 0.0782 - val_laloss: 2.8911\n",
            "Epoch 148/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0734 - acc: 0.3594 - kullback_leibler_divergence: 0.0734 - laloss: 2.9251 - val_loss: 0.0836 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0836 - val_laloss: 2.8386\n",
            "Epoch 149/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0748 - acc: 0.3511 - kullback_leibler_divergence: 0.0748 - laloss: 2.8826 - val_loss: 0.0796 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0796 - val_laloss: 2.9488\n",
            "Epoch 150/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0737 - acc: 0.3823 - kullback_leibler_divergence: 0.0737 - laloss: 2.8947 - val_loss: 0.0801 - val_acc: 0.2700 - val_kullback_leibler_divergence: 0.0801 - val_laloss: 2.9486\n",
            "Epoch 151/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0728 - acc: 0.3605 - kullback_leibler_divergence: 0.0728 - laloss: 2.9438 - val_loss: 0.0804 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0804 - val_laloss: 3.0169\n",
            "Epoch 152/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0733 - acc: 0.3767 - kullback_leibler_divergence: 0.0733 - laloss: 2.9592 - val_loss: 0.0810 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0810 - val_laloss: 2.9277\n",
            "Epoch 153/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0742 - acc: 0.3649 - kullback_leibler_divergence: 0.0742 - laloss: 2.9325 - val_loss: 0.0867 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0867 - val_laloss: 3.0273\n",
            "Epoch 154/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0756 - acc: 0.3443 - kullback_leibler_divergence: 0.0756 - laloss: 2.8854 - val_loss: 0.0848 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0848 - val_laloss: 2.8208\n",
            "Epoch 155/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0743 - acc: 0.3613 - kullback_leibler_divergence: 0.0743 - laloss: 2.9570 - val_loss: 0.0785 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0785 - val_laloss: 2.6980\n",
            "Epoch 156/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0727 - acc: 0.3606 - kullback_leibler_divergence: 0.0727 - laloss: 2.8445 - val_loss: 0.0793 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0793 - val_laloss: 2.7769\n",
            "Epoch 157/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0730 - acc: 0.3670 - kullback_leibler_divergence: 0.0730 - laloss: 2.8321 - val_loss: 0.0786 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0786 - val_laloss: 2.7167\n",
            "Epoch 158/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0728 - acc: 0.3733 - kullback_leibler_divergence: 0.0728 - laloss: 2.9279 - val_loss: 0.0802 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0802 - val_laloss: 3.0203\n",
            "Epoch 159/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0740 - acc: 0.3839 - kullback_leibler_divergence: 0.0740 - laloss: 2.8045 - val_loss: 0.0798 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.8433\n",
            "Epoch 160/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0741 - acc: 0.3739 - kullback_leibler_divergence: 0.0741 - laloss: 2.8155 - val_loss: 0.0804 - val_acc: 0.2850 - val_kullback_leibler_divergence: 0.0804 - val_laloss: 2.9781\n",
            "Epoch 161/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0742 - acc: 0.3711 - kullback_leibler_divergence: 0.0742 - laloss: 2.9374 - val_loss: 0.0782 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0782 - val_laloss: 2.7771\n",
            "Epoch 162/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0728 - acc: 0.3715 - kullback_leibler_divergence: 0.0728 - laloss: 2.8348 - val_loss: 0.0808 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0808 - val_laloss: 2.9096\n",
            "Epoch 163/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0756 - acc: 0.3526 - kullback_leibler_divergence: 0.0756 - laloss: 2.8023 - val_loss: 0.0782 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0782 - val_laloss: 2.7039\n",
            "Epoch 164/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0734 - acc: 0.3564 - kullback_leibler_divergence: 0.0734 - laloss: 2.7621 - val_loss: 0.0895 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0895 - val_laloss: 2.8261\n",
            "Epoch 165/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0764 - acc: 0.3494 - kullback_leibler_divergence: 0.0764 - laloss: 2.9196 - val_loss: 0.0780 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0780 - val_laloss: 2.9615\n",
            "Epoch 166/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0723 - acc: 0.3767 - kullback_leibler_divergence: 0.0723 - laloss: 2.9270 - val_loss: 0.0799 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.8337\n",
            "Epoch 167/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0733 - acc: 0.3710 - kullback_leibler_divergence: 0.0733 - laloss: 2.9375 - val_loss: 0.0796 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0796 - val_laloss: 2.9343\n",
            "Epoch 168/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0746 - acc: 0.3818 - kullback_leibler_divergence: 0.0746 - laloss: 2.9090 - val_loss: 0.0781 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0781 - val_laloss: 2.7235\n",
            "Epoch 169/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0740 - acc: 0.3742 - kullback_leibler_divergence: 0.0740 - laloss: 2.8885 - val_loss: 0.0790 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0790 - val_laloss: 2.8583\n",
            "Epoch 170/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0742 - acc: 0.3732 - kullback_leibler_divergence: 0.0742 - laloss: 2.8815 - val_loss: 0.0817 - val_acc: 0.2800 - val_kullback_leibler_divergence: 0.0817 - val_laloss: 2.9072\n",
            "Epoch 171/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0724 - acc: 0.3791 - kullback_leibler_divergence: 0.0724 - laloss: 2.9534 - val_loss: 0.0801 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0801 - val_laloss: 3.0000\n",
            "Epoch 172/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0721 - acc: 0.3744 - kullback_leibler_divergence: 0.0721 - laloss: 2.8536 - val_loss: 0.0782 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0782 - val_laloss: 2.7236\n",
            "Epoch 173/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0738 - acc: 0.3474 - kullback_leibler_divergence: 0.0738 - laloss: 2.8350 - val_loss: 0.0792 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0792 - val_laloss: 2.8806\n",
            "Epoch 174/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0745 - acc: 0.3570 - kullback_leibler_divergence: 0.0745 - laloss: 2.8120 - val_loss: 0.0812 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0812 - val_laloss: 2.9169\n",
            "Epoch 175/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0752 - acc: 0.3750 - kullback_leibler_divergence: 0.0752 - laloss: 2.9161 - val_loss: 0.0891 - val_acc: 0.2850 - val_kullback_leibler_divergence: 0.0891 - val_laloss: 2.8890\n",
            "Epoch 176/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0772 - acc: 0.3593 - kullback_leibler_divergence: 0.0772 - laloss: 2.8311 - val_loss: 0.0782 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0782 - val_laloss: 2.7144\n",
            "Epoch 177/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0729 - acc: 0.3782 - kullback_leibler_divergence: 0.0729 - laloss: 2.8097 - val_loss: 0.0803 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0803 - val_laloss: 2.8950\n",
            "Epoch 178/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0726 - acc: 0.3867 - kullback_leibler_divergence: 0.0726 - laloss: 2.7627 - val_loss: 0.0800 - val_acc: 0.3550 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.7894\n",
            "Epoch 179/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0717 - acc: 0.3861 - kullback_leibler_divergence: 0.0717 - laloss: 2.7810 - val_loss: 0.0825 - val_acc: 0.3000 - val_kullback_leibler_divergence: 0.0825 - val_laloss: 2.7394\n",
            "Epoch 180/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0746 - acc: 0.3545 - kullback_leibler_divergence: 0.0746 - laloss: 2.6878 - val_loss: 0.0789 - val_acc: 0.3450 - val_kullback_leibler_divergence: 0.0789 - val_laloss: 2.7176\n",
            "Epoch 181/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0718 - acc: 0.3963 - kullback_leibler_divergence: 0.0718 - laloss: 2.7903 - val_loss: 0.0806 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0806 - val_laloss: 2.8601\n",
            "Epoch 182/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0738 - acc: 0.3813 - kullback_leibler_divergence: 0.0738 - laloss: 2.7235 - val_loss: 0.0801 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0801 - val_laloss: 2.7969\n",
            "Epoch 183/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0746 - acc: 0.3819 - kullback_leibler_divergence: 0.0746 - laloss: 2.7784 - val_loss: 0.0806 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0806 - val_laloss: 2.7988\n",
            "Epoch 184/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0744 - acc: 0.3818 - kullback_leibler_divergence: 0.0744 - laloss: 2.7948 - val_loss: 0.0782 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0782 - val_laloss: 2.7944\n",
            "Epoch 185/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0727 - acc: 0.3712 - kullback_leibler_divergence: 0.0727 - laloss: 2.7946 - val_loss: 0.0813 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0813 - val_laloss: 2.7061\n",
            "Epoch 186/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0747 - acc: 0.3543 - kullback_leibler_divergence: 0.0747 - laloss: 2.8111 - val_loss: 0.0778 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0778 - val_laloss: 2.8746\n",
            "Epoch 187/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0721 - acc: 0.3861 - kullback_leibler_divergence: 0.0721 - laloss: 2.9512 - val_loss: 0.0797 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.6941\n",
            "Epoch 188/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0749 - acc: 0.3660 - kullback_leibler_divergence: 0.0749 - laloss: 2.8557 - val_loss: 0.0785 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0785 - val_laloss: 2.8492\n",
            "Epoch 189/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0730 - acc: 0.3776 - kullback_leibler_divergence: 0.0730 - laloss: 2.9089 - val_loss: 0.0779 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0779 - val_laloss: 2.9003\n",
            "Epoch 190/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0722 - acc: 0.3722 - kullback_leibler_divergence: 0.0722 - laloss: 2.8936 - val_loss: 0.0808 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0808 - val_laloss: 2.6931\n",
            "Epoch 191/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0747 - acc: 0.3441 - kullback_leibler_divergence: 0.0747 - laloss: 2.8442 - val_loss: 0.0783 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0783 - val_laloss: 2.8332\n",
            "Epoch 192/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0720 - acc: 0.3789 - kullback_leibler_divergence: 0.0720 - laloss: 2.8898 - val_loss: 0.0815 - val_acc: 0.3500 - val_kullback_leibler_divergence: 0.0815 - val_laloss: 2.7833\n",
            "Epoch 193/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0735 - acc: 0.3517 - kullback_leibler_divergence: 0.0735 - laloss: 2.9037 - val_loss: 0.0778 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0778 - val_laloss: 2.6583\n",
            "Epoch 194/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0736 - acc: 0.3921 - kullback_leibler_divergence: 0.0736 - laloss: 2.8944 - val_loss: 0.0871 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0871 - val_laloss: 2.8464\n",
            "Epoch 195/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0751 - acc: 0.3618 - kullback_leibler_divergence: 0.0751 - laloss: 2.8518 - val_loss: 0.0794 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0794 - val_laloss: 2.8563\n",
            "Epoch 196/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0744 - acc: 0.3836 - kullback_leibler_divergence: 0.0744 - laloss: 2.7605 - val_loss: 0.0777 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0777 - val_laloss: 2.8425\n",
            "Epoch 197/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0733 - acc: 0.3856 - kullback_leibler_divergence: 0.0733 - laloss: 2.8212 - val_loss: 0.0801 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0801 - val_laloss: 2.6748\n",
            "Epoch 198/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0729 - acc: 0.3457 - kullback_leibler_divergence: 0.0729 - laloss: 2.7208 - val_loss: 0.0799 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.6642\n",
            "Epoch 199/200\n",
            "8/8 [==============================] - 0s 26ms/step - loss: 0.0737 - acc: 0.3413 - kullback_leibler_divergence: 0.0737 - laloss: 2.7329 - val_loss: 0.0774 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0774 - val_laloss: 2.6917\n",
            "Epoch 200/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0732 - acc: 0.3735 - kullback_leibler_divergence: 0.0732 - laloss: 2.7524 - val_loss: 0.0824 - val_acc: 0.3000 - val_kullback_leibler_divergence: 0.0824 - val_laloss: 2.8486\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qw3xEWzO8-Qi",
        "outputId": "6fdc9cf2-c8b4-486c-a073-b660f4a79eec"
      },
      "source": [
        "y_pred_baseline = model_baseline.predict(X_test)\n",
        "\n",
        "eval_baseline_sll = clf_eval(y_sl_test,get_single_label(y_pred_baseline),y_pred)\n",
        "eval_baseline_mll = clf_eval(y_ml_test,get_multi_label(y_pred_baseline),y_pred)\n",
        "eval_baseline = ldl_eval(y_test,y_pred_baseline)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "acc: 0.336\n",
            "f1 score(micro): 0.3444881889763779\n",
            "precision score(micro): 0.35\n",
            "recall score(micro): 0.3391472868217054\n",
            "AUC score(micro): 0.6041549638617383\n",
            "hamming loss: 0.222\n",
            "label ranking loss: 0.32535000000000003\n",
            "confusion matrix: [[[373  11]\n",
            "  [101  15]]\n",
            "\n",
            " [[418   0]\n",
            "  [ 82   0]]\n",
            "\n",
            " [[187 162]\n",
            "  [ 57  94]]\n",
            "\n",
            " [[447   0]\n",
            "  [ 53   0]]\n",
            "\n",
            " [[475   0]\n",
            "  [ 25   0]]\n",
            "\n",
            " [[259 152]\n",
            "  [ 23  66]]]\n",
            "acc: 0.1\n",
            "f1 score(micro): 0.4031981724728726\n",
            "precision score(micro): 0.37236286919831224\n",
            "recall score(micro): 0.4396014943960149\n",
            "AUC score(micro): 0.5843888218452538\n",
            "hamming loss: 0.34833333333333333\n",
            "label ranking loss: 0.32190555555555556\n",
            "confusion matrix: [[[187 177]\n",
            "  [ 37  99]]\n",
            "\n",
            " [[326  12]\n",
            "  [154   8]]\n",
            "\n",
            " [[106 184]\n",
            "  [ 45 165]]\n",
            "\n",
            " [[370   0]\n",
            "  [130   0]]\n",
            "\n",
            " [[395  48]\n",
            "  [ 51   6]]\n",
            "\n",
            " [[218 174]\n",
            "  [ 33  75]]]\n",
            "Kullback–Leibler divergence:  0.07822978994203554\n",
            "Euclidean distance:  0.15877945818945868\n",
            "MSE:  0.005309593031847453\n",
            "Chebyshev distance:  0.1281805452083467\n",
            "Clark distance:  0.3932137098298129\n",
            "Canberra distance:  0.8354767095205718\n",
            "Cosine similarity:  0.9228523772059668\n",
            "Intersection similarity:  0.8487188461872034\n",
            "Kurtosis_signed_offset:  -0.17324235567122914\n",
            "Kurtosis_abs_offset:  1.1024632706664494\n",
            "Laloss:  0.6378528131688392\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s8v8mP8IRLxJ",
        "outputId": "7c1685d4-3a04-4893-8bb3-9920f81f42c1"
      },
      "source": [
        "model = model_baseline\n",
        "model.compile(optimizer=keras.optimizers.RMSprop(learning_rate=0.0001),loss= label_ambiguity_loss, metrics=['acc','kullback_leibler_divergence',laloss])\n",
        "history = model.fit(partial_X_train,partial_y_train,batch_size=256,epochs=100,validation_data=(X_val,y_val),verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/100\n",
            "8/8 [==============================] - 1s 37ms/step - loss: 0.0806 - acc: 0.3556 - kullback_leibler_divergence: 0.0762 - laloss: 2.8134 - val_loss: 0.0823 - val_acc: 0.2850 - val_kullback_leibler_divergence: 0.0806 - val_laloss: 2.8006\n",
            "Epoch 2/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0785 - acc: 0.3551 - kullback_leibler_divergence: 0.0774 - laloss: 2.6844 - val_loss: 0.0778 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0794 - val_laloss: 2.8353\n",
            "Epoch 3/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0730 - acc: 0.3580 - kullback_leibler_divergence: 0.0745 - laloss: 2.6465 - val_loss: 0.0763 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0793 - val_laloss: 2.7404\n",
            "Epoch 4/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0734 - acc: 0.3529 - kullback_leibler_divergence: 0.0752 - laloss: 2.7153 - val_loss: 0.0747 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0793 - val_laloss: 2.6485\n",
            "Epoch 5/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0711 - acc: 0.3648 - kullback_leibler_divergence: 0.0751 - laloss: 2.7029 - val_loss: 0.0730 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0795 - val_laloss: 2.5181\n",
            "Epoch 6/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0739 - acc: 0.3596 - kullback_leibler_divergence: 0.0755 - laloss: 2.6943 - val_loss: 0.0731 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0795 - val_laloss: 2.5777\n",
            "Epoch 7/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0708 - acc: 0.3604 - kullback_leibler_divergence: 0.0740 - laloss: 2.6234 - val_loss: 0.0721 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0795 - val_laloss: 2.4172\n",
            "Epoch 8/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0693 - acc: 0.3781 - kullback_leibler_divergence: 0.0734 - laloss: 2.7333 - val_loss: 0.0721 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0794 - val_laloss: 2.4844\n",
            "Epoch 9/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0735 - acc: 0.3667 - kullback_leibler_divergence: 0.0746 - laloss: 2.6498 - val_loss: 0.0720 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0794 - val_laloss: 2.6015\n",
            "Epoch 10/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0708 - acc: 0.3636 - kullback_leibler_divergence: 0.0733 - laloss: 2.6865 - val_loss: 0.0720 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.6541\n",
            "Epoch 11/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0715 - acc: 0.3728 - kullback_leibler_divergence: 0.0742 - laloss: 2.5871 - val_loss: 0.0727 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.5790\n",
            "Epoch 12/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0708 - acc: 0.3694 - kullback_leibler_divergence: 0.0745 - laloss: 2.7010 - val_loss: 0.0740 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0801 - val_laloss: 2.5831\n",
            "Epoch 13/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0722 - acc: 0.3607 - kullback_leibler_divergence: 0.0744 - laloss: 2.7434 - val_loss: 0.0739 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0801 - val_laloss: 2.5455\n",
            "Epoch 14/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0703 - acc: 0.3653 - kullback_leibler_divergence: 0.0741 - laloss: 2.6239 - val_loss: 0.0721 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0796 - val_laloss: 2.6559\n",
            "Epoch 15/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0697 - acc: 0.3739 - kullback_leibler_divergence: 0.0738 - laloss: 2.4793 - val_loss: 0.0719 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0796 - val_laloss: 2.6586\n",
            "Epoch 16/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0718 - acc: 0.3639 - kullback_leibler_divergence: 0.0758 - laloss: 2.7283 - val_loss: 0.0716 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0795 - val_laloss: 2.5950\n",
            "Epoch 17/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0691 - acc: 0.3640 - kullback_leibler_divergence: 0.0722 - laloss: 2.6321 - val_loss: 0.0721 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.5631\n",
            "Epoch 18/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0667 - acc: 0.3819 - kullback_leibler_divergence: 0.0727 - laloss: 2.6910 - val_loss: 0.0715 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.6587\n",
            "Epoch 19/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0720 - acc: 0.3667 - kullback_leibler_divergence: 0.0754 - laloss: 2.5831 - val_loss: 0.0715 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.6413\n",
            "Epoch 20/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0700 - acc: 0.3652 - kullback_leibler_divergence: 0.0739 - laloss: 2.7626 - val_loss: 0.0717 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.6620\n",
            "Epoch 21/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0695 - acc: 0.3562 - kullback_leibler_divergence: 0.0748 - laloss: 2.6486 - val_loss: 0.0716 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.6210\n",
            "Epoch 22/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0678 - acc: 0.3655 - kullback_leibler_divergence: 0.0729 - laloss: 2.6072 - val_loss: 0.0714 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.4178\n",
            "Epoch 23/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0687 - acc: 0.3727 - kullback_leibler_divergence: 0.0737 - laloss: 2.6055 - val_loss: 0.0709 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.5055\n",
            "Epoch 24/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0701 - acc: 0.3835 - kullback_leibler_divergence: 0.0738 - laloss: 2.6510 - val_loss: 0.0709 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.4837\n",
            "Epoch 25/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0706 - acc: 0.3573 - kullback_leibler_divergence: 0.0752 - laloss: 2.6019 - val_loss: 0.0708 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.4455\n",
            "Epoch 26/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0705 - acc: 0.3638 - kullback_leibler_divergence: 0.0753 - laloss: 2.5972 - val_loss: 0.0709 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.5774\n",
            "Epoch 27/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0696 - acc: 0.3701 - kullback_leibler_divergence: 0.0750 - laloss: 2.6283 - val_loss: 0.0713 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.6534\n",
            "Epoch 28/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0674 - acc: 0.3798 - kullback_leibler_divergence: 0.0743 - laloss: 2.6487 - val_loss: 0.0727 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0802 - val_laloss: 2.5166\n",
            "Epoch 29/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0708 - acc: 0.3524 - kullback_leibler_divergence: 0.0752 - laloss: 2.4414 - val_loss: 0.0712 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.6622\n",
            "Epoch 30/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0696 - acc: 0.3639 - kullback_leibler_divergence: 0.0743 - laloss: 2.5074 - val_loss: 0.0716 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0801 - val_laloss: 2.6312\n",
            "Epoch 31/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0705 - acc: 0.3608 - kullback_leibler_divergence: 0.0745 - laloss: 2.6787 - val_loss: 0.0728 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0802 - val_laloss: 2.5564\n",
            "Epoch 32/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0683 - acc: 0.3542 - kullback_leibler_divergence: 0.0745 - laloss: 2.5147 - val_loss: 0.0719 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.5280\n",
            "Epoch 33/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0683 - acc: 0.3601 - kullback_leibler_divergence: 0.0739 - laloss: 2.6605 - val_loss: 0.0709 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.5998\n",
            "Epoch 34/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0732 - acc: 0.3570 - kullback_leibler_divergence: 0.0767 - laloss: 2.6129 - val_loss: 0.0705 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.5605\n",
            "Epoch 35/100\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0692 - acc: 0.3721 - kullback_leibler_divergence: 0.0740 - laloss: 2.6604 - val_loss: 0.0719 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.6018\n",
            "Epoch 36/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0698 - acc: 0.3628 - kullback_leibler_divergence: 0.0738 - laloss: 2.6172 - val_loss: 0.0709 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.6519\n",
            "Epoch 37/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0707 - acc: 0.3756 - kullback_leibler_divergence: 0.0745 - laloss: 2.5035 - val_loss: 0.0729 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.5545\n",
            "Epoch 38/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0688 - acc: 0.3677 - kullback_leibler_divergence: 0.0734 - laloss: 2.6559 - val_loss: 0.0724 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.4686\n",
            "Epoch 39/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0702 - acc: 0.3654 - kullback_leibler_divergence: 0.0744 - laloss: 2.6555 - val_loss: 0.0738 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.6979\n",
            "Epoch 40/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0697 - acc: 0.3743 - kullback_leibler_divergence: 0.0740 - laloss: 2.6981 - val_loss: 0.0739 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.6967\n",
            "Epoch 41/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0713 - acc: 0.3494 - kullback_leibler_divergence: 0.0760 - laloss: 2.7013 - val_loss: 0.0741 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.6833\n",
            "Epoch 42/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0692 - acc: 0.3620 - kullback_leibler_divergence: 0.0742 - laloss: 2.6532 - val_loss: 0.0732 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.6074\n",
            "Epoch 43/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0705 - acc: 0.3628 - kullback_leibler_divergence: 0.0746 - laloss: 2.7324 - val_loss: 0.0717 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.5093\n",
            "Epoch 44/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0699 - acc: 0.3677 - kullback_leibler_divergence: 0.0752 - laloss: 2.5932 - val_loss: 0.0720 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.4545\n",
            "Epoch 45/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0710 - acc: 0.3585 - kullback_leibler_divergence: 0.0751 - laloss: 2.4876 - val_loss: 0.0723 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.5208\n",
            "Epoch 46/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0693 - acc: 0.3568 - kullback_leibler_divergence: 0.0741 - laloss: 2.5186 - val_loss: 0.0737 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0801 - val_laloss: 2.5986\n",
            "Epoch 47/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0708 - acc: 0.3589 - kullback_leibler_divergence: 0.0757 - laloss: 2.6509 - val_loss: 0.0727 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.4961\n",
            "Epoch 48/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0691 - acc: 0.3674 - kullback_leibler_divergence: 0.0746 - laloss: 2.5263 - val_loss: 0.0730 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.6171\n",
            "Epoch 49/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0691 - acc: 0.3518 - kullback_leibler_divergence: 0.0747 - laloss: 2.6301 - val_loss: 0.0720 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.4838\n",
            "Epoch 50/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0672 - acc: 0.3712 - kullback_leibler_divergence: 0.0726 - laloss: 2.5576 - val_loss: 0.0703 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.4039\n",
            "Epoch 51/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0687 - acc: 0.3600 - kullback_leibler_divergence: 0.0744 - laloss: 2.4992 - val_loss: 0.0708 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.6099\n",
            "Epoch 52/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0714 - acc: 0.3493 - kullback_leibler_divergence: 0.0751 - laloss: 2.5858 - val_loss: 0.0711 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.6430\n",
            "Epoch 53/100\n",
            "8/8 [==============================] - 0s 26ms/step - loss: 0.0713 - acc: 0.3637 - kullback_leibler_divergence: 0.0761 - laloss: 2.5208 - val_loss: 0.0713 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.5934\n",
            "Epoch 54/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0701 - acc: 0.3592 - kullback_leibler_divergence: 0.0760 - laloss: 2.5797 - val_loss: 0.0709 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.5500\n",
            "Epoch 55/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0681 - acc: 0.3581 - kullback_leibler_divergence: 0.0746 - laloss: 2.5318 - val_loss: 0.0714 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.6274\n",
            "Epoch 56/100\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0672 - acc: 0.3629 - kullback_leibler_divergence: 0.0731 - laloss: 2.6498 - val_loss: 0.0721 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.4916\n",
            "Epoch 57/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0659 - acc: 0.3650 - kullback_leibler_divergence: 0.0731 - laloss: 2.5062 - val_loss: 0.0710 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.5673\n",
            "Epoch 58/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0701 - acc: 0.3656 - kullback_leibler_divergence: 0.0753 - laloss: 2.4585 - val_loss: 0.0720 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.4412\n",
            "Epoch 59/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0683 - acc: 0.3635 - kullback_leibler_divergence: 0.0739 - laloss: 2.5914 - val_loss: 0.0732 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.6776\n",
            "Epoch 60/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0685 - acc: 0.3595 - kullback_leibler_divergence: 0.0752 - laloss: 2.4956 - val_loss: 0.0715 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.5645\n",
            "Epoch 61/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0708 - acc: 0.3682 - kullback_leibler_divergence: 0.0754 - laloss: 2.5276 - val_loss: 0.0708 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.6543\n",
            "Epoch 62/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0678 - acc: 0.3527 - kullback_leibler_divergence: 0.0753 - laloss: 2.5062 - val_loss: 0.0706 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.3977\n",
            "Epoch 63/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0653 - acc: 0.3745 - kullback_leibler_divergence: 0.0728 - laloss: 2.5839 - val_loss: 0.0702 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.4658\n",
            "Epoch 64/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0703 - acc: 0.3529 - kullback_leibler_divergence: 0.0748 - laloss: 2.5428 - val_loss: 0.0701 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.4906\n",
            "Epoch 65/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0677 - acc: 0.3801 - kullback_leibler_divergence: 0.0744 - laloss: 2.5246 - val_loss: 0.0704 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.2440\n",
            "Epoch 66/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0681 - acc: 0.3782 - kullback_leibler_divergence: 0.0745 - laloss: 2.3752 - val_loss: 0.0701 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.0964\n",
            "Epoch 67/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0676 - acc: 0.3617 - kullback_leibler_divergence: 0.0735 - laloss: 2.4746 - val_loss: 0.0696 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.1088\n",
            "Epoch 68/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0686 - acc: 0.3751 - kullback_leibler_divergence: 0.0730 - laloss: 2.4447 - val_loss: 0.0693 - val_acc: 0.3350 - val_kullback_leibler_divergence: 0.0801 - val_laloss: 2.4915\n",
            "Epoch 69/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0677 - acc: 0.3659 - kullback_leibler_divergence: 0.0744 - laloss: 2.5009 - val_loss: 0.0701 - val_acc: 0.3250 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.1639\n",
            "Epoch 70/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0672 - acc: 0.3655 - kullback_leibler_divergence: 0.0745 - laloss: 2.4890 - val_loss: 0.0707 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.3834\n",
            "Epoch 71/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0687 - acc: 0.3557 - kullback_leibler_divergence: 0.0744 - laloss: 2.5051 - val_loss: 0.0705 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0796 - val_laloss: 2.5077\n",
            "Epoch 72/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0688 - acc: 0.3779 - kullback_leibler_divergence: 0.0739 - laloss: 2.5195 - val_loss: 0.0705 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0796 - val_laloss: 2.4889\n",
            "Epoch 73/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0679 - acc: 0.3704 - kullback_leibler_divergence: 0.0734 - laloss: 2.4942 - val_loss: 0.0708 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.2343\n",
            "Epoch 74/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0664 - acc: 0.3740 - kullback_leibler_divergence: 0.0727 - laloss: 2.5601 - val_loss: 0.0708 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.3862\n",
            "Epoch 75/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0662 - acc: 0.3704 - kullback_leibler_divergence: 0.0734 - laloss: 2.3484 - val_loss: 0.0716 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.6244\n",
            "Epoch 76/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0691 - acc: 0.3506 - kullback_leibler_divergence: 0.0746 - laloss: 2.4533 - val_loss: 0.0722 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0796 - val_laloss: 2.6011\n",
            "Epoch 77/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0657 - acc: 0.3681 - kullback_leibler_divergence: 0.0726 - laloss: 2.4532 - val_loss: 0.0722 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.4626\n",
            "Epoch 78/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0687 - acc: 0.3540 - kullback_leibler_divergence: 0.0733 - laloss: 2.5065 - val_loss: 0.0729 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.4425\n",
            "Epoch 79/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0664 - acc: 0.3650 - kullback_leibler_divergence: 0.0737 - laloss: 2.4776 - val_loss: 0.0733 - val_acc: 0.3000 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.5810\n",
            "Epoch 80/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0690 - acc: 0.3735 - kullback_leibler_divergence: 0.0751 - laloss: 2.5460 - val_loss: 0.0724 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.4290\n",
            "Epoch 81/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0699 - acc: 0.3565 - kullback_leibler_divergence: 0.0760 - laloss: 2.4849 - val_loss: 0.0712 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.3628\n",
            "Epoch 82/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0667 - acc: 0.3834 - kullback_leibler_divergence: 0.0725 - laloss: 2.5054 - val_loss: 0.0700 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.5074\n",
            "Epoch 83/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0676 - acc: 0.3731 - kullback_leibler_divergence: 0.0749 - laloss: 2.4338 - val_loss: 0.0709 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.2604\n",
            "Epoch 84/100\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0677 - acc: 0.3711 - kullback_leibler_divergence: 0.0747 - laloss: 2.4776 - val_loss: 0.0704 - val_acc: 0.3150 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.5145\n",
            "Epoch 85/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0691 - acc: 0.3771 - kullback_leibler_divergence: 0.0748 - laloss: 2.4725 - val_loss: 0.0713 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.3927\n",
            "Epoch 86/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0654 - acc: 0.3756 - kullback_leibler_divergence: 0.0724 - laloss: 2.4650 - val_loss: 0.0718 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.5606\n",
            "Epoch 87/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0694 - acc: 0.3675 - kullback_leibler_divergence: 0.0745 - laloss: 2.5106 - val_loss: 0.0703 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.4047\n",
            "Epoch 88/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0691 - acc: 0.3693 - kullback_leibler_divergence: 0.0757 - laloss: 2.4080 - val_loss: 0.0703 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.5036\n",
            "Epoch 89/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0701 - acc: 0.3624 - kullback_leibler_divergence: 0.0756 - laloss: 2.4133 - val_loss: 0.0709 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.5192\n",
            "Epoch 90/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0682 - acc: 0.3538 - kullback_leibler_divergence: 0.0749 - laloss: 2.4542 - val_loss: 0.0706 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0797 - val_laloss: 2.5459\n",
            "Epoch 91/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0685 - acc: 0.3609 - kullback_leibler_divergence: 0.0748 - laloss: 2.3288 - val_loss: 0.0702 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0798 - val_laloss: 2.4654\n",
            "Epoch 92/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0690 - acc: 0.3725 - kullback_leibler_divergence: 0.0747 - laloss: 2.3065 - val_loss: 0.0713 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.4662\n",
            "Epoch 93/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0672 - acc: 0.3657 - kullback_leibler_divergence: 0.0736 - laloss: 2.4526 - val_loss: 0.0705 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.5087\n",
            "Epoch 94/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0666 - acc: 0.3627 - kullback_leibler_divergence: 0.0739 - laloss: 2.4599 - val_loss: 0.0702 - val_acc: 0.3200 - val_kullback_leibler_divergence: 0.0799 - val_laloss: 2.4814\n",
            "Epoch 95/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0650 - acc: 0.3863 - kullback_leibler_divergence: 0.0740 - laloss: 2.3457 - val_loss: 0.0731 - val_acc: 0.3000 - val_kullback_leibler_divergence: 0.0804 - val_laloss: 2.5153\n",
            "Epoch 96/100\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0679 - acc: 0.3617 - kullback_leibler_divergence: 0.0741 - laloss: 2.4188 - val_loss: 0.0709 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0801 - val_laloss: 2.5007\n",
            "Epoch 97/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0667 - acc: 0.3711 - kullback_leibler_divergence: 0.0734 - laloss: 2.4235 - val_loss: 0.0716 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0803 - val_laloss: 2.3940\n",
            "Epoch 98/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0672 - acc: 0.3632 - kullback_leibler_divergence: 0.0741 - laloss: 2.4628 - val_loss: 0.0711 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0800 - val_laloss: 2.3440\n",
            "Epoch 99/100\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0676 - acc: 0.3593 - kullback_leibler_divergence: 0.0741 - laloss: 2.2441 - val_loss: 0.0699 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0801 - val_laloss: 2.4697\n",
            "Epoch 100/100\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0673 - acc: 0.3816 - kullback_leibler_divergence: 0.0748 - laloss: 2.3120 - val_loss: 0.0704 - val_acc: 0.3050 - val_kullback_leibler_divergence: 0.0802 - val_laloss: 2.4229\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e8AQ7aEIlZYq",
        "outputId": "408b4652-6e9b-4458-8e23-ce3e990b79ee"
      },
      "source": [
        "y_pred = model.predict(X_test)\n",
        "\n",
        "eval_sll = clf_eval(y_sl_test,get_single_label(y_pred),y_pred)\n",
        "eval_mll = clf_eval(y_ml_test,get_multi_label(y_pred),y_pred)\n",
        "eval = ldl_eval(y_test,y_pred)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "acc: 0.336\n",
            "f1 score(micro): 0.35236220472440943\n",
            "precision score(micro): 0.358\n",
            "recall score(micro): 0.34689922480620156\n",
            "AUC score(micro): 0.6088360858330525\n",
            "hamming loss: 0.21933333333333332\n",
            "label ranking loss: 0.33914999999999995\n",
            "confusion matrix: [[[384   0]\n",
            "  [116   0]]\n",
            "\n",
            " [[418   0]\n",
            "  [ 82   0]]\n",
            "\n",
            " [[100 249]\n",
            "  [ 24 127]]\n",
            "\n",
            " [[447   0]\n",
            "  [ 53   0]]\n",
            "\n",
            " [[475   0]\n",
            "  [ 25   0]]\n",
            "\n",
            " [[339  72]\n",
            "  [ 37  52]]]\n",
            "acc: 0.104\n",
            "f1 score(micro): 0.4242424242424242\n",
            "precision score(micro): 0.3861082737487232\n",
            "recall score(micro): 0.47073474470734744\n",
            "AUC score(micro): 0.5985899485940014\n",
            "hamming loss: 0.342\n",
            "label ranking loss: 0.33656666666666674\n",
            "confusion matrix: [[[334  30]\n",
            "  [108  28]]\n",
            "\n",
            " [[333   5]\n",
            "  [156   6]]\n",
            "\n",
            " [[ 64 226]\n",
            "  [ 20 190]]\n",
            "\n",
            " [[169 201]\n",
            "  [ 39  91]]\n",
            "\n",
            " [[386  57]\n",
            "  [ 51   6]]\n",
            "\n",
            " [[310  82]\n",
            "  [ 51  57]]]\n",
            "Kullback–Leibler divergence:  0.07486880607699026\n",
            "Euclidean distance:  0.15361646616715408\n",
            "MSE:  0.005036652605300524\n",
            "Chebyshev distance:  0.12604370790029545\n",
            "Clark distance:  0.3800501997564805\n",
            "Canberra distance:  0.827307927122582\n",
            "Cosine similarity:  0.9274275774254322\n",
            "Intersection similarity:  0.8516695946182499\n",
            "Kurtosis_signed_offset:  0.2106348136994221\n",
            "Kurtosis_abs_offset:  0.9433599528124046\n",
            "Laloss:  0.36636256955649127\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "4TmKMK72LVnv",
        "outputId": "8683508f-a1b9-42c2-8f7e-17ad0507b033"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "loss = history.history['loss']\n",
        "val_loss = history.history['val_loss']\n",
        "epochs = range(1,len(loss)+1)\n",
        "\n",
        "plt.plot(epochs[10:],loss[10:],'ro',label='Training loss')\n",
        "plt.plot(epochs[10:],val_loss[10:],'r',label='Validation loss')\n",
        "plt.title('Training and validation loss')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.legend()\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd7hU1dX/P+ty6SgqVeHSRCRoAKUpdjH2iAULIZYYG8afLcbXRBN9VRKNJhpeS4IxaiwRoykYNRYQQZMQEBGlKVKkKiBVOnf9/ljncM+de6beOVP353nmmZnTZs+ZM/t7Vtlri6ricDgcDkeqVOS7AQ6Hw+EoLpxwOBwOhyMtnHA4HA6HIy2ccDgcDocjLZxwOBwOhyMtnHA4HA6HIy2ccDjyjoi8JiIXZ3vbfCIii0TkhAiOqyLS3Xv9WxH5aSrbZvA5I0TkjUzbmeC4x4rI0mwf15FbKvPdAEdxIiKbAm+bAduAXd77K1X12VSPpaqnRLFtqaOqV2XjOCLSBVgINFTVnd6xnwVS/g0d5YUTDkdGqGoL/7WILAIuU9W3YrcTkUq/M3I4HKWBc1U5sorvihCR/xGRlcATIrK3iPxDRFaJyFrvdcfAPhNF5DLv9SUi8q6I3O9tu1BETslw264iMklENorIWyLysIg8E6fdqbTxLhF5zzveGyLSOrD+QhFZLCJrROTWBOdnkIisFJEGgWVnichM7/VAEfm3iKwTkRUi8pCINIpzrCdF5O7A+x95+ywXkUtjtj1NRD4QkQ0iskRE7gisnuQ9rxORTSJyuH9uA/sPFpGpIrLeex6c6rlJhIh8w9t/nYjMEpEzAutOFZHZ3jGXichN3vLW3u+zTkS+EpHJIuL6shziTrYjCtoD+wCdgSuw6+wJ730nYAvwUIL9BwHzgNbAL4HHRUQy2PY54L9AK+AO4MIEn5lKG78DfA9oCzQC/I6sF/Cod/z9vM/rSAiqOgX4Gjg+5rjPea93ATd43+dwYAhwdYJ247XhZK893wIOAGLjK18DFwF7AacBI0XkTG/d0d7zXqraQlX/HXPsfYBXgNHed/s18IqItIr5DnXOTZI2NwReBt7w9vt/wLMicqC3yeOY23MP4GBggrf8h8BSoA3QDvgJ4Gon5RAnHI4oqAZuV9VtqrpFVdeo6kuqullVNwKjgGMS7L9YVR9T1V3AU8C+WAeR8rYi0gkYAPxMVber6rvAuHgfmGIbn1DVT1R1C/AC0NdbPgz4h6pOUtVtwE+9cxCPPwHDAURkD+BUbxmq+r6q/kdVd6rqIuB3Ie0I4zyvfR+r6teYUAa/30RV/UhVq1V1pvd5qRwXTGg+VdWnvXb9CZgLfDuwTbxzk4jDgBbAPd5vNAH4B965AXYAvURkT1Vdq6rTA8v3BTqr6g5Vnayu6F5OccLhiIJVqrrVfyMizUTkd54rZwPmGtkr6K6JYaX/QlU3ey9bpLntfsBXgWUAS+I1OMU2rgy83hxo037BY3sd95p4n4VZF2eLSGPgbGC6qi722tHDc8Os9Nrxc8z6SEatNgCLY77fIBF523PFrQeuSvG4/rEXxyxbDHQIvI93bpK2WVWDIhs87jmYqC4WkXdE5HBv+X3AfOANEVkgIrek9jUc2cIJhyMKYu/+fggcCAxS1T2pcY3Ecz9lgxXAPiLSLLCsKsH29WnjiuCxvc9sFW9jVZ2NdZCnUNtNBebymgsc4LXjJ5m0AXO3BXkOs7iqVLUl8NvAcZPdrS/HXHhBOgHLUmhXsuNWxcQndh9XVaeq6lDMjfU3zJJBVTeq6g9VtRtwBnCjiAypZ1scaeCEw5EL9sBiBus8f/ntUX+gdwc/DbhDRBp5d6vfTrBLfdr4InC6iBzpBbLvJPl/6zngOkyg/hzTjg3AJhHpCYxMsQ0vAJeISC9PuGLbvwdmgW0VkYGYYPmswlxr3eIc+1Wgh4h8R0QqReR8oBfmVqoPUzDr5GYRaSgix2K/0fPebzZCRFqq6g7snFQDiMjpItLdi2Wtx+JCiVyDjizjhMORCx4EmgKrgf8A/8zR547AAsxrgLuBsdh4kzAybqOqzgJ+gInBCmAtFrxNhB9jmKCqqwPLb8I69Y3AY16bU2nDa953mIC5cSbEbHI1cKeIbAR+hnf37u27GYvpvOdlKh0Wc+w1wOmYVbYGuBk4PabdaaOq2zGhOAU7748AF6nqXG+TC4FFnsvuKuz3BAv+vwVsAv4NPKKqb9enLY70EBdTcpQLIjIWmKuqkVs8Dkcp4ywOR8kiIgNEZH8RqfDSVYdivnKHw1EP3MhxRynTHvgLFqheCoxU1Q/y2ySHo/hxriqHw+FwpIVzVTkcDocjLcrCVdW6dWvt0qVLvpvhcDgcRcX777+/WlXbxC4vC+Ho0qUL06ZNy3czHA6Ho6gQkdiKAYBzVTkcDocjTZxwOBwOhyMtnHA4HA6HIy2ccDgcDocjLZxwOBwOhyMtnHA4HA6HIy2ccDgcDocjLZxwOIqPLVvgd7+D7dvz3RKHoyyJVDhE5GQRmSci88OmdxSRxiIy1ls/RUS6eMtHiMiMwKNaRPrG7DtORD6Osv2OAuXOO+Gqq+CfuZrWw+FwBIlMOLy5mh/GJmnpBQwXkV4xm30fWKuq3YEHgHsBVPVZVe2rqn2xyVwWquqMwLHPxiZxcZQbs2fD/ffb608+yW9bHI4yJUqLYyAwX1UXeDN9PY/NhxBkKPCU9/pFYIg3HWSQ4d6+AIhIC+BGbEa33DFzJgwdCu++m9OPdQRQhauvhj32gJYtnXA4HHkiSuHoACwJvF/qLQvdRlV3YvMHt4rZ5nxsmk2fu4BfYXMVx0VErhCRaSIybdWqVem3PpZ//APGjYOjjoJLLoEvv6z/MR3p8cwz8M47cM89cPDBTjgcjjxR0MFxERkEbFbVj733fYH9VfWvyfZV1TGq2l9V+7dpU6e4Y/osX253uT/+MTz3HBx4IPznP/U/riM11q6FH/4QBg2Cyy6DHj2ccDgceSJK4VgGVAXed/SWhW4jIpVAS2BNYP0F1LY2Dgf6i8gi4F2gh4hMzGqr47FsGVRVwc9/bm6rLVvgpZdy8tFlz7x5cM45sGYNPPooVFSYcKxYARs35rt1DkfZEaVwTAUOEJGuItIIE4FxMduMAy72Xg8DJqg3JaGIVADnEYhvqOqjqrqfqnYBjgQ+UdVjI/wONSxbBh08T1vPntCmDXz1VU4+umxZuxZuuMHcUtOmwW9/C4ccYut69LDnTz/NX/scjjIlsvk4VHWniFwDvA40AP6gqrNE5E5gmqqOAx4HnhaR+cBXmLj4HA0sUdUFUbUxLZYvtw7MZ5997A7YEQ1bt0KfPibYl11mKbjt2tWs94Xjk0/g0EPz00aHo0yJdCInVX0VeDVm2c8Cr7cC58bZdyJwWIJjLwIOjrc+q+zaBStX1lgcAK1aOeGIkkWLYMkSszKuvLLu+v33BxEX53A48kBBB8cLhi++MPHYb7+aZfvs41xVUbJihT0fcED4+qZNoVOnzIVj7Vrb/733Mtvf4ShjnHCkwvLl9uwsjtyxcqU9t28ff5v6ZFbNm2cWzeTJme3vcJQxTjhSYZmXDBYmHBbLd2Qb3+LYd9/42/jCkclv4B//s8/S39fhKHOccKSCLxyxrqqdO2GTq3wSCStWQOPGsNde8bfp0QPWr4fVqzM7PsCCwsi9cDiKCSccqbB8OTRoAG3b1ixr5Q1wd+6qaFixwqyNOhVoAvjxj0zcVb4rzAmHw5E2TjhSYdky87U3aFCzzAlHtPjCkYhgSm4mxwf4/HPYsSP9/R2OMsYJRyosX147vgHmqgKXWRUVqQhH587QsGH9hKO62sTD4XCkjBOOVAiOGvdxFke0rFyZXDgqK208R6bCseee9toFyB2OtHDCkQrLltUOjIMTjijZutXGWSRKxfXJNCV3xQo4zBtf6uIcDkdaOOFIxubNsG5dXYtj773t2bmqso8fuE5mcYAJx6efmsspVXbtskGd/fpZ5pYTDocjLZxwJCNs8B+Yb33PPZ3FEQWpjOHw6dEDtm2zwXypsmqVCU2HDtC1qxOOYuWrr+Dmm61StSOnOOFIRtgYDh83ejwa0hUOSM9dFTx+t24uxlGs/PGPcN99MGlSvltSdjjhSEY8iwNcvaqoiFo4gq6wbt3M4nAVAIqPV16xZyf8OccJRzKcxZF7Vq60yZpSmbmxfXto0aJ+FseGDcVxA/Dcc/DrX+e7FYXBxo02jTDA/Pn5bUsZ4oQjGcuWQfPmNambQZxwRMOKFTb3RnDAZTxELCU3nTiFLxzt29u+UBxxjjFj4JFH8t2KwuDNN23gZsOGTjjygBOOZPiD/8JKXzhXVTSsWJFaKq7PvvvWuJ9SPf7ee0OTJmZxQHG4O5YsyawuVynyyivQsiWcdFJx/HYlRqTCISIni8g8EZkvIreErG8sImO99VNEpIu3fISIzAg8qkWkr7funyLyoYjMEpHfikgKt6X1IGwMh0+rVjbeYNeuSJtQdqQyajxI+/bpC4cvTF272nOhWxzV1bB0qRV1LPcSKdXV8OqrJho9e5pwpJOOXQxs3259S4ESmXB4HfrDwClAL2C4iPSK2ez7wFpV7Q48ANwLoKrPqmpfVe0LXAgsVNUZ3j7nqWofbPa/NsSZQTBrhJUb8WnVyoKq69ZF2oSyIxPh+OKL1APcweM3b25usUIXji+/tM4EnJX7wQd2o3DaadC9u6Vj+0kspcI998CAAfluRVyitDgGAvNVdYGqbgeeB4bGbDMUeMp7/SIwRKSOT2i4ty8AqrrBe1kJNAKiS4dRtQsynsXh6lVln127rJNMVzh27Ej9Di1WmGJjJFOmmAurkDqj4DiVcndXvfKKuY5POaUmRlVqcY5Fi2DhwoLN9otSODoAwVFZS71loduo6k5gPdAqZpvzgT8FF4jI68CXwEZMcKJhzRq7m0lkcfjbObKDPzgvXeGA1NxVqnXrYMWO5bjrLvvTvv9+6m2IGiccNbzyCgwaZFl33bvbslITjk2b7H9QoPP9FHRwXEQGAZtV9ePgclU9CdgXaAwcH2ffK0RkmohMW7VqVWYNSDSGA2osDiccmfPf/9ZkOUF6Yzh82rWz51SEY/16q4UVKxxLlpgraPbsmvEBCxem3oaoCVbwLefr7YsvYOpUc1MBVFVZZlWpBch9wShQN3iUwrEMqAq87+gtC91GRCqBlkDwX3EBMdaGj6puBf5OXfeXv36MqvZX1f5tUhkPEEaiMRxQY3E4V1XmfPvb8MMf1rzPRDjSsTjCjt+tm1kiixfbOImmTS3jqpCEw1kcxmuv2W/lC0eDBpbgUIoWB5SlcEwFDhCRriLSCBOBcTHbjAMu9l4PAyaomlNPRCqA8wjEN0SkhYjs672uBE4D5kb2DcLmGg/iXFX1o7raXFNvvlmTFRMcY5Eq2RAOgH/9C55+Gr73PVtWaMLRqZO9Lnfh2G8/6Nu3Zln37qUrHOvX57cdcYhMOLyYxTXA68Ac4AVVnSUid4rIGd5mjwOtRGQ+cCMQTNk9GliiqsF0l+bAOBGZCczA4hy/jeo77HZVxbv7bdnSRjiXi8Wham6ebLF+vR1z9WqY4SXNZSIcLVtaldsvvki+bdjx/QDrbbdZkP2GG+wuttCE44ADLAusnG9U/vUvOOaY2uOq9t/fXFUFGkjOiAK3OCqjPLiqvgq8GrPsZ4HXW4mTTquqE4HDYpZ9AeQuR23ZMgvANWoUvr6iwgaSlcsfeexYGDnS/O177FH/4wWzoN58Ew49tPbgvFQRSX0sR5jF0b69fd7SpXD22XYH27WrFc9TTTzvea74/HMbt/Dpp+VrcSxbZr/RoEG1l3fvbiVIVq2Ctm3z07ZsU+DCUdDB8bwTNvNfLKVQduTzz1Or9fTBB3Yhz82SdzBWOCC1mf/CSEc4mjatXUKmoqJmIOBNN9lz167WGRXCIKwdO6zdVVXQunX5CseUKfYcKxy+xVhKAfKNG+253FxVJUGiMRw+pVB25NprYcSI5NstXWrPc+Zk53P9TnnAAHj3XZs0K93Bfz7t2qUuHPvuW9eKOOIIOPFEOPxwe+8LSSG4q5YvN8vHF45iv1HJlClTLIMqGN+A0kvJra6Gr7+2187iKELOPhvOOivxNqVgcSxeXDvdMx6+cGTb4jjvPBsvM3ly5sKRqsURz6J57DELvPoUknD4v02nTs7iOOSQum7MLl3MaiwV4diypSZe44SjCLntNrjsssTblIJwrFxpnVGymltRCcfQoRZHevPN+gnHqlWwc2fi7RIdvyLwdygk4fBTcauq7HorR+HYuROmTavrpgJLjKiqKh1XVXDQnxOOEqXYXVV+iY/q6sQdkmp0wtGhAwweDC++aJZHOhlVPu3b12RoJSJVYWrZ0oL0hSYcrVuXZ6HDWbPMfRMmHFBaKblB4XAxjhKlVSsLZPkF6IqN1atrxlAkcvWsXm3fsWVL+4Nmo+Nau9YsjaZNLb6weLEtz9TigMTfYcsW+yOmevxCScn9/HMTsRYtTDiguG9WMiFeYNxn//1LUzicxVGiFPvo8WBHm2gchG9tHHeciUY2qsmuXWsdogh861s1y6MSjnTHiBSKcCxZYtYG1Fxv5eaumjLFvrufQRVL9+7mMi7QjjYtfOGorCzY7+OEo74Ue4XcdIXjhBPsORvuqq++MuEAC3r65zLTrCpITTjSsTgWLcr/XA9B4fAtjmKPq6XLlClmbcQbU+NnVpVCnMNPxd1vP+eqKlmKvexIsKNN1OlGIRy+xQFWc2jIEHtdSMKxbVt6k0RFQbDciC8c5WRxbNhgxSfjuamgtMZy+BZHx44Fa3FEOnK8LCgV4aisTG5xVFband1++2VPOIJuoxtusEB5JqPSmze3/bIpHF262PPChcnH80TF5s12bZWzq2rqVEt8SEU4SiHO4QtHVZVlkhVK9YIAzuKoL6XgqmrRwu5ukgnHfvuZZdCzZ/YtDrDBdw88kPmfxJ8JMB7Llpn4+XftySiElNxgRhWUp3D4gfGBA+Nv07y5/f6lJBwdOlhCSjbrw2UJJxz1pdgtDj89NdkAuqVLTVzAhGPOnPoXlVu7tkZ4s0Gi77BzJ7zwgt21VqR42fsWx6JF2WhdZsQKR9Om5VfocMoUOPDA2jcZYXTunNpA1kIn6KqCgoxzOOGoL82bWxmEYv0jr1xpHW67dsktjqBwrF+fWjXaeFRX2zGSdQbpkEg4/vpXsxyCc38ko2lTO2Y+LY7gqHGfchoEqAr/+U9iN5VPVVXteUuKlU2bzLL343YFGOdwwlFfROyPXMyuqmTC4Q/+84XjG9+w5/q4q/yS6tkUjnj1qlTh/vvND37GGXXXJyLfKblLltg1Fiy2WS5lRzZtguuuswGqgwcn375TJztfxV5efdMmcx/7/w0nHCVKMZcdCQpHvJId69ZZkDZocUD9hMMfNZ5ti2Pduro+4ffesylqb7zR7uTSoRCEo1272qX9y6HQ4VtvwTe/CQ89BP/v/8EllyTfp6rKBnkW602cz8aNJhwtW9p756oqUfbZJ7U/8vbtVlajUO6I/JHU7dsnLtnhp+L6d70dOpiLrhCFA+wONcivfmW/USqdTyxdu1rnnawGVlR8/nltNxWUvqvqiSdsQGjjxjYnyujR9joZfhyo2N1VvsWx11723lkcJUqqrqo//hHOPddKiBcCvmvKtziCy4L4wuFbHCI1AfJMiVI4gu6qTz+Fv/8drr4amjVL/5hdu1o9ryg7o+nTzdoLIzj4z6fUXVXPPGPu0Bkz4MgjU9/PP0/FHiAvd+EQkZNFZJ6IzBeRW0LWNxaRsd76KSLSxVs+QkRmBB7VItJXRJqJyCsiMldEZonIPVG2P2VatbJUzw0bEm/31lv2/N//Rt+mVPA72KBwhMUIYoUD6p+SmyvheOABS174wQ8yO2bUKbk7dsDRR8OoUXXXqcYXjlItdLhtm00Pe+KJ6c0CCaVlceyxR3kKh4g0AB4GTgF6AcNFpFfMZt8H1qpqd+AB4F4AVX1WVfuqal/gQmChqnqTUnO/qvYEDgGOEJFTovoOKTNsmInGccfVdZP4VFfDhAn2eurU3LUtEUHh8DvdeBaHSO2Bcz172p2dP+FMuuRCOFavhiefhO9+N7OKuxC9cMyZY+cwbPzB2rW2Lkw4oPh9+WFMnWoxqmOPTX/fdu3sJqEUhKNFC8vqq6wsuxjHQGC+qi5Q1e3A88DQmG2GAk95r18EhojUGf013NsXVd2sqm97r7cD04GO5JuTTzZ3yJw5NpNcWCfz0UfmjmjatDCFI5mrqn17+1P6+JlVqUw5G4bf6WVTOPz5pv3v9cgjFsdJJwU3lqoqC6hHJRzvv2/PfmXgIP74EX88iU8pDwKcONGejzoq/X0rKswqLhXhEDGro5wsDqADEPwFl3rLQrdR1Z3AeqBVzDbnA3+KPbiI7AV8Gxgf9uEicoWITBORaavi+Y+zyamnwvjxFiQfPLhuRzPea+b3v2+VZQshK2blSrs427atucOJJxwdY/S5vplVwZLq2aJhQ+tUV640wXjoITjtNOgVa+imQWWlJQNE5TefPt2eFy2qmzThX0O+1eNTyoUOJ06E3r1rxDFdSmEshy8cUJbCUW9EZBCwWVU/jlleiYnJaFUNre+tqmNUtb+q9m/Tpk0OWouVzJg82TrF+++vvW78eOjRo2Yq2mnTctOmRKxYAW3aWOcoEn8cRJhwdO9u+8ybl/xzZs+uG0gPllTPJn7ZkT/+0Sy8H/2o/sds2zZ+8Lq++BbHpk017jufZMJRahbH9u0W38jETeVTVVX8wXE/HRcsJbfMXFXLgKBztqO3LHQbTwxaAsHbqAsIsTaAMcCnqvpg1lqbLQ46CM4/3zouvzzy9u3wzjtW/bVfP1tWCO4qfwyHT7xaT2HC0bix3Ymn4sK58kq44oray7JdbsSnfXtYvtxScPv3t8BzfWnTJhrh2LXLMof8GEasu2rhQrvj9IOkPqXqqpo61SzFY47J/BhVVZaokmwa5EJFtewtjqnAASLSVUQaYSIwLmabccDF3uthwARVs9dFpAI4Dy++4SMid2MCc32Eba8fV19tP/4zz9j7//7XgpwnnGB3EAceWJjCETZ6fMMGe8QKB9TMV5GMJUvqxkJiCxxmi/btrbbRp5/CTTdlx6KJSjjmzrWO8uyz7X2YcMRaG1D89dHi4cc36iP2VVU25qY+5XDyyZYtJh7lKhxezOIa4HVgDvCCqs4SkTtFxK/78DjQSkTmAzcCwZTdo4ElQVeUiHQEbsWytKZ7qbqXRfUddvPssxagrKiw52efTbz9wIFmWTzyiF0E48dbB+ab4AMGFK5wxLqqlnlGYphwdOmS3OJQNZfYl1/WWGAQrXCoWtvOOSc7x4xKOPz4RrrC4Rc6LDWL4513bLR4qtWLw8hmSu7OndamXOIXOCxX4QBQ1VdVtYeq7q+qo7xlP1PVcd7rrap6rqp2V9WBQZFQ1YmqeljM8ZaqqqjqN/x0XVX9fZTfgWefNTfL4sXWIS1ebO8TiYeIWR0ff2wxj/Hj4dBDa1wzAwZYZ7os1nMX+Mx997XjpCJUmaAa7qpavbr2KOmwMRw+Xbvad0g03/q6dTXrgyITlXD42WE33GCxm2zQpo3dCWaaehyP9983ERg82AYnBq03VXsfJhxQeqPHd+yw0jD1cVNBzSj7bAjH44/bzd6nn9b/WKniC4c/J00ZxjhKg1tvtTpNQTZvtuWJuOAC6xh/+Uv4979rZs4DEw4ItzqefdYyr/w7/1SEKhP8Dj3W4ogtO5JIOLp0sfEpif6k/uRJUHt2tqiE48QTLQHh0kuzd0w/uSLbVsf06dC3rwlcly61LY6VK208QzzhKLXR49Om2f+qPoFxyK7F8Y9/ZO9YqRJmcXz9dcEN9nTCkYx4GRr+8nhurGbN4Hvfg1desTt4f1pUqOkswoTj1ltt9GyQVIQqXYJjOHzCRo/7whE2A57fqSWKcwSP5QtHFCXVffr0gb/8peaPlw2iEI7qavjgg5pkic6dawtHvIwqn1IrdJiN+AbYNdWsWf0zq7Ztqxmwm8t4SZhwQMFZHU44khFbYC64PJkb66qr7LlRIxsY6NO0KRx8cLhwhA0Eg+ynGCYSjuAfZelS6zjDyj8Ep1aNR9DiWOB5IqMoqR4lUQjHp59aJ3HoofY+XeEoNVfVxImWkVjf1HmR7IzlmDy5xtOQT+Eo0Aq5TjiSMWpU3eJ4zZrZ8mRurAMOsADtaafVPcaAATXzCcceO4xMS2bEI0w4wsqOhKXi+nTsaKOqU7E4unevsTiiKDcSJVEIhx8YD1oca9bUdBzxRo37lJKrqrraxm9kI3UasiMcr71mN3yVlbkVDj+BJNbiCAbIr7kGLrood20KwQlHPHwX1IUXmoXQqpXdzXTuDGPGwIgRyd1YAH/+M7z0Ut1tBgywDjTo958zx4KwsUFdEctLz2Zw1rcEgvWnwlxVixfHF47KSrO8ElkcK1eatXLooTXfNYpyI1EShXC8/76NhfFLt3TubM++1bFwoQl5vJH1pVTo8PPPTTD79s3O8fwJnerDa69ZoD7ZzJjZJp6rKigc48bZNMhbtuSuXTE44Qgj1gW1Zo39SE8/bXeCI0bYdoncWD4i4WMJwgLkv/iFdRSjR1tH4gvVT35i6aw//WlWvh5gHXrjxjWmMNQtOzJrlj0Szb7WpUtii8Of03z//e187txZfBbHHnvY3We2LY4+fWrqf/mWRVA44rmpoGYsRykUOpw9257rUxomSFWVXd+Jsv0SsXix3cSdckrhCcdXX5kobttm7rQ84YQjjGQuKN8aWby4rij4bqxkHHSQ3YnffLONNL/tNnjuOYuLjBxpnXF1tT3ffbcte/BBm385G/ipuMH2i9QePf5//2ficlmCoTLJZsjzP6dbNxONJUuKTzhEsjuWQ9WEw49vQLjFEc9NBaVVdsQvR+NbX/WlqsrOcbx092S89po9n3xy/oQjmI4LNTGODz+s2faNN3LXrhiccISRyAUVtEbALlC/8w26sZLRsCE8/DAccohZHaNGmZDEq+R67732h7j00rpZV5kQO4bDx5LF5GUAACAASURBVB8EuHatWVjf+U7iAVlduphVETtdq0/Q4gALkBebcEB2hWPBAusI/PgG2G/RqJHdKOzcaddaIovDdytm2jkWErNnWz2wTAsbxlLflNzXXrP/cs+e+RGOioqaZJRYi8MXjoMOgjffzF27YnDCEUYiF1SYNaJqF1rQjRVLWNrupZeav3LBAguKLVoUnvYKdgcyZozdnd11V2bfK0gi4fjiC5u+c/Nmm+85EX7nFi8bzP8cXzg++6xGOKKoVRUV2RQO3z0ZtDgqKqzDW7zYEhJ27UosHL1727NfJLGYmTMne24qqJ9wbNtmA3ZPOaWm8OeXX+ZuuudgSXWw/71IbeFo1876mZkz81ZaxQlHGIkyqVIJiMeSyujzFi2Sl1o46SSbN/uee+o3bSskFo4VK6wk+ZFHmkWUCN+dEhbn2LbNfLL77msFERs2rBGOxo2zW1I9arIpHOPHmwvC7/x9/JTcZKm4YKLbvXthlK6pD6pmcWTLTQX1E45337UklFO8+eHatbNYSa7KfgQLHILdUOy5Z21XVZ8+Nic71MwqmmOccIQxYoTd3QcD1L4LKpWAeCyZjj4P4557LPbx5z+nv6/Pjh3mGw8TjvbtLRlg4UK49trkx0o0Q15wTvMGDWxb31VVTG4qyJ5wqJqL4fjj62bP+XGzVIQDrCZaoUxDnCkrVlinmE2Lo0ULu74yEQ4/Dff44+19ognOoiBYUt3Hr1e1Y4clq/TpYzd0++yTN3eVE454jBhRO0Dtu6ASWSPxyMRKiUe7dtZhvPpq+vsGP1e1dipu8PhgFsKZZyY/1r771vjmY4lN+d1//xqLoxiFY+PG1ONLy5ebVbl8ee3ln35q4uDfMQbp3NnO2dy5Na6rRAwYYDGO2M8oJrIdGPfJdF6Od96xeXX8zjvbwvGLX9iUC/GItTigRjjmzTPrp08fuxEbMsSEI1dutABOONIlkTUSj0yslESceqrdaWZ6B+y7yE48se46/49y9dW1p4qNR0WFnYMwiyN2kGG3bsUtHJDaOd+xA847Dx57zDLTgvh3iGHn3s+smjTJOr5k53/gQHsuZndVtlNxfTIZBKhqov3Nb9Ysy7Zw/OY3VjU7HomEY8YMe++PdznxRLtp8M9hDnHCkQnxrJF4ZGKlJOLUU+0i/+c/09931y6r+vmtb4W7Qo4/Hn7wAxOOVIk3lsO3OHzh2H9/m9tj/vzSFo4f/cgqvXbqZHOyBCcVeuMNO+9+skAQXzimTUvupgJzVzRoUNzCMWeOxXuyXRkhE+FYscI6bn9aZMiucGzZYsf56CPrO8IIEw6/Qu6HH1ps8MADbblvtebBXeWEIxdkYqUk4tBD7YLOxF01fryZ8PHGZuy9twXGY2edS0S8sRzBOc2hprNcvLh0hWPsWLurvO46uO8+y5B6+21bt2OHvQ6zNqBGOJJlVPk0bWp3x8Uc55g926yNbE8hXFVliRnpVFvwp0H2O2awFOEGDbIjHP7N1ebNtStGBNm0qWYMh49vcXz4oaXh+rGxzp2trJETjhImXSslERUVlvXx+uu1585Ihccesz/D0KGZf34sXbpYhxr7J12xwjLFfJdL8C67FIVjzhwriT94sJXTP+MMu1v0fdpTplicJCy+AVbapcL7S6YiHGDuqqlT8+LnzgrZTsX18eNDfnXnVAgTjooK++2zIRzBm6uZM8O3SeSq8jOqgnzrWxaXyXSUfIZEKhwicrKIzBOR+SJyS8j6xiIy1ls/RUS6eMtHeLP7+Y9qEenrrRslIktEZFOUbS94Tj3VYgVTpqS+z5dfwt//bgXSGjfOXlvilVdfubJ2AD7YGZaicFx9tbkgX3jBEgaaNLGqAC+9ZB3Cm29aR+Rn7MTSsKElJUDqwjFggHUq8+en/l0KhTVr7JrMdmAcMkvJnTvXfj//N/DJ1iDA4P8jXeFYv97OVaxwHHec3bDFO15ERCYcItIAeBg4BZvqdbiIxN5afB9Yq6rdgQeAewFU9Vl/hj/gQmChqnqRIV4GBkbV7qLhxBPNhH7lldT3efppc5ckKiGSCfHGcqxYUdt33axZjZAUm3DstZed73jC8e67Vhr81ltrdzwXXWSuib/8xeIbAwYk/u6+uyodiwOK013lZ1QVksVx4IE1Vp9PtoRj4UK7YevRo3bpEB/V8HTcYD25WOHo1s2ecznZFNFaHAOB+aq6QFW3A88Dsf6RocBT3usXgSEidZydw719AVDV/6jqCsqdli1tgF6qcQ5V+P3vzY2S7T9qvLEcsRYH1Lirik04KirM7RZPOEaNMqvk8strLx882P7cDz1knXu8+IZPusLRq5cJcjEGyP1soCgsDl+80+lQfeGIJZsWR5culhUVZiFs3Wqu7DCLwydWOPzK1ekIZBaIUjg6AMFfbam3LHQbVd0JrAdiC9acD/wp3Q8XkStEZJqITFuV7Sk/C4VTT7U7l1TqFb33npni2bY2wILfTZvWnTM7bHS6f4dUTOVGfOINApw61TLcfvjDutlzImZ1TJ1qnUK8+IZPv35WdibVLKPKSkuWKFaLo1mzzNPSE9Gkif1eqQrH1q12/SYSjvrGkfyKx3362OsNG2qvj62M6+MLR6dOdW+4Wrc2t2gJCUe9EZFBwGZV/TjdfVV1jKr2V9X+beo7q1ihctpp9vzwwzZe4LvftTtaP4vH5913zde+995w7rnZb4eI3UkFLY6vvjK3WKlYHBBfOEaNsu8zcmT4fhdeaM8tWsBhhyX+jGuvtUGCse6SRAwcaNPQFtvcHLNnW+prOt81HdJJyf30UxOGeMKxdWvNJEuZ4lc89kvNfPRR7fXxhMN3VcVaG2DnrkOHnBe7jFI4lgHBoa8dvWWh24hIJdASCE6kfAEZWBtlQ69e5tr4xS+sw5kwwf6Mxx9vndUXX8CvfgXHHmt3dm+/nd25uIPEjuUIm2HQbzOEj1ovdMKEY+ZMSzi47jqrKRRGt25w+ukwbFjyQX0NGsSfBTIeAwdax/Zx2vdX+SWqjCqfqqrwO/GpU+Gpp2ov8zOqgmM4fFIZy5FsUqUNG+xmqmvXGuGIdVclszjChAPMXVVCFsdU4AAR6SoijTARGBezzTjgYu/1MGCCqtmDIlIBnEcgvuGIQQT++lfL2lmyxEaRfvqpze0xdqz9cW66yVJvp02Lf+Flg549TbT8iz9shkGAs8+2TLCwAXCFTphw/PznlnefrIrwyy9bxeEoCJsUrBAZPdomJVu50u7elyyJJr7hE8/i+OUvzWUbdBX5wtGjR93tkwnHqlV2bTz4YPy2+DdVXbtau/baK75wxI7j6NLF3MHx4mOlJBxezOIa4HVgDvCCqs4SkTtF5Axvs8eBViIyH7gRCKbsHg0sUdUFweOKyC9FZCnQTESWisgdUX2HouCQQ6wz9oNkTZta2fWPPjLBeOABePHF2pkZUXD22XbX+/e/2/t4FkdFRU0mULHRpo2lQPsuoVWrLPX2qqvyG7Pp2tXG5sS6KAuJJUssBvSLX9RMyQzRWhwdO1oaa6yL6ZNPbPxTcCKkefNs++bN6x4nmXBMmGApsbfeGt81FpxDXsSsjtjMqngWR6tW9tlHHRV+bF84cjiWJ9IYh6q+qqo9VHV/VR3lLfuZqo7zXm9V1XNVtbuqDgyKhKpOVNU6DmFVvVlVO6pqhfd8R5TfoV6EzcGRKw480CroXn999kflhjF4sAXvnnvO3sezOIoZP1a2xvOmvvWW/VmjiBulgx+Af+GF8DTPQuDBB+1cjR9vbfWzAQ8+OLrPDBvLUV1tVjnUTmWfOzc8vgHJhWP8eOvsVeGGG8K3ia143Lt33dIjvsCl607u2NGKb65Zk3zbLFHQwfGiJpU5OEqJigq44AK7i1u92iyOZs2ii6nkg9hBgG+8YZZGcEKmfPHTn1qA/vrrC28U+bp1VmLn/PMt/jZmjJXX/+c/bU6RqAgby7F0qcUjGjc24aiutvM1b154fAMsc0kksXAcf7y5iF96KbyG3MKFZs34sxz26WMWRjAuGM/iSEYeUnKdcERFNufgKBa+8x1zAbz4Ys2UsbmwdnJFUDhUTTiGDLGAdr7Ze29zUU6caIMNc0mwiGMYv/uddYo/+lHNso4dbWKyKAmzOD75xJ4vush+x6lTTRA2bIhvcVRWmniECceiRSaCQ4aYK65HD7jmmrpTKS9aZNaG/3/wA+RBC9EJhyOrc3AUC717m8/6uefizzBYzASFY/ZsS0ZINqAvl1x+ubl+brop/hzw2Wb9euu4rr46vOLrtm1W9PGEE2rKgeeK/fazjjpMOK691qzkf/wjvEZVLPEGAU6YYM9DhpgV8/DDVsDwl7+svZ0/hsPnoIOsbcEAuRMOR9bn4CgGRGD4cJg82f4QpSwcfkXSZAP6ckllpSVDLFpkz7ngnXfsJuHRR21a49iim889Z9Zn0NrIFY0aWYcfKxzNm1vHPXhw/YVj/Hhb5wf5TzgBzjrLYjp+EoVqzRgOn+bNrbJtrHCIpD+lcrt2ZvU64SgBsj0HR7EwfLg9r1lTWoFxMP+0iAnHG2+YW8IvEVIonHCCVeX9+c9t3EDUTJhgo7Rvv91qoV1wgVVq3bXLrOv77zdLNF8CGzuW45NP7HcTsbE1M2ZY59+0aeIZF8OEQ9W+//HH13bJXnyxZd9NnGjv1661wHdsGZnYzKpNm0xQ0h0Q2aCBWVdOOEqAbM/BUSzsvz8MGmSvS83iaNDAguFLl1qnUEhuqiB33mmd0JNPRv9ZEyZYzbQ77jAr56WXrBNr0sSu+dmz4X/+J3+xrtixHL5wgAkHWJt79EjcYYcJx5w5Zm0NGVJ7+YknmgC8+KK9jzeHfO/eNTNiQvhcHKmS47EcKQmHiDT3BuQhIj1E5AwRSWFe0TInm3NwFBO+1VFqFgeYu+of/7DMnEIVjj59zA3z29/Gn2kuG6xaZSmlfpn4668319Spp8LNN1tQfPJkS5rIF75wqJoltHBhjXD06mXuo127ErupwITj669rzzkzfrw9xwpH06YmSn/7mx07OIYjyKmn2rM/ij2sMm6qFKJwAJOAJiLSAXgDK3X+ZFSNcmRAPseMxDJihLkmjj46f22IijZtbF6Eykor5VKoXH21jVfwO7co8F0xwflFhg+3iatGjbL08yOPjO7zU6FjR7uTX7/esp+qq2uEQ6Sm3lsqwgG1rY7x462cTKwgAJxzjl0n774b3+Lo18/Oz+jRJjBhc3GkStggwI8/NiFPVg4lA1IVDlHVzcDZwCOqei5wUNZb48iMQhsz0rq1xQCizNHPF36A/PDDM3cr5IJhw+x3eOSR6D5jwgQ7B/36RfcZ9SU4lsPPqAqWFfHdVclKn8QKx86dJpyx1obPKaeYu+7FF0049t47vHrDddfZ+pdfrr9wfP21CaTPc89ZvCWCcT0pC4eIHA6MAPzhlgWQvO4AynPMSL7whaNQ3VQ+jRvbNLbjxsV3YXz9Nfz4x3ajkQkTJsAxx9TMgV2IBMdy+MJxwAE16086yeq6nXNO4uPECscHH1gnHW82xxYt4OSTbUzNggXhVgnAmWdapuVvflN/4YDav/XkySbq6RbNTIFUheN64MfAX716U92AAi6MU2aU45iRfOELRyGl4cbjyivtbnPMmPD1jz4K99xjBf/SvSv17+DjdZyFQlA45s2z3y9Y0l8EzjvPUncTERSO996zeA4k/v7nnGNjfd5+O/7EXJWVViBz4kQLtmdLOLZutTla4tW3qicpCYeqvqOqZ6jqvV6QfLWqXhtJixzpU45jRvLFt78N3/se9O+f75Ykp2tXC8A+9ljduTq2brWS+3vvbTW3nk+zCLVfULHQhWPffS3u5wtdWPXbVGjb1p5vu83iEp9+aoLsLw/j9NOtjP62bfEtDjDLsFkzswCzJRxTp1oyQD6FQ0SeE5E9RaQ58DEwW0TyMKLHEUq5jhnJBwMHwh/+UBhlRlJh5EhLGR07tvbyJ5+sWd6/P9x4Y23/eDImTLBxLd/8Zlabm3UqK008fFdVpsLRsKGlmjdoYIK7cGHdaYJj2WuvGss00VTAe+9tgych87iZX97HF47Jk+35iCMyO14SUnVV9VLVDcCZwGtAVyyzypEp2cyCKtcxI47knHyyld6/5pqaEdI7d8K999p4mxNOsLTdL7+0u+lU8Ae+HXdcdLP3ZZOqKhtPsnJl5sIB8P77llp7443h5dfD8GMnyeaQv9Zz4ATnF0+Hhg1t3FRQOA46qKaoYrZR1aQPYBbQEPgzcIy37MNU9i2ER79+/bSgeOYZ1WbNVO0vaI9mzWy5w5FtFi1Sbd1a9cADVdetU336abvm/v73mm2uuUZVRHXq1OTH++wz2//hh6NrczY591zVigpr80sv5fazt25VHT1addu25Nu++abqypWZf9aAAaonnaS6c6fqHnuoXnVV5sfyAKZpSJ+a6u3C74BFQHNgkoh0BjYk3MMRH5cFlZxCGpdS7HTubKOjP/vMBuP94hdWDNFPRQW4+24LAP/kJ8mP5xf2K/T4hk9VVc1AyPpYHJnQuLEFv5MF38GsPz8Inwn+WI4PP7TBhBHFNyD14PhoVe2gqqd6QrQYOC7ZfiJysojME5H5InJLyPrGIjLWWz9FRLp4y0eIyIzAo1pE+nrr+onIR94+o0WKsG63y4JKTKGNSykFjj4a/u//bAKl2bNNIIJuppYtbVa+iRNrj44OY/p0c6kkGzRXKPiBY5HinLI4VXzh8OMb+RYOEWkpIr8WkWne41eY9ZFonwbAw8ApQC9guIjEzhP5fWCtqnYHHgDuBVDVZ1W1r6r2xWIpC1V1hrfPo8DlwAHe4+RUvkNBUZ8sqHK4E8/UIiuHc1MfrrrKSq4fdVT4rIUnnGDZV37HE49Zs2rKghcDfkpup07pV54tJvypcl97za7/REUb60mqrqo/ABuB87zHBuCJJPsMBOar6gJV3Q48DwyN2WYo4BVq4UVgSIgFMdzbFxHZF9hTVf/j+d/+iAXsi4tMs6DK5U48E4usXM5NfbnvPpg0KXzQ3pFHmkvlrbfi769qpSwOKqLCEX4Hmms3Va7xLau33orU2oDUhWN/Vb3dE4EFqvq/QLck+3QAgjO3L/WWhW6jqjuB9UBsGsD5wJ8C2weHwYYdEwARucK3kFb5U33mglTuejPNgiqX2EgmFlm5nJsoadbMxCORcHzxhZVrd8JRePjCsWtXwQjHFhHZXa1MRI4Asl85KwYRGQRsVtWP091XVceoan9V7d/GH+0bNenc9WZSObdcYiOZWGTlcm6i5oQTLLgab37tWbPs+eCDc9em+tK+PQwdauU9ShlfOKBghOMq4GERWSQii4CHgCuT7LMMCDrZOnrLQrcRkUqgJbAmsP4CaqwNf/vA2Qk9Zv6I+q436hHihRIjyMQic6Pns8MJJ9iznzkViy8cxWRxVFRYiXP/u5Uq++1nz23aRJ+4EJajG+8B7InFGACuT7JtJbAAGyzYCPgQOChmmx8Av/VeXwC8EFhXgYlCt5h9/gscBgg2GPHUZO3O2TgOkdpjM/yHSHaOHzb+w//Mzp3rNw4kqrElzzxjbROpfxuTfY4bG1N/du5U3Xtv1UsvDV9/+eWqrVqpVlfntl2O1GjfXvXss7N2OOKM48h4UB3weQrbnAp8AnwG3OotuxM4w3vdBBtUON8ThG6BfY8F/hNyzP5Y2ZPPMMtHkrUjZ8LRuXO4cHTunL3P8DvioGiEdZTpdthRtD1KoQt+hv89W7WyR9QiVeqcc45qVVW4OAwerHr00blvkyM13n1XdcGCrB0uCuFYkum+uX7kTDhyedebqKPPpB1RWEvx2pitc+OsjGh49FE7l/Pm1V5eXa3asqXqyJH5aZcj58QTjvoUmsn+7CDFTi5rRiUKBmcSa4kiRpAsMF3f+I/LpIoGvzBfbHbV8uU2TqCY4huOSEgoHCKyUUQ2hDw2AvvlqI3FRa7mGU/U0WeSYRRFhd1URCeVrKd4QXuXSRUN/nSoscJRjIFxRyQkFA5V3UNV9wx57KGqBTztV5GRSTZToo4+E+shmbWUrTam0yb/c+OlOLtMqmgQsQykCROskq7Px15WvBMOR5j/qtQeBVcdN0h9/PTxAuDZ9v2ne7ywgHWyYH48sh3LcaTGCy/Y+Xz99Zpll16q2rZt/trkyDlkOzheTI+CFo6oMrGymQabThsTdeaZtClZ0D5X6b7lxtatqu3aqZ52Ws2yQYNUjzsuf21y5Jx4wiG2rrTp37+/Tps2Ld/NCKeiIny+Z5GaUtD5Jp02duli7qRYOne2mE+6ZPt4jtS58064/XaYO9fKdbRsCRdfbFV2HWWBiLyvqnXmSS6C6btKnGLw06fTxmwHrN20uPnjqqus6OHo0Tb16saNLr7hAJxw5J9i6BjTaWO2hTDX0+ImSgIolJIsuaJtWzvPTz5ZU2rdCYcDXIyjICgGP32qbSzmgHWy+Eyxfq/68OGH9l07dbLnNWvy3SJHDsEFxx1pk6mgFYMQBgmWcYmXBJCLcjKFSq9etb9vof+ejqzhhMMRTq5SeguVsO8ZlsEVdQHLQuWZZ1QbNy7968ARSjzhcDGOQidKv3qiwXXlUs4j7HvG0qlTcSQxRMGtt8K2bbWXleJ14EgLJxyFTFTTofpi9N3vxheHUivnkW7ZEh8/CaAYkhiioNSuA0d2CDNDSu1RtK6qbPnVY0dyN2qU3DVTSj79RG63ZLGNoEsmndhNscV54lFK14EjbXAxjiIkG371VHz4pV7OIxdlS5KJc7Geu1K6Dhxp44SjGMnG3V6yOTFiH/WZDKpQibpsSariXKx36aVyHTjSJi/CAZwMzMNm+LslZH1jYKy3fgrQJbCuN/BvYBbwEdDEW34+MNNbfm8q7Sha4cjG3V68TjMV10ypELW7JVVxjioDK50xNk4AHGmQc+EAGmDTu3ajZs7xXjHbXE3tOcfHeq8rPXHo471v5R2vFfA50MZb/hQwJFlbilY4VOv/Z0+lU8uX66FU5iNPVZyjsDhS/W6l4HJywpdz8iEchwOvB97/GPhxzDavA4drjVisBgSbq/yZkGMOAMYH3l8IPJKsLUUtHPUlrMNo2DD/c3PnuiOLstPJpzinak0Ve5C7FISvCMmHcAwDfh94fyHwUMw2HwMdA+8/A1oD1wNPe8IyHbjZW783sBTo4gnNS8DLcT7/CmAaMK1Tp07RndlioBDv1OrTkRXa98mnOKeaQFHsAxiLXfiKlHjCUajjOCqBI4ER3vNZIjJEVdcCI7G4yGRgEbAr7ACqOkZV+6tq/zZt2uSm1YVKrqazTYdMxwekM7YlV0UJwwoxPvEErF6d3ymEM9muPkR5vusznqTcilPmgjA1ycaD+rmqLgCeCmz3U+BHIZ9xBfDLZG0pa1dVoZLpHWSq+5WLa6NQYhxRHz/T66VcroOIIA+uqkpgAdCVmuD4QTHb/IDawfEXvNd7Yy6qZt5x3gJO89a1DWwzA+iRrC1OOAqQTP/Qqbpcysm1UQhZVVGf70yvl3K6DiIg58Jhn8mpwCdY7OJWb9mdwBne6ybAn7F03P8C3QL7fhdLuf04aFUAfwJme48LUmmHE44CJZOOLNWOoNh9+sVGLs53FFMPOxISTzjc1LGO4sKPcQRrbDVrVndyJzflbG4p1PNdqO0qEtzUsY7SINUZAYulKGGhBG7r245CPd+F2q5iJ8wMKbWHc1WVKYWWthtLoQRuo6jXVUjnu1DbVQTgXFXOVeUoMArFjZJOO/y5Wj7/3FJ5R40qjPRuRyQ4V5XDUWgUylwXqbYj3flh4rm/onDPFYrLr1wIM0NK7eFcVRHgzP/6UyipolGULYnn/ho5MvvuuUJx+ZUguLLqjqzh/qjZoVDOY6rtSCe1NZ7INGiQfbEsFAEuQeIJh3NVOdKnXOYjh2hdIKlmiKVDJu1NtR3plC2J5/7aFVohqH7uuUJx+ZUTYWpSag9ncWSZUhhUlYqrrdiyjaJubzrHdxZHSYBzVTmyRrH/UVPtAJN9z1yKTypk43dJ1t50ypu4GEfR44TDkT2i+KPmMtiejbIl2RKfbJINSzCb7U1kuWT7t85nskYJJ4o44XBkl2z+WXJ9x5iNQon5rJkV79xno9MvBTdkLilxa8cJh6NwybXrKxul2XNRpTdMIBK1KRudWLG7IXNNiZ8vJxyO6Kiv9ZHru9x0Otj63t1n2pnH269Vq8SfG9vekSPT+21K/A4665S4heaEwxENxXqXW1+xy4b4JCLeOYn3COuo6iNaJeqzzzrO4ijdhxOOCMlFJk+hEmUHG+9ONt4j7HxnIyvMkZhivXZTxAmHIxqyZaq7Tqw26Vgc8TqqbGSFOZJTwtduPOGItDquiJwM/AZoAPxeVe+JWd8Y+CPQD1gDnK+qi7x1vYHfAXsC1cAAVd0qIsOBnwAKLAe+q6qrE7XDVceNkEKp8FpqhE1YFUbnzvEr1Cb6bcD9bo6k5Lw6rog0AB4GTgF6AcNFpFfMZt8H1qpqd+AB4F5v30rgGeAqVT0IOBbY4S3/DXCcqvYGZgLXRPUdHCngJsqJhmAZkHj4nXy8EiWJfptiL9ORz2q4rhJvdK4q4HDg9cD7HwM/jtnmdeBw73UlsBoQbK7yZ0KO2RBYBXT2tvstcEWytjhXVcSUsKleENTHrRTlmI98kU83W5m5+Mh1jAMYhrmn/PcXAg/FbPMx0DHw/jOgNXA98LQnLNOBm2OOuwFYAUwCGsT5/CuAacC0Tp06RXZiHY60yFRksy3Oue4AE7U/3e+Wzyy8dJITSoBiE46bgIXe62bAv4EhnsUxHtjfszgeAm5L1hZncTgKgkK7Wy2E4ouZnJNCGPeTq8/OM/GEI8qy6suAqsD7jt6y0G28+EVLLEi+FJikqqtVdTPwKnAo0BdAVT/zvtQLwOAIv4PDkT0KrRz9iBEWI6muThwrpaHyTQAAFaBJREFUqS+Jvne8dd/9bvz4QTrl3bNBWBtz9dkFSpTCMRU4QES6ikgj4AJgXMw244CLvdfDgAmeILwOfFNEmnmCcgwwGxOaXiLSxtvnW8CcCL+Dw5E9ij0gnSmJvnei7x5vatpcJ2Qk+33KMBkkMuFQ1Z1YxtPrWOf+gqrOEpE7ReQMb7PHgVYiMh+4EbjF23ct8GtMfGYA01X1FVVdDvwvMElEZmIWyM+j+g4OR1bJ9Z1yoZDoeyf77mEWWRQTYEH8bKlEbczWZxcbYf6rUnu4GIejIMjWXBfFRroxjnzED6IqHhllafkcXC+4keMORwGQqCMppMB5tkklqyqfGUtRlGeJ95vWZzKr4LmKTRKI4HqJJxyRjhwvFNzIcUfB40bgh4+Wb9YsN66gigrrfmMRseSBTIj3mzZoED73erLfOpVqAlm+XnI+ctzhcKRBuQbOg0QVu0iFKOJP8X67MNEAE5lEo9FTye7K0fXihMPhKATKNXAeS65ShH38gPjixSZWQeqbLRXvt2vQIP4+qvGzyVIRhRxdL044HI5CwNX8yj2+68d3J6nWiEc2rJ14v+kVV9RdHkswm8wXt2RhhVxeL2GBj1J7uOC4oygo1ayqXJPqecxF6ZJUsqoSZZMlyzrz989ktscUwGVVORyOkied7LRk85VkW8QzKTiZLNss4qy8eMLhsqocDkfpkE52WrxtW7WCLVuym92VKGMM4q+78MLUsr0iyspzWVUOh6P0SZadFhwdvmkTNGpUezs/9pDtmmKJ6nUlyiZLNWkix1l5TjgcDkfpkKijDQbDVWHNGntu1ap2h/3VV+HHqE8nnKxjj5dNlmrSRI6z8iojOWoRsGPHDpYuXcrWrVvz3RRHEpo0aULHjh1p2LBhvpviKHRGjQp3+4waFX7Xv2MHtGgBqwOzT996a7jbpz6dcKdOmR3TF5BbbzWR6dQpfKrgRN87CsICH6X2CAuOL1iwQFetWqXV1dUZhIwcuaK6ulpXrVqlCxYsyHdTHMVCvCB0qvN4RBFozkVJmQgC+risqtrMnj3biUaRUF1drbNnz853MxzFTjrpt1HM1FiE6dbxhKOsYxwSO1LUUZC436mAiVeKvBBJZ5BlJiPYY2MosSPAcz0qPkLKWjgcDkc9SNZRZuszsiVMUdfCKrQZHiPECUeqZPnOas2aNfTt25e+ffvSvn17OnTosPv99u3bE+47bdo0rr322qSfMXhwdmbVnThxIqeffnpWjuUoIaLuKKMQpijv+suoUGXZZlWlRezgHf8ChowvvFatWjFjxgwA7rjjDlq0aMFNN920e/3OnTuprAz/efr370///nXG5NThX//6V0ZtczhSIuqOMtnYh0Ij08ypIiRSi0NEThaReSIyX0RuCVnfWETGeuuniEiXwLreIvJvEZklIh+JSBMR2UNEZgQeq0XkwSi/A5AzE/SSSy7hqquuYtCgQdx8883897//5fDDD+eQQw5h8ODBzJs3D6htAdxxxx1ceumlHHvssXTr1o3Ro0fvPl6LFi12b3/ssccybNgwevbsyYgRIywzAnj11Vfp2bMn/fr149prr01qWXz11VeceeaZ9O7dm8MOO4yZM2cC8M477+y2mA455BA2btzIihUrOProo+nbty8HH3wwkydPzur5cuSZqMcOFNsdfBkVqozM4hCRBsDDwLeApcBUERmnqrMDm30fWKuq3UXkAuBe4HwRqQSeAS5U1Q9FpBWwQ1W3YvOM+5/xPvCXqL7DbnJ4AS9dupR//etfNGjQgA0bNjB58mQqKyt56623+MlPfsJLL71UZ5+5c+fy9ttvs3HjRg488EBGjhxZZ8zDBx98wKxZs9hvv/044ogjeO+99+jfvz9XXnklkyZNomvXrgwfPjxp+26//XYOOeQQ/va3vzFhwgQuuugiZsyYwf3338/DDz/MEUccwaZNm2jSpAljxozhpJNO4tZbb2XXrl1sTjaXgKO4iHrsQLHdwac65qIEiNLiGAjMV9UFqrodeB4YGrPNUOAp7/WLwBCxFJoTgZmq+iGAqq5R1Vqzn4hID6AtEP1tbA5HZZ577rk08Or1r1+/nnPPPZeDDz6YG264gVmzZoXuc9ppp9G4cWNat25N27Zt+eKLL+psM3DgQDp27EhFRQV9+/Zl0aJFzJ07l27dutG1a1eAlITj3Xff5cILLwTg+OOPZ82aNWzYsIEjjjiCG2+8kdGjR7Nu3ToqKysZMGAATzzxBHfccQcfffQRe+yxR6anxVGIRB1sLsY7+BLKnEpElMLRAVgSeL/UWxa6jaruBNYDrYAegIrI6yIyXURuDjn+BcBY9X0uMYjIFSIyTUSmrVq1qn7fJIcXcPPmzXe//ulPf8pxxx3Hxx9/zMsvvxx3lHvjxo13v27QoAE7d+7MaJv6cMstt/D73/+eLVu2cMQRRzB37lyOPvpoJk2aRIcOHbjkkkv44x//mNXPdBQAUXaU+ZwR0JGQQs2qqgSOBEZ4z2eJyJCYbS4A/hTvAKo6RlX7q2r/Nm3a1K81ebqA169fT4cOprVPPvlk1o9/4IEHsmDBAhZ51TPHjh2bdJ+jjjqKZ72slokTJ9K6dWv23HNPPvvsM775zW/yP//zPwwYMIC5c+eyePFi2rVrx+WXX85ll13G9OnTs/4dHCVOmdzBFxtRZlUtA6oC7zt6y8K2WerFNVoCazDrZJKqrgYQkVeBQ4Hx3vs+QKWqvh9h+2szYkTOL9qbb76Ziy++mLvvvpvTTjst68dv2rQpjzzyCCeffDLNmzdnwIABSffxg/G9e/emWbNmPPWUeRoffPBB3n77bSoqKjjooIM45ZRTeP7557nvvvto2LAhLVq0cBaHw1EiRDYfhycEnwBDMIGYCnxHVWcFtvkB8E1VvcoLjp+tqueJyN6YSBwJbAf+CTygqq94+90DbFPV21NpS9h8HHPmzOEb3/hGfb9m0bNp0yZatGiBqvKDH/yAAw44gBtuuCHfzaqD+70cjtyT8/k4vJjFNcDrwBzgBVWdJSJ3isgZ3maPA61EZD5wI3CLt+9a4NeY2MwApvui4XEeCdxUjtR57LHH6Nu3LwcddBDr16/nyiuvzHeTHA5HgVO2MwC6O9jiwv1ejrLh2WcLJqU3nsXhRo47HA5HoRBBlYooKNSsKofD4Sg/iqRQohMOh8PhKBSKpMyKEw6Hw+EoFHI8d3imOOHIE8cddxyvv/56rWUPPvggI0eOjLvPscceix/kP/XUU1m3bl2dbe644w7uv//+hJ/9t7/9jdmza0qG/exnP+Ott95Kp/mhuPLrDkc9KZIyK0448sTw4cN5/vnnay17/vnnU6oXBVbVdq+99sros2OF48477+SEE07I6FgOhyOLFEmZFZdVBXD99eDNjZE1+vaFB+NXfB82bBi33XYb27dvp1GjRixatIjly5dz1FFHMXLkSKZOncqWLVsYNmwY//u//1tn/y5dujBt2jRat27NqFGjeOqpp2jbti1VVVX069cPsDEaY8aMYfv27XTv3p2nn36aGTNmMG7cON555x3uvvtuXnrpJe666y5OP/10hg0bxvjx47npppvYuXMnAwYM4NFHH6Vx48Z06dKFiy++mJdffpkdO3bw5z//mZ49e8b9fl999RWXXnopCxYsoFmzZowZM4bevXvzzjvvcN111wE2JeykSZPYtGkT559/Phs2bGDnzp08+uijHHXUUfX8ARyOIiUPVSrSxVkceWKfffZh4MCBvPbaa4BZG+eddx4iwqhRo5g2bRozZ87knXfe2T3nRRjvv/8+zz//PDNmzODVV19l6tSpu9edffbZTJ06lQ8//JBvfOMbPP744wwePJgzzjiD++67jxkzZrD//vvv3n7r1q1ccskljB07lo8++mh3J+7TunVrpk+fzsiRI5O6w/zy6zNnzuTnP/85F110EcDu8uszZsxg8uTJNG3alOeee46TTjqJGTNm8OGHH9K3b9+Ex3Y4HPnFWRyQ0DKIEt9dNXToUJ5//nkef/xxAF544QXGjBnDzp07WbFiBbNnz6Z3796hx5g8eTJnnXUWzTy/6BlnnLF73ccff8xtt93GunXr2LRpEyeddFLC9sybN4+uXbvSo0cPAC6++GIefvhhrr/+esCECKBfv3785S+Jp0F59913d88dElZ+fcSIEZx99tl07NiRAQMGcOmll7Jjxw7OPPNMJxwOR4HjLI48MnToUMaPH8/06dPZvHkz/fr1Y+HChdx///2MHz+emTNnctppp8Utp56MSy65hIceeoiPPvqI22+/PePj+Pil2etTlt2VX3c4csCzz0KXLlBRYc/1mac9BCcceaRFixYcd9xxXHrppbuD4hs2bKB58+a0bNmSL774YrcrKx5HH300f/vb39iyZQsbN27k5Zdf3r1u48aN7LvvvuzYsWN3KXSAPfbYg40bN9Y51oEHHsiiRYuYP38+AE8//TTHHHNMRt/NlV93OPKEP/p88WJQrRl9nkXxcK6qPDN8+HDOOuus3RlWffr04ZBDDqFnz55UVVVxxBFHJNz/0EMP5fzzz6dPnz60bdu2Vmn0u+66i0GDBtGmTRsGDRq0WywuuOACLr/8ckaPHs2LL764e/smTZrwxBNPcO655+4Ojl911VUZfS9Xft3hyBOJRp9nKejuihw6igL3ezkcKVJRYZZGLCI2IVYa5LysusPhcDjyQA5GnzvhcDgcjlIiB6PPy1o4ysFNVwq438nhSIMcjD6PVDhE5GQRmSci80XklpD1jUVkrLd+ioh0CazrLSL/FpFZIvKRiDTxljcSkTEi8omIzBWRczJpW5MmTVizZo3rlAocVWXNmjU0adIk301xOIqHESNg0SKLaSxalPWR6JFlVYlIA+Bh4FvAUmCqiIxT1dmBzb4PrFXV7t6c4/cC53vzlT8DXKiqH4pIK2CHt8+twJeq2kNEKoB9Mmlfx44dWbp0KatWrcrsCzpyRpMmTejYsWO+m+FwODyiTMcdCMxX1QUAIvI8MBQICsdQ4A7v9YvAQyIiwInATFX9EEBV1wT2uRTo6S2vBlZn0riGDRvStWvXTHZ1OByOsiZKV1UHYEng/VJvWeg2qroTWA+0AnoAKiKvi8h0EbkZQET8crB3ecv/LCLtwj5cRK4QkWkiMs1ZFQ6Hw5E9CjU4XgkcCYzwns8SkSHe8o7Av1T1UODfQGi1PVUdo6r9VbV/mzZtctRsh8PhKH2iFI5lQFXgfUdvWeg2XlyjJbAGs04mqepqVd0MvAoc6q3bDPgV9v7sLXc4HA5HjogyxjEVOEBEumICcQHwnZhtxgEXY5bDMGCCqqqIvA7cLCLNgO3AMcAD3rqXgWOBCcAQasdMQnn//fdXi8ji7HytnNKaDGM4JYo7H7Vx56Mu7pzUpr7no3PYwkhLjojIqcCDQAPgD6o6SkTuBKap6jgvxfZp4BDgK+CCQDD9u8CPAQVeVVU/ztHZ22cvYBXwPVUtrJncs4SITAsb7l+uuPNRG3c+6uLOSW2iOh9lUauqWHF/gtq481Ebdz7q4s5JbaI6H4UaHHc4HA5HgeKEo7AZk+8GFBjufNTGnY+6uHNSm0jOh3NVORwOhyMtnMXhcDgcjrRwwuFwOByOtHDCUQCISJWIvC0is71qwNd5y/cRkTdF5FPvee98tzWXiEgDEflARP7hve/qVVGe71VVbpTvNuYSEdlLRF70qkLPEZHDy/kaEZEbvP/LxyLyJxFpUm7XiIj8QUS+FJGPA8tCrwkxRnvnZqaIZDx42glHYbAT+KGq9gIOA34gIr2AW4DxqnoAMN57X05cB8wJvL8XGwjaHViLVVcuJ34D/FNVewJ9sHNTlteIiHQArgX6q+rB2Fgxv8J2OV0jTwInxyyLd02cAhzgPa4AHs30Q51wFACqukJVp3uvN2IdQgesevBT3mZPAWfmp4W5R0Q6AqcBv/feC3A8VkUZyu98tASOBh4HUNXtqrqOMr5GsMoXTb1yRc2AFZTZNaKqk7DB00HiXRNDgT+q8R9gLxHZN5PPdcJRYHiTWR0CTAHaqeoKb9VKILQScInyIHAzUO29bwWs86ooQ3i15VKmK1Yp4QnPffd7EWlOmV4jqroMK3D6OSYY64H3Ke9rxCfeNZFKxfKUcMJRQIhIC+Al4HpV3RBcp5Y3XRa50yJyOjZZ1/v5bksBUYkV9HxUVQ8BvibGLVVm18je2B10V2A/oDl1XTZlT1TXhBOOAkFEGmKi8ayq+tV/v/BNSe/5y3y1L8ccAZwhIouA5zH3w28w09ovzBlWbbmUWQosVdUp3vsXMSEp12vkBGChqq5S1R1YxewjKO9rxCfeNZFKxfKUcMJRAHj++8eBOar668Aqv3ow3vPfc922fKCqP1bVjqraBQt4TlDVEcDbWBVlKKPz8f/bu7/QGuM4juPvj6EmtfwpKdYSuRCmXCy5kFvulCVS4sIucEOTGylXrjR2w5UicmO5WoSkKNTYcrsmFya7sFokra+L32/2mJ30nHZ25Hxeddo5352e8zzrt77n93ue5/sFiIhR4IOkjTk0VRm6IccIaYmqQ9KS/P8z9fdo2DFSUGlM3AcO56urOoDxwpJWKb5z/B8gaSfwDBhiek3/HOk8x12gFXgP7I+ImSfC/muSdgGnI2KvpHWkGchyYAA4FBHf67l/80lSO+ligcXAMHCE9OWvIceIpAtAJ+mqxAHgGGnNvmHGiKTbpDYTK4FPwHmgj1nGRE6wV0lLel9JlcVfV/W5ThxmZlaGl6rMzKwUJw4zMyvFicPMzEpx4jAzs1KcOMzMrBQnDrMqSZqU9KbwmLMCg5LaihVPzf4lC//+FjOr4FtEtNd7J8zmm2ccZnNM0oikS5KGJL2UtD7H2yQ9zr0QHklqzfFVku5JepsfO/KmmiRdzz0nHkhqzu8/mXu3DEq6U6fDtAbmxGFWveYZS1Wdhd+NR8Rm0p26l3PsCnAjIrYAt4CeHO8BnkbEVlL9qXc5vgHojYhNwBdgX46fBbbl7Ryv1cGZVeI7x82qJGkiIpbOEh8BdkfEcC5eORoRKySNAasj4keOf4yIlZI+A2uKpTFyef2HuRkPkrqBRRFxUVI/MEEqLdEXERM1PlSz33jGYVYbUeF5GcUaS5NMn5PcA/SSZievCtVgzeaFE4dZbXQWfr7Iz5+Tqv0CHCQVtoTU3rMLfvVZb6m0UUkLgLUR8QToBlqAP2Y9ZrXkbypm1WuW9Kbwuj8ipi7JXSZpkDRrOJBjJ0gd/M6QuvkdyfFTwDVJR0kziy5SV7vZNAE3c3IR0JNbyJrNG5/jMJtj+RzH9ogYq/e+mNWCl6rMzKwUzzjMzKwUzzjMzKwUJw4zMyvFicPMzEpx4jAzs1KcOMzMrJSfMAXlo0ZuUXkAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "Od034qbDME_E",
        "outputId": "f9ba71ab-1919-4391-8a41-1e5022945699"
      },
      "source": [
        "plt.clf()\n",
        "kullback_leibler_divergence = history.history['kullback_leibler_divergence']\n",
        "val_kullback_leibler_divergence = history.history['val_kullback_leibler_divergence']\n",
        "epochs = range(1,len(kullback_leibler_divergence)+1)\n",
        "\n",
        "plt.plot(epochs[10:],kullback_leibler_divergence[10:],'ro',label='Training kullback_leibler_divergence')\n",
        "plt.plot(epochs[10:],val_kullback_leibler_divergence[10:],'r',label='Validation kullback_leibler_divergence')\n",
        "plt.title('Training and validation kullback_leibler_divergence')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Kullback_leibler_divergence')\n",
        "plt.legend()\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEWCAYAAACe8xtsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deZwUxfXAv4/lFgVFjApyKYoosMACngjBW5SoIAIxoFEENd4hXlGDEmP0ZzxiVAwqMSgoKmJExSNgFBV2dUFRUMBFQDwA5VrO3ff7o3p2e2fn6tmZnVn2fT+f/kx3dXX165rqevVeVVeJqmIYhmEYQaiTaQEMwzCMmocpD8MwDCMwpjwMwzCMwJjyMAzDMAJjysMwDMMIjCkPwzAMIzCmPKoJEXlNREakOm4mEZEiETkxDemqiBzi7T8qIn9MJG4S9xkuIrOSlTNGun1FZFWS1/qf/SkRubOqaca5X9k90nGNiCwSkb7e/u0i8u8YcdNSnmLdR0RuEpF/pvueuyN1My1ANiMim32HjYHtQIl3fKmqTk40LVU9LR1xd3dUdXQq0hGRtsDXQD1V3eWlPRlI+D80gqOqR2Rahlio6p8zLUNNxZRHDFS1SWhfRIqAi1X1rfB4IlI3VCEZhlG9ZOP7JyI5qloSP2bNxdxWSRByIYjIH0TkO+BJEdlbRP4jIj+KyE/efivfNbNF5GJvf6SIvCci93pxvxaR05KM205E3hWRTSLylog8HM01kKCMd4jI+156s0RkX9/5C0RkhYisE5GbY+RPbxH5TkRyfGFni8hCb7+XiHwgIj+LyBoR+buI1I+SVgUXiYj83rvmWxG5KCzuGSLyiYhsFJGVInK77/S73u/PIrJZRI4O5a3v+mNEZL6IbPB+j0k0b2IhIleKyOci0sr/33rnKsiQQFo3ichaz/UyPMFnR0SOE5G5Xp6vFJGREdLeU0T+KyIPiogEkGmAiBR6ac8VkS6+c+GuqIYiMtXLw49FpGuUNOuIyA0isswrb8+JyD7eubbi3Hu/FZFvgHfiyBe13IrPlSbOXXxF2PkFInKOt99RRN4UkfUiskREzvPFe0pEHhGRmSKyBegnIt29/2STiDzvPbe/LMfLt+tFZKFXHqeKSEPf+YHetRu9PDrVC28qIhO9d2S1iNwpvvcwlZjySJ79gX2ANsAoXF4+6R23BrYCf49xfW9gCbAv8FdgYowXNlbcZ4B5QHPgduCCGPdMRMZhwIXAfkB94HoAEekEPOKlf6B3v1ZEQFU/ArYAvwxL9xlvvwS4xnueo4H+wGUx5MaT4VRPnpOADkC4f3wL8BugGXAGMEZEfuWd6+P9NlPVJqr6QVja+wCvAg96z3Yf8KqINA97hkp5E0fmW4GRwAmqWtU+i/1xedYSGAFMEJHDvHNRn11E2gCvAQ8BLYBcoDBMzubA28D7qnqlJjhvkYh0A54ALsXl22PADBFpEOWSgcDzuHfnGWC6iNSLEO93wK+AE3Dl7Sfg4bA4JwCHA6fEkC/hcgs8CwwNu7YNrhzsAbzpybwfcD7wDy9OiGHAeGBP3Dv5EvCU96zPAmf70k4k384DTgXaAV1w5QgR6QX8C/g97v/uAxR51zwF7AIOAboBJwMXkwZMeSRPKXCbqm5X1a2quk5VX1DVYlXdhCtEJ8S4foWqPu6ZtpOAA4BfBIkrIq2BnsCtqrpDVd8DZkS7YYIyPqmqX6rqVuA5XEUDMAj4j6q+q6rbgT96eRCNshdRRPYETvfCUNUCVf1QVXepahHuxYmVVyHO8+T7TFW34JSl//lmq+qnqlqqqgu9+yWSLrgK9ytVfdqT61lgMXCmL060vImEiMh9uJe3n6r+mKAc8fijV+bm4JTdeRD32YcBb6nqs6q60ysHfuVxIDAHeF5VbwkozyjgMVX9SFVLVHUSrm/wqCjxC1R1mqruxCnohlHijgZuVtVVXnm7HRgkIn5X++2qusX7P6IRpNy+BOR6yhZgOPCid90AoEhVn/TKxyfAC8Bg3/Uvq+r7qlqKKxt1gQe9PH8Rp1BCJJJvD6rqt6q6HniF8vL2W+AJVX3T+79Xq+piEfkF7j272suXH4C/4RRdyjHlkTw/quq20IGINBaRxzzzeCPOTdIshsn4XWhHVYu93SYB4x4IrPeFAayMJnCCMn7n2y/2yXSgP22v8l4X7V64Fto5XkvqHOBjVV3hyXGoOJfZd54cf8a1qONRQQZgRdjz9fbcLj+KyAZcBZSQa8lLe0VY2ApcKz9EtLyJRDNcBXGXqm5IUIZ4/OTlu1++AyHusx8ELIuR7hlAI+DRJGRqA1znuV5+FpGfvfsdGCW+vwyVAquixG0DvORL8wucxepvYEUt6z4SLrdeg+pVyivboZQPqGgD9A57zuE4azCSPAcCq8MsOP/5RPItWnmL9n+2AeoBa3xpPoazlFKOKY/kCTfrrwMOA3qr6l6Uu0kS9h0nwRpgHxFp7As7KEb8qsi4xp+2d8/m0SKr6ue4yu00KrqswLkRFgMdPDluSkYGnOvNzzM4y+sgVW2KqwxD6cZzw3yLe/n8tAZWJyBXJH7CtVafFJFjfeFbcCP3QuxP4uztuU/88n3r7cd69pXAwTHSfRx4HZgZln4irATGq2oz39bYs9wi4S9DdXAupG8jxFsJnBaWbkNV9f8fibjWApVbPItZRI7GWUX/9ckzJ0yeJqo6Joo8a4CWYa5of9kNmm9+ov2fK3HWy76+NPfSNI14M+WROvbE9SH87PnPb0v3Db2WfD5wu4jU9wr8mTEuqYqM04AB4jpe6wPjiF9+ngGuwimp58Pk2AhsFpGOwJgI10biOWCkiHTyKoFw+ffEWWLbPL/wMN+5H3HuivZR0p4JHCoiw0SkrogMAToB/0lQtkqo6mw814cnD7i+hnM8K/AQnAsiCH/y/uvjccoplK+xnn0ycKKInOc9W3MRCXe5XYHrV3tFRBoFkOdxYLRn+YiI7CGu837PKPF7iMg5nvvpalxl92GEeI8C40MuJBFpISIDA8gVImi5nYlrRIwDpnrWEbhycKi4zvd63tZTRA6Pks4HOEvpCi/PBwK9fOeD5puficCFItJf3MCCliLSUVXXALOA/xORvbxzB4tIoq7bQJjySB3340z/tbiX4fVquu9wXKfzOuBOYCruhYxE0jKq6iLgcpxCWINrWcfrAA753d9R1bW+8Otxldsm3Es0NUEZXvOe4R1gKZVH2VwGjBORTcCtOGUTurYY18fzvmfSV/Czq+o6XGV8HS4vxwIDwuQOjKq+CVyEq5S743zQO4Dvcf1XQb4z+Q6X7996141W1cXeuVjP/g3OF34dsB6nwCqMcvLcK6Nw/+nL4hvZE+f58oFLcAMvfsL9LyNjXPIyMMSLewFwjtf/Ec4DOEtqlvdMH+IGjgQiaLn1+jdexA3GeMYXvgnXf3U+Lv+/A+4GIg4MUNUdOHftb4GfgV/jFNB273zQfPOnPQ83cONvwAZcf1XIav4NbjDH516603B9pClH1BaD2q0QkanAYlVNu+VjGEbiiMhHwKOq+mSmZUkFZnnUcDzT+WDPRD0VNxRyeqblMozajoicICL7e26rEbjhttXlkUg7pjxqPvsDs4HNuG8UxnjDCA0jacTNSbU5wjY8/tXVi7g5yiLJuijDoh0GLMC5ra4DBnn9ErsF5rYyDMMwAmOWh2EYhhGYWjMx4r777qtt27bNtBiGYRg1hoKCgrWq2iLSuVqjPNq2bUt+fn6mxTAMw6gxiEj4rAtlmNvKMAzDCIwpD8MwDCMwpjwMwzCMwJjyMAzDMAJjysMwDMMIjCkPwzAMIzCmPAzDMIzAmPIwDMPIRhYsgLfeyrQUUak1HwkahmHUKK65BvLzYc0a2CPoAo/pxywPwzCMbGPnTvjwQ9i0CV56KdPSRMSUh2EYRrZRWAhbt4IITJqUaWkiYsrDMAwj23jvPfd78cXw9tuwcmVm5YmAKQ/DMIxs4/33oV07+MMfQBWefrri+eJi2L49M7J5mPIwDMPIJlSd5XHccXDwwXD88c51FVq4b8UK6NABzjknsfTSpGRMeRiOV16BN9/MtBSGYSxfDt9/D8ce645HjoQvv3Qd6OvXw6mnwrffwsyZMHdu7LTuugv69IGNG1MupikPw5nAF1wAZ5/tCm4sli6F116rHrkMozYS6u847jj3O2gQNGoEjz0GAwe6d/TVV2HffeGOO6Knc889cNNNzkpJw1BfUx4GTJsGGzbAjh1w4YVQWlo5jqorvF27wumnw/Tp1S+nYdQG3n8fmjWDww93x3vtBeee61xX773n+j9OPx2uvx5efx3mzaucxn33wdixcP758NRTkJOTcjFNeQRh/XqYMQO+/rrc/7g78M9/wiGHOOXw7rvw4IMVz//wg2vxjB7tTOkePZwp/fXXGRHXMHZr3nvPvWd1fNXzqFFOAdx3H5x3ngu77DLYZx8YN648nqqLc911MHiwUzR10/QtuKqmdQNOBZYAS4EbIpxvAEz1zn8EtPXC6wGTgE+BL4AbE00z0tajRw+tMueeq+r+HtW991Y96STVzz+verqZZPFi9zx/+YtqaanqgAGqDRu68JUrVf/wB9VmzVQbNFD9299US0pUly9XbdpUNS9Pddu2TD9B1SgpybQENYN161Rzc1WfeirTkuzerF3r3se77qp8buPGymHjx7v4+fmqS5eqnnyyOz7nHNUdO6osDpCv0er2aCdSsQE5wDKgPVAfWAB0CotzGfCot38+MNXbHwZM8fYbA0VA20TSjLRVWXnMneuy64orVB99VPWSS1Tr1VO97rqqpZtprr9etW5d1TVr3PG336rus4/qAQe48Dp1nNL89NOK1730Unl+1FQmTHCKcfbsTEuS/Qwd6v7vnj0zLcnuzYwZLp/ffTex+Bs2uDJ82GGu0bfnnqoPPaS6a1dKxMmk8jgaeMN3fKPfgvDC3gCO9vbrAmsBAYYCr3hhzYEvgX0SSTPSViXlUVqqetxxqr/4heqmTeXhJ5ygmgqLprpYsEB16lT3PKqq27ertmihevbZFeO98IJTHtdc46yMaFx7rStCU6akR9433lA9/nh3ny+/TF26paWqf/6zk13EWVChPAmxYYPqtGkuj7KF0lLX0nzkkeq973PPubw69FD3G6tMVAfbt6suWeIq2OefV33ySdU333Qt72z6vxKhpKSiRTF2rGr9+qpbtyaexh13uP9l0CDV1atTKl4mlccg4J++4wuAv4fF+Qxo5TteBuzrua2mAD8CW4BRiabpOzcKyAfyW7dunXwOTp/usurRRyuG3367a5n/9FPyaVcXJSWqRxzhnmPYMNXNm13lCKqvvppcmjt2OKXaqJHqJ58kL9uuXRUr761bVa++2snWqpWzgED1xBOrbiWUlDilCKrDh6s+/rjbf+658jilpaoDB7rwQw5RffnlysolETZsUP2//0uda++RR5xM9es7l2J1sGaNavPmTsF+9ZWWuTgzxSuvqLZsqWXu4/AtJ8e5V2sCixer9u7tGjCnnurex6OOUj366GDplJS4tNJATVUexwKTPSWyn9fH0T6I8vBvSVseO3eqduzozMKdOyuemz3bZeHLLyeXdnXyyitO1gEDnMI78khXcFu1qpqJ+9137mVu00b1hx/Kwz/4QPXyy1VXrap8zY4dTp6xY92LUq+es4B++UunNLp00TKXWHGxq8DuuMPJmpOj+sQTyclaUqJ64YUu7auucse7drm86NCh3EccqqQvvVT18MPdfv/+wV/QkJJ68MHk5PXz3nsun44/3inTyy6repohvv8+snIsLVU980zX3xXq2+vdW7Vbt9TdO1F+/NEpe3CNoKeeUp01y1nTS5e6d/HJJ93/VL9+2irTlFBS4lxLjRq5vtPf/c6V7ZAC/P3vMy1hGTXVbfUwcIEv3hPAedXutnrsMZdNL71U+dzWrc7PeM01yaVdnRx/vGrr1q6CnDXLtSZB9dZbq572/PmugunbV7WoSPXXvy5/EQ4+WHXFivK4a9eq9unjztWrp3rMMc4tdfHFqr16uRfqF79Q/c9/Kt9n40Y3SAFU77wzmDVQWqo6alT5M/uvDfmZH31U9bPP3H966qnuJd+xw73oe++tusceqs8+m9j9vv7aVWIirmKoijtl1SqXJx06OCv30ktd2v58TZb773fP3rmze86ffnKd4xMnOksPnPUU4r77XFgq3Yjx+OYb9/x167r/LpYlt2aN6wPo169y+fj22+QsyFSycaMrW6B62mnlbqZdu1yZv/jirFJ8mVQedYHlQDtf5/YRYXEup2KH+XPe/h+AJ739PYDPgS6JpBlpS0p5bN6suv/+qsceG73Q9evnRqFkM6HO/vvvLw8rKnKd5T/+mJp7/OtfWtaH0KCB6s03q779thuV1aaN6rJlzk99yCGu4ps40VkV4ezaFdsS2r69vAU6apSrpEOUlrp+kv79XZ/NH//onq+01FkxoHrjjZX/y9JS9x8fcICrRPfbz1lUflaudIouZBHFc0VdcIFTQiG32MSJseNHY9s2p1SbNFFdtMiFrVjhFO+llyaXZohnn3Wy9evn+u7AyRxyE7Zr51yz/hFpK1e6c3fcUbV7B+G665zFWVCQWPxHH3UyTprkjktKnNIB1SFDMtcvsnatG3CQk6P68MOZV2QJkDHl4e7N6V5n9zLgZi9sHHCWt98QeB437HYe0N4Lb+KFL/IUx+9jpRlvS0p5lJa6DuZ586LHGTfOVZjr1gVPv7o4+2zXcvZ39qeD8eOd1eHvUC0ocKO3WrZ0MrRoofr++1W7T0mJM+1D1k3Hjs5F1rWrOz7wQNVTTnH7jRu7yhGchRPthX3vvfL0ovUB7djhKrKQRXXBBap//atTWP5hkYWFrkyMHevu1727sxriKcVI3HKLu98LL1QMv+wyp0CKitzx11+7+911l3PjxOPtt931ffqUd84WFKheeaXL2/nzo+fVccc5JVsdbNrkGiBDhiR+TUmJc4c2b+76aU4/3eXh8ce735NOSv+7EM6qVaqdOrmGVU1wc3tkVHlky5aS7zwi8b//aVS3VjaweLGryG65JXMyFBY6pXH44c4CSRVLlrjO0VNOcS9lx46uPyRkFSxapPqb37iW9O9+F7+lN3as6t13x7/vyy+7ex54YLnCOfJI1Tlz3PnTTnOKcv16dxwamBBtVNqSJS7+FVdUlLGw0Ml+wQWVr1m50llw556rOnKka82GLAZwCuv6653CvOwyN7T82mvdKLMHHnBDOo84olzGIDz4oLtHKr9xKi11+bp2bcXwUP9T0AbHwoUuP+rWdUrykUfcPZ580uVVz54V++jisWqVU+A33ugs3xNPdP95v37xLaI5c5wV16SJ6jvvBHuODGPKI53KY9s256e/8srkrl+0qLz16GfVKtV//lP19dcrv1DRKClRve0218odPNi95IMHu4r1+++Tky9VbNiQXnfBzp3RlcOWLem777p1zjpt08a9TiGL5557yuOUlDjF1qVLZRm3b3cuo5wcd9348S58506nAPbbL/r/H3LFNWrkBgCsXOnK0r33OldXvXqun6Z5c+d+bdy4XLm0bp38iK1vv3UNkttuS+76SDz0kJPruOPKB6aUlroGR48eybl47rzTVdpz51YMnzHDuedycpwb9YwzVMeMcR/W5eW5PG/WzLkx27d3eRfKt3r1XJpHH636q1+5vpg6ddxAj/CP+P77X9cPCC6tWB6MLMWURzqVh6prhQQ140tLXSs3VCh79HCtwokTnc9epPwcuEJ87bWuEo7E5s2u8IPz3x90UPm1o0dX/RmN2GzZ4vpYGjRweR8+Tn/SJI1ood5wgwufNq28L+epp9xwWHDfMUTj559V//GPYA2D4mKnNIJ8RxCJvn1dpTppUtWHIs+Z4yyETp3cM99wgwt/802t0HeRDNGUzscfO2t88GCn1Pfe2ymqU05xVtrvfuc6r4cPd9brAw+ofvhh5Xxbv969XyJOkXTr5r6HOeAAJ/v++zvrOJ0NmDRiyiPdyuPOO11WJtr5vGuX89GD6nnnOSXSq1dFRXHrrc70fvttV5EMHOgKaMuW7rsTP0VFrtO+Th03Gib0wqxYofrii66SMaqHb76J3KLfscNZhHXruspx82bXMhVxlZSqs0L693dxGjRwfVXZ2qn64Yfllf3++7u+v0QqyI0b3bOHWLnStfQPPdSV09CIuJkz3TDh/farGVPgfPCBs0QGDHD9MxddpPr3v0ceFFKDMOWRbuXx/vta1nqMR3GxK2TgfNL+kSzffONaRNEqjA8/dBZO6LuDvn3L/e577eVeOCN7+f571z8RchsdeKCrNP2V6YYNruN/772deyibCY1uO+00LbN4ozVUSkvdsPfGjZ1iPPVUN+IoNJIs1H9SXFxuCYg4a87IGKY80q08tm93L8Xll8eOV1xc7pJK9sOxHTvciJo2bdzXqCNGOHfXV18ll55R/bz7rutsrV/fjWoKp7i48lDhbOe551x/QPfulTui16xx/QqhRs8117i+hpClHT6SbMkSp1Dq1k35dBtGMGIpD3Hnd3/y8vI0Pz8/fTc44wy3SFJeHpx4Ipx0klvBKzSP/tatblrzt95y8+v/5jfpk8XIfnbuhHXrYP/9My1J6njtNbc0art2cO+9sHgxfPKJC9+yBe6+G664wk01rupWx1u7tnzFPD/vvQerVrn1KIyMISIFqpoX8ZwpjxSxZo1bD+Ott9xykSUl0Lo1XHwxDBvm5t5/80148kkYMSJ9chhGJpkzB848EzZtcscHHgi9esGf/1y+uJFRYzDlQTUoDz8bN7oVvh5/3CkTABGYONGt1GcYuzNFRc6qyM2F/fbLtDRGFTDlQTUrDz/Lljk3VefO5SuAGYZh1ABiKY80rU9olHHwwbEXqTcMw6iB2BrmhmEYRmBMeRiGYRiBMeVhGIZhBCaQ8hCRRiJyWLqEMQzDMGoGCSsPETkTKARe945zRWRGugQzDMMwspcglsftQC/gZwBVLcSt5mcYhmHUMoIoj52quiEsrHZ8JGIYhmFUIMh3HotEZBiQIyIdgCuBuekRyzAMw8hmglgevwOOALYDzwAbgKvTIZRhGIaR3SRseahqMXCztxmGYRi1mCCjrd4UkWa+471F5I30iGUYhmFkM0HcVvuq6s+hA1X9CYg7ZaaInCoiS0RkqYjcEOF8AxGZ6p3/SETaeuHDRaTQt5WKSK53boiILBSRRSJyd4BnMAzDMFJAEOVRKiKtQwci0oY4o61EJAd4GDgN6AQMFZFOYdF+C/ykqocAfwPuBlDVyaqaq6q5wAXA16paKCLNgXuA/qp6BLC/iPQP8ByGYRhGFQmiPG4G3hORp0Xk38C7wI1xrukFLFXV5aq6A5gCDAyLMxCY5O1PA/qLiITFGepdC9Ae+EpVf/SO3wLODfAchmEYRhUJ0mH+uoh0B47ygq5W1bVxLmsJrPQdrwJ6R4ujqrtEZAPQHPCnPYRypbMUOMxzb60CfgXUj3RzERkFjAJo3bp1pCiGYRhGEgSdGLEBsB7YCHQSkT6pF6kiItIbKFbVz6Csr2UMMBX4H1AElES6VlUnqGqequa1aNEi3aIahmHUGhK2PLyO6SHAIqDUC1ac+yoaq4GDfMetvLBIcVaJSF2gKbDOd/584Fn/Bar6CvCKJ9cooigPwzAMIz0E+cL8V8Bhqro9wDXzgQ4i0g6nJM4HhoXFmQGMAD4ABgHvqLc2rojUAc4DjvdfICL7qeoPIrI3cJkXxzAMw6gmgiiP5UA93BfmCeH1YVwBvAHkAE+o6iIRGQfkq+oMYCLwtIgsxbnEzvcl0QdYqarLw5J+QES6evvjVPXLAM9hGIZhVBHxGvnxI4q8AHQF3sanQFT1yvSIllry8vI0Pz8/02IYhmHUGESkQFXzIp0LYnnM8DbDMAyjlhNkqO4kEWkEtFbVJWmUyTAMw8hybCVBwzAMIzBVXUmwfRpkMgzDMLKcqq4kWBoxpmEYhrFbYysJGoZhGIFJdiXBZ3FTlNhKgoZhGLUQW0nQMAzDCEyQua1eofL6HRuAfOAxVd2WSsEMwzCM7CWI22o5sBl43Ns2ApuAQ71jwzAMo5YQpMP8GFXt6Tt+RUTmq2pPEVmUasEMwzCM7CWI5dEkbBna1kAT73BHSqUyDMMwspoglse1uGVolwECtAMuE5E9KF9G1jAMw6gFJKQ8vHU19gQ6AB294CW+TvL70yCbYRiGkaUk5LZS1VJgrKpuV9UF3majqwzDMGopQfo83hKR60XkIBHZJ7SlTTLDMAwjawnS5zHE+73cF6bY5IiGYRi1jiBfmLdLpyCGYRhGzSHIeh6NReQWEZngHXcQkQHpE80wDMPIVoL0eTyJ+57jGO94NXBnyiUyDMMwsp4gyuNgVf0rsBPKJkqUeBeJyKkiskRElorIDRHONxCRqd75j0SkrRc+XEQKfVupiOR654aKyKcislBEXheRfQM8h2EYhlFFgiiPHd4a5gogIgfjpmePiojkAA8DpwGdgKEi0iks2m+Bn1T1EOBvwN0AqjpZVXNVNRe4APhaVQtFpC7wANBPVbsAC4ErAjyHYRiGUUWCLkP7OnCQiEwG3gbGxrmmF7BUVZer6g5gCjAwLM5Ayr9Qnwb0F5Fwi2aody04a0eAPbx4ewHfBngOwzAMo4oEGW01S0QKgKNwlfdVqro2zmUtgZW+41VA72hxVHWXiGwAmgP+tIfgKR1V3SkiY4BPgS3AV1QcPmwYhmGkmSCjrV4BTgZmq+p/ElAcKUFEegPFqvqZd1wPGAN0Aw7Eua1ujHLtKBHJF5H8H3/8sTrENQzDqBUEcVvdCxwPfC4i00RkkIg0jHPNauAg33ErLyxiHK8/oymwznf+fNyytyFyAVR1maoq8BzlI8AqoKoTVDVPVfNatGgRR1TDMAwjURJWHqo6R1Uvw31R/hhwHvBDnMvmAx1EpJ2I1McpghlhcWYAI7z9QcA7nlIITch4HuX9HeCUTScRCWmDk4AvEn0OwzAMo+oEmZ4Eb7TVmbg+iO7EmYrd68O4AngDyAGeUNVFIjIOyFfVGcBE4GkRWQqsx+VydWUAACAASURBVCmYEH2Alaq63JfmtyLyJ+BdEdkJrABGBnkOwzAMo2qI18iPH1HkOdzoqdeBqcAcb7bdGkFeXp7m5+dnWgzDMIwag4gUqGpepHNBLI+JwFBVLUmNWIZhGEZNJa7yEJFfquo7wB7AwPBPMFT1xTTJZhiGYWQpiVgeJwDv4Po6wlHAlIdhGEYtI67yUNXbvN8L0y+OYRiGURNIxG11bazzqnpf6sQxahs7d+5k1apVbNtmqxobRqZo2LAhrVq1ol69eglfk4jbak/v9zCgJ+XfaZwJzAskoWGEsWrVKvbcc0/atm1L5SnNDMNIN6rKunXrWLVqFe3aJb7mXyJuqz8BiMi7QHdV3eQd3w68mpy4huHYtm2bKQ7DyCAiQvPmzQk6hVOQ6Ul+gVsMKsQOL8wwqoQpDsPILMm8g0G+8/gXME9EXvKOfwU8FfiOhmEYRo0nyNxW44ELgZ+87UJVvSt0XkT2Tr14hhHG5MnQti3UqeN+J0+uUnLr1q0jNzeX3Nxc9t9/f1q2bFl2vGPHjpjX5ufnc+WVV8a9xzHHRJy3MzBPPfUUV1yR+Lpn/vi333479957LwB9+/YlFbMttG3blrVrE5tcu0mTJnHjhPJp9uzZDBgwoMr3TBT//WbMmMFf/vKXlKa/uxJobitV/Rj4OMrpt3HzXRlGepg8GUaNguJid7xihTsGGD48qSSbN29OYWEh4CrYJk2acP3115ed37VrF3XrRn5N8vLyyMuLOHNDBebOnZuUbLWNdORTSUkJOTk5Ccc/66yzOOuss6p831jlZnchSJ9HPMxxbaSXm28uVxwhiotdeAoZOXIko0ePpnfv3owdO5Z58+Zx9NFH061bN4455hiWLFkCVGyx3n777Vx00UX07duX9u3b8+CDD5alF2p1z549m759+zJo0CA6duzI8OHDCc0tN3PmTDp27EiPHj248soro7a8Q7z66qscffTRrF27tkJrPD8/n759+8Z9xqeffprc3FyOPPJI5s1zgyajPWdJSQnXX389Rx55JF26dOGhhx6qkNbWrVs57bTTePzxx+PeF+Cee+6hZ8+edOnShdtuu61SPgFs3LiRM844g8MOO4zRo0dTWlp5Gr1///vf9OrVi9zcXC699FJKSkrK0rnuuuvo2rUrH3zwQUQZXn/9dTp27Ej37t158cXy75xD1tqGDRto06ZN2X23bNnCQQcdxM6dO1m2bBmnnnoqPXr04Pjjj2fx4sVA5XKzbNkyjjrqKDp37swtt9xS4fki5UFRURGHH344l1xyCUcccQQnn3wyW7duBWDp0qWceOKJdO3ale7du7Ns2bKYeVkdpFJ5JDbDomEkyzffBAuvAqtWrWLu3Lncd999dOzYkf/973988sknjBs3jptuuiniNYsXL+aNN95g3rx5/OlPf2Lnzp2V4nzyySfcf//9fP755yxfvpz333+fbdu2cemll/Laa69RUFAQd9TLSy+9xF/+8hdmzpzJvvvum9TzFRcXU1hYyD/+8Q8uuugigKjPOWHCBIqKiigsLGThwoUM91l5mzdv5swzz2To0KFccsklce87a9YsvvrqK+bNm0dhYSEFBQW8++67leLNmzePhx56iM8//5xly5ZVqOABvvjiC6ZOncr7779PYWEhOTk5TPZcmFu2bKF3794sWLCA4447rlLa27Zt45JLLuGVV16hoKCA7777rlKcpk2bkpuby5w5cwD4z3/+wymnnEK9evUYNWoUDz30EAUFBdx7771cdtllZdf5y81VV13FVVddxaeffkqrVq0SyoOvvvqKyy+/nEWLFtGsWTNeeOEFAIYPH87ll1/OggULmDt3LgcccEDCeZkudm+7yti9aN3auaoihaeYwYMHl7k7NmzYwIgRI/jqq68QkYhKAeCMM86gQYMGNGjQgP3224/vv/++QqUB0KtXr7Kw3NxcioqKaNKkCe3bty8bYz906FAmTJgQ8R7vvPMO+fn5zJo1i7322ivp5xs6dCgAffr0YePGjfz8889s2rQp4nO+9dZbjB49uswNs88++5SlM3DgQMaOHVtBocRi1qxZzJo1i27dugFO+Xz11Vf06dOnQrxevXrRvn37Mlnfe+89Bg0aVHb+7bffpqCggJ49ewLO+tlvv/0AyMnJ4dxzz40qw+LFi2nXrh0dOnQA4Ne//nXE/B4yZAhTp06lX79+TJkyhcsuu4zNmzczd+5cBg8eXBZv+/btZfv+cvPBBx8wffp0AIYNG1bmDo2WB61bt6Zdu3bk5uYC0KNHD4qKiti0aROrV6/m7LPPBtwHfUHyMl2kUnmY28pIL+PHV+zzAGjc2IWnmD322KNs/49//CP9+vXjpZdeoqioKKpbqEGDBmX7OTk57Nq1K6k4sTj44INZvnw5X375ZVl/S926dcvcK4l+qR8+NFNEEn5OP8ceeyyvv/46w4YNS2i4p6py4403cumllwaWLzydESNGcNdddxFOw4YNA/VzROOss87ipptuYv369RQUFPDLX/6SLVu20KxZs7J+snD85SYa0fKgqKioUvkIua2CpFNdJOS2EpEcEVkcJ1r/FMhjGNEZPhwmTIA2bUDE/U6YkHRneaJs2LCBli1bAs4nnmoOO+wwli9fTlFREQBTp06NGrdNmza88MIL/OY3v2HRokWAG4FUUFAAUObmiEfoHu+99x5NmzaladOmUZ/zpJNO4rHHHitTdOvXry87N27cOPbee28uv/zyhO57yimn8MQTT7B582YAVq9ezQ8/VF6QdN68eXz99deUlpYyderUSu6n/v37M23atLJr169fz4pIVmkEOnbsSFFRUVm/wbPPPhsxXpMmTejZsydXXXUVAwYMICcnh7322ot27drx/PPPA64CX7BgQcTrjzrqqLL/Y8qU8sVQE82DEHvuuSetWrUqs2K2b99OcXFx4HRSTULKw1vDY4mIRPUPqOr6aOcMI2UMHw5FRVBa6n7TrDgAxo4dy4033ki3bt0CWwqJ0KhRI/7xj3+UdcLuueeeNG3aNGr8jh07MnnyZAYPHsyyZcu47bbbuOqqq8jLy0u4xd2wYUO6devG6NGjmThxIhD9OS+++GJat25Nly5d6Nq1K88880yFtB544AG2bt3K2LFj49735JNPZtiwYRx99NF07tyZQYMGsWnTpkrxevbsyRVXXMHhhx9Ou3btylw2ITp16sSdd97JySefTJcuXTjppJNYs2ZNws8+YcIEzjjjDLp3717m7orEkCFD+Pe//82QIUPKwiZPnszEiRPp2rUrRxxxBC+//HLEa++//37uu+8+unTpwtKlS8v+00TzwM/TTz/Ngw8+SJcuXTjmmGP47rvvkkonlQRZSfBdoBtuPqstoXBVrfq4tmrAVhLMTr744gsOP/zwTIuRcTZv3kyTJk1QVS6//HI6dOjANddck2mxjCpQXFxMo0aNEBGmTJnCs88+G1XRZAOR3sVUrST4x6oIZhhGdB5//HEmTZrEjh076NatW8b82EbqKCgo4IorrkBVadasGU888USmRUopCVseACLSBuigqm+JSGMgJzRRYrZjlkd2YpbH7sO6devo379y1+fbb79N8+bNMyARnH322Xz99dcVwu6++25OOeWUjMiTzaTN8hCRS4BRwD7AwUBL4FGso9wwDCp+rZ8tvPTSS/EjGUkR5CPBy4FjgY0AqvoVEL2nyUNEThWRJSKyVERuiHC+gYhM9c5/JCJtvfDhIlLo20pFJFdE9gwLXysi9wd4DsMwDKOKBOnz2K6qO0LjrUWkLnG+KheRHOBh4CRgFTBfRGao6ue+aL8FflLVQ0TkfOBuYIiqTgYme+l0BqaraqhZk+u7RwG2jrphGEa1EsTymCMiNwGNROQk4HnglTjX9AKWqupyVd0BTAEGhsUZCEzy9qcB/aXy10ZDvWsrICKH4qyf/wV4DsMwDKOKBFEeNwA/Ap8ClwIzgVviXNMSWOk7XuWFRYyjqruADUB479oQINKXPOcDUzVKr7+IjBKRfBHJD7pKlmEYhhGdIOt5lKrq46o6WFUHeftpnwxRRHoDxar6WYTT5xNZqQCgqhNUNU9V81q0aJE2GY2aS79+/XjjjTcqhN1///2MGTMm6jX+9TBOP/10fv7550px/OtnRGP69Ol8/nm5B/fWW2/lrbfeCiJ+RGzdD1v3ozqI2+chIp8So29DVbvEuHw1cJDvuJUXFinOKq8fpSmwznc+ooIQka5AXVUtiPkAhhGDoUOHMmXKlApDN6dMmcJf//rXhK6fOXNm0veePn06AwYMoFOnToCb6sOIj637kR0kYnkMAM6MscViPtBBRNqJSH2cIpgRFmcGMMLbHwS8E7JoRKQOcB4R+jtw/SBRrQ6jBnL11dC3b2q3q6+OectBgwbx6quvlq0aWFRUxLfffsvxxx/PmDFjyMvL44gjjoi6VoK/hTt+/HgOPfRQjjvuuLK1MMB9ANizZ0+6du3KueeeS3FxMXPnzmXGjBn8/ve/Jzc3l2XLljFy5EimTZsGuG8junXrRufOnbnooovKZm5t27Ytt912G927d6dz585la0lEw9b9sHU/YuVlVYirPFR1RWgDtgNdgS640VcxZyLz+jCuAN4AvgCeU9VFIjJOREJqeyLQXESWAtfi+lZC9AFWquryCMmfhykPo4rss88+9OrVi9deew1wVsd5552HiDB+/Hjy8/NZuHAhc+bMYeHChVHTKSgoYMqUKRQWFjJz5kzmz59fdu6cc85h/vz5LFiwgMMPP5yJEydyzDHHcNZZZ3HPPfdQWFjIwQcfXBZ/27ZtjBw5kqlTp/Lpp5+ya9cuHnnkkbLz++67Lx9//DFjxoyJ6RqzdT9s3Y90rvsR5CPBi4FbgXdw068/JCLjVDXmN/eqOhPXue4Pu9W3vw0YHH6dd242cFSUc+0Tld2oIdyfmc91Qq6rgQMHMmXKlLKJAp977jkmTJjArl27WLNmDZ9//jldukT20v7vf//j7LPPpnHjxgAVXBqfffYZt9xyCz///DObN2+O+3XzkiVLaNeuHYceeigAI0aM4OGHH+Zqz4o655xzALfeQ3hlGcLW/XDYuh/pW/cjiLPt90A3VV0HICLNgbnA7jVhi1HrGDhwINdccw0ff/wxxcXF9OjRg6+//pp7772X+fPns/feezNy5MiE18oIZ+TIkUyfPp2uXbvy1FNPMXv27CrJG1rzIdZ6ILbuR3k6tu5Hetb9CDJUdx3gn8dqExU7tg2jRtKkSRP69evHRRddVNbS3rhxI3vssQdNmzbl+++/L3NrRaNPnz5Mnz6drVu3smnTJl55pfwTqE2bNnHAAQewc+fOMpcJuHUaIk2hfdhhh1FUVMTSpUsB1+9wwgknBHomW/fDYet+pG/dj7jKQ0SuFZFrgaXARyJyu4jcBnwIfFllCQwjCxg6dCgLFiwoUx5du3alW7dudOzYkWHDhnHsscfGvL579+4MGTKErl27ctppp5W5SQDuuOMOevfuzbHHHkvHjh3Lws8//3zuueceunXrVlZBgWsRP/nkkwwePJjOnTtTp04dRo8eHfiZbN0PW/cjnet+xJ1V11MUUVHVP1VZimrAZtXNTmxWXaO2ky3rfqR8Vt2aohwMwzBqIjV13Y9EPhK8X1WvFpFXiPCxYE1ZSdAwjOzA1v2oyPHHHx+1PySbSWS01dPeb+y5FgwjSVQ1oRE4xu6BrfuRfSQz01QibqsC73eOiDQCWqvqkjiXGUZCNGzYkHXr1tG8eXNTIIaRAVSVdevWlX0XkihBPhI8E2d91AfaiUguMM7cVkZVaNWqFatWrcJmPTaMzNGwYcMKX7cnQpCPBG/Hrc8xG0BVC0WkXaC7GUYY9erVo107K0aGUdMI8pHgTlXdEBaW9inZDcMwjOwjiOWxSESGATki0gG4Ejc9iWEYhlHLCGJ5/A44Ajez7rPARiD2fNeGYRjGbknCloeqFgM3e5thGIZRi0nkI8GIHweGsNFWhmEYtY9ELA/7ONAwDMOoQCIfCc5JJCEReUFVo6+sYhiGYew2BOkwj4et7GcYhlFLSKXysG8+DMMwagmpVB6GYRhGLSGVysNmtTMMw6glJKw8RKRHhLABvsM/RLnuVBFZIiJLReSGCOcbiMhU7/xHItLWCx8uIoW+rdSbjBERqS8iE0TkSxFZLCLWUW8YhlGNBLE8HheRI0MHIjIU+GPoWFVnhV8gIjnAw8BpQCdgqIh0Cov2W+AnVT0E+Btwt5feZFXNVdVc4ALga1UNLQJwM/CDqh7qpZvQiDDDMAwjNQRRHoOAf4lIRxG5BLgMODnONb2Apaq6XFV3AFOAgWFxBgKTvP1pQH+pvLDDUO/aEBcBdwGoaqmqrg3wHIZhGEYVSVh5qOpy4HzgReBc4OQIs+yG0xJY6Tte5YVFjKOqu4ANQPhalENw82khIs28sDtE5GMReV5EfhHp5iIySkTyRSTf1oswDMNIHXGVh4h8KiILRWQhzjLYB2gHfOSFpRUR6Q0Uq+pnXlBdoBUwV1W7Ax8Q5St4VZ2gqnmqmteiRYt0i2oYhlFrSGR6kgHxo0RlNXCQ77iVFxYpzioRqQs0Bdb5zp+PZ3V4rAOKcRYQwPO4fhPDMAyjmohreajqClVdARwArPcd/wTsH+fy+UAHEWknIvVximBGWJwZwAhvfxDwjnqrsYtIHeA8fP0d3rlXgL5eUH/g83jPYRiGYaSOIB3mjwCbfcebvbCoeH0YVwBvAF8Az6nqIhEZJyKh2XgnAs1FZClwLeAfztsHWOn1t/j5A3C75za7ALguwHMYhmEYVSTISoISsgjAjXLy3EwxUdWZwMywsFt9+9uAwVGunQ0cFSF8BU6xGIZhGBkgiOWxXESuFJF63nYVEG4RGIZhGLWAIMpjNHAMroN7FdAbGJUOoQzDMIzsJsgytD/gOrwNwzCMWk7CykNEGuKGxB4BNAyFq+pFaZDLMAzDyGKCuK2exg3NPQU3l1QrYFM6hDIMwzCymyDK4xBV/SOwRVUnAWfg+j0MwzCMWkYQ5bHT+/3Zm123KbBf6kUyDMMwsp0g33lMEJG9cdOwzwCa4JuS3TAMw6g9BBlt9U9vdw7QPj3iGIZhGDWBICsJNheRh7xp0AtE5H4RCZ863TAMw6gFBOnzmAL8gFvLYxCwFpiaDqEMwzCM7CZIn8cBqnqH7/hOERmSaoEMwzCM7CeI5TFLRM4XkTredh5utlzDMAyjlhHX8hCRTYACAlyN+1gQIAc3Lfv1aZPOMAzDyEoSmVJ9z+oQxDAMw6g5JGJ5dI91XlU/Tp04hmEYRk0gkQ7z/4txToFfpkgWwzAMo4aQiNuqX3UIYhiGYdQcEnFbnRPrvKq+mDpxDMMwjJpAIm6rM2OcU8CUh2EYRi0jEbfVhVW5gYicCjyAG9r7T1X9S9j5BsC/gB7AOmCIqhaJyHDg976oXYDuqlooIrOBA4Ct3rmTvZUODcMwjGogyEqCt0YKV9VxMa7JAR4GTsKtez5fRGao6ue+aL8FflLVQ0TkfOBunAKZDEz20ukMTFfVQt91w1U1P1H5DcMwjNQR5AvzLb6tBDgNaBvnml7AUlVdrqo7cPNjDQyLMxCY5O1PA/qLiITFGepdaxiGYWQBQaZkrzBkV0TuJf70JC2Blb7jVVRefbAsjqruEpENQHPcxIshhlBZ6TwpIiXAC8CdqqrhNxeRUcAogNatW8cR1TAMw0iUIJZHOI1x65inFRHpDRSr6me+4OGq2hk43tsuiHStqk5Q1TxVzWvRokW6RTUMw6g1BFnP41MRWehti4AlwP1xLlsNHOQ7buWFRYwjInVxy9uu850/H3jWf4GqrvZ+NwHP4NxjqWfyZGjbFurUgX33dVudOi5s8uS03NIwDKMmEFd5iEio8h+AG7Z7JnAycCBQFOfy+UAHEWknIvVximBGWJwZwAhvfxDwTsgFJSJ1gPPw9XeISF0R2dfbr+fJ9RmpZvJkGDUKVqwAVVi3zm2qLmzUKFMg6cavvE1hG0ZWkYjl8aaItFXVFb5tNc5V9ECsC1V1F3AFrm/kC+A5VV0kIuNE5Cwv2kSguYgsBa4FbvAl0QdYqarLfWENgDdEZCFQiLNcHk/gOYJx881QXBz9fHExjBhhFVu6CFfeprAziylyIwyJ0M9cMYLI6Tj31Bmq+pUXdiMwDDhNVVelXcoUkJeXp/n5AUb21qnjKq1EadwYJkyA4cODC2dUpm1bpzDCadMGioqqW5raTUiR+xtTVt5rBSJSoKp5kc7FtTxUdSYwBnhNRI4Ukftxrqs+NUVxJEXQ0VnFxfDrX2dXq6wmtxa/+SZYuJE+IlnhxcUu3Ki1JNRhrqpvAxcCs4H2wC9V9ac0ypV5xo93raugZIt7paa7faIpbxtyXf1EU9grVtS8RomRMhLpMN8kIhuBmcBeQH/gB1/47snw4c4sb9MGRKB5c7eJQE5O7GuzoVVW01uLkZR348Yu3KheYinsmtYoMVJGIm6rPVV1L++3vqru4TveqzqEzBjDhzv/emkprF3rttJSmDQpvlWSafdKTXf7hCvvNm3Mxx6LdLoo41nhNalRYqSMqnwkWHvxV2zRyLR7JVm3TzoqoWTT9CvvoqKarTjSWbmn20WZSHmvKY2S3YFs6ctU1Vqx9ejRQ9PCv/+t2rixqntt3da4sQvPJMnIlY5nydb8qU7SnQdt2lRMO7S1aVM1mdu0URVxvyFZ03EvI3Gq+X0C8jVKnZrxSr26trQpD9XoL1qmCSpXOiqG6qhssjX/Q6Q7D0Qipy+SXHqxKihrDGSWalbepjzSrTySJdsqvWiVUKgiSkbGVFds4aTLWkrl/5LuPEh1hRIvvWwrt7WJIGUpBf+TKY9sVB6RKr1QwcjUCxmt0ogm45gx8QtnultKqU4/njJK5oVMdx6kSoGGni3Wf59NZJMSqy5ZEi1LKSoTpjyyUXnEq6jr1VNt3rxyYUxVIY2UTqQCF2SLVDjTbRnEkide/kTKg1gvZ7LPUh2unqqWi0T++2zq18gm91l1ypJoozNFDRZTHpqFyiNexRepYh4zJnUtzFg+7UQq5iAVTCpbZUEVXDSroXlz1fr1K8eN1equyguZ6pZpqtOL15jJtn6NbOq4r84BC/5zfsUR/j+lyFVqykOzUHkk4iIK33JyUlN5RUsn0dZLrC3dro1kZIpmNQTN43T3XSRKOlxrsRoLmXYJRSJb/ot0yBLp/43kiYiltMzy2I2VR1VdREEKabKt9WRkTHfLLxmLKJbVEO35I+VHtrR20+Fay5ZnS5Rskre6BizEKqPh5d36PGqo8gh3j0Tqu/DHC/3h8QpMtFZxTk5yndaJKIEgMlaHayPZ1laiSsffqgvlaWhgQKQ8yIQ7J1ZLN9mRUqnudE93x3Gm+zziuUCrMvgl0bIazxNho61qmPKI11qPVsAT8cdH6vNIJP1kW+uxZEx0tFWqSfYbhERbc4l0+AcdcZZqqqIkYym/dHS6p7NCz9Roq1hupao0MBItp+FppzG/TXloNSqPRP78qnSwJtJ3EZ5+NJni9X1kE0GtuURa1tFGtPlJ1kWUzoqtqkoyXf91dbiSsmF4bjr6GpJ1Eac5P0x5aDUqj6DukXTdK1GXRKbN/0TIpEslGRdR8+ap79BO9FmSqYRS1cmcrk7sREYX+eMFaVQkQ6znTHbgQSylH80TUQ3vqCkPzTLLI2gBiFbwgwyvrK4XK9lniUUmW7Sp6EcJrzyqU7EEla0qeRcr7XC5wq3HaO6/RJRhIlZgKhtIyVge8e4bT/FmyOIy5aFZ1OcRtAIM+lKks4JNlFR3ymZyipNUuohiWSshF2Kq+yTS0ZiJl3fJltVk8zjeQIGquJKCluNk38lUNZBSrGRMeWgWjLaKVfBjkejomWTTTzXJVLbxXpB0Wx6J5nFVXUTJWCuJKpZoxOrwT0WexrM4ErWSq5JfseLFciWFhrT6/9tYI+v8fWSJjqJM9J1MhXWUBhe0KQ/Ngu88kq0AE211V4drJxGScfMk851KKn2+VbFsEm0dV6VDO17FmaiM/oouVdZcoumk6/ucRPI12sCQSH1TQbZ4ZTDoO1lVqyENdUBGlQdwKrAEWArcEOF8A2Cqd/4joK0XPhwo9G2lQG7YtTOAzxKRI+PKI9kKMNECkemO70RaW1Up3On0+abipavK4IWqbMlalqmqaBJNJ1nLI5blVJV8TUaZBc2vSDIlMsIvWdLg3s2Y8gBygGVAe6A+sADoFBbnMuBRb/98YGqEdDoDy8LCzgGeqTHKQzW5CjCIUshQp1qVOjXT+TJVRf6gijdIZez/n2INl65q5ZXuZ46XTrj7NnzEUKwtmb6eRC2QVCmORCrmeHmQygbe7mR5AEcDb/iObwRuDIvzBnC0t18XWAtIWJw/A+N9x02A94BONUp5JEumlEKiBBlhUp0vUxCqmsfJVsaxWtbxFEtV8ypV5SpSOvEaComOtkqGZIZXV4fyTrdreXfq8wAGAf/0HV8A/D0szmdAK9/xMmDfsDjLgCN9x38DzgbaxlIewCggH8hv3bp10hloxCHVY9szMUIsFSRbGQfpkK/KtBfVSSZdlMn0u8WyLCA1DZ1k35Mg7C6jrVKhPIDewKe+41xghrcfU3n4txpteWQ76R4MUBWy3WqLR02VP5ODI5IdXh3PAkqXUkuVJZkGarTbyrMybvIdjwG+BYqAVcAOYHY8WUx5pJF0DwaobrmMqpPpYdnJfKORbpL9BiSDZFJ51AWWA+18HeZHhMW5PKzD/DnfuTrAaqB9lPTN8sgW0j0YIBl2N7dYTSLZ/3Z3t0az7busOGRMebh7czrwpeeOutkLGwec5e03BJ7HDdWd51cUQF/gwxhpm/Ko6UT6oDJVL3V1VERGdJKppGuLwq8hz5lR5ZEtmymPLCcdVkgNeUENH7XFKq764QAABl1JREFU1VhDnjOW8qiDYWQDN98MxcUVw4qLXXiyjB8PjRtXDGvc2IUb2cnw4TBhArRpAyLud8IEF747sRs8pzjlsvuTl5en+fn5mRbDiEadOq79FY4IlJYmn+7kyU4BffMNtG7tFEcNekENI5OISIGq5kU6V7e6hTGMiLRuDStWRA6vCsOHm7IwjDRgbisjOzAXk2HUKEx5GNnBbuADNozahLmtjOzBXEyGUWMwy8MwDMMIjCkPwzAMIzCmPAzDMIzAmPIwDMMwAmPKwzAMwwhMrfnCXER+BCJ8hZb17Iubpt5wWH5UxPKjMpYnFalKfrRR1RaRTtQa5VFTEZH8aNMD1EYsPypi+VEZy5OKpCs/zG1lGIZhBMaUh2EYhhEYUx7Zz4RMC5BlWH5UxPKjMpYnFUlLflifh2EYhhEYszwMwzCMwJjyMAzDMAJjyiNLEJGDROS/IvK5iCwSkau88H1E5E0R+cr73TvTslYnIpIjIp+IyH+843Yi8pGILBWRqSJSP9MyVici0kxEponIYhH5QkSOrs1lRESu8d6Xz0TkWRFpWNvKiIg8ISI/iMhnvrCIZUIcD3p5s1BEuid7X1Me2cMu4DpV7QQcBVwuIp2AG4C3VbUD8LZ3XJu4CvjCd3w38DdVPQT4CfhtRqTKHA8Ar6tqR6ArLm9qZRkRkZbAlUCeqh4J5ADnU/vKyFPAqWFh0crEaUAHbxsFPJLsTU15ZAmqukZVP/b2N+EqhZbAQGCSF20S8KvMSFj9iEgr4Azgn96xAL8EpnlRalt+NAX6ABMBVHWHqv5MLS4juDWJGolIXaAxsIZaVkZU9V1gfVhwtDIxEPiXOj4EmonIAcnc15RHFiIibYFuwEfAL1R1jXfqO+AXGRIrE9wPjAVKvePmwM+quss7XoVTsLWFdsCPwJOeK++fIrIHtbSMqOpq4F7gG5zS2AAUULvLSIhoZaIlsNIXL+n8MeWRZYhIE+AF4GpV3eg/p25cda0YWy0iA4AfVLUg07JkEXWB7sAjqtoN2EKYi6qWlZG9cS3pdsCBwB5Udt/UetJVJkx5ZBEiUg+nOCar6ote8Pchs9L7/SFT8lUzxwJniUgRMAXningAZ2aHlk9uBazOjHgZYRWwSlU/8o6n4ZRJbS0jJwJfq+qPqroTeBFXbmpzGQkRrUysBg7yxUs6f0x5ZAmeP38i8IWq3uc7NQMY4e2PAF6ubtkygareqKqtVLUtrhP0HVUdDvwXGORFqzX5AaCq3wErReQwL6g/8Dm1tIzg3FVHiUhj7/0J5UetLSM+opWJGcBvvFFXRwEbfO6tQNgX5lmCiBwH/A/4lHIf/024fo/ngNa4KeXPU9XwzrHdGhHpC1yvqgNEpD3OEtkH+AT4tapuz6R81YmI5OIGENQHlgMX4hqBtbKMiMifgCG40YqfABfjfPi1poyIyLNAX9zU698DtwHTiVAmPCX7d5x7rxi4UFXzk7qvKQ/DMAwjKOa2MgzDMAJjysMwDMMIjCkPwzAMIzCmPAzDMIzAmPIwDMMwAmPKwzCqgIiUiEihb0vZpIQi0tY/U6phZBN140cxDCMGW1U1N9NCGEZ1Y5aHYaQBESkSkb+KyKciMk9EDvHC24rIO95aCm+LSGsv/Bci8pKILPC2Y7ykckTkcW/Nilki0siLf6W39stCEZmSocc0ajGmPAyjajQKc1sN8Z3boKqdcV/03u+FPQRMUtUuwGTgQS/8QWCOqnbFzVe1yAvvADysqkcAPwPneuE3AN28dEan6+EMIxr2hblhVAER2ayqTSKEFwG/VNXl3oSX36lqcxFZCxygqju98DWquq+I/Ai08k+j4U3N/6a3oA8i8gegnqreKSKvA5tx01BMV9XNaX5Uw6iAWR6GkT40yn4Q/HMylVDeT3kG8DDOSpnvm0XWMKoFUx6GkT6G+H4/8Pbn4mYJBhiOmwwT3FKhY6Bs3fam0RIVkTrAQar6X+APQFOgkvVjGOnEWiuGUTUaiUih7/h1VQ0N191bRBbirIehXtjvcCsB/h63KuCFXvhVwAQR+S3OwhiDWx0vEjnAvz0FI8CD3nK0hlFtWJ+HYaQBr88jT1XXZloWw0gH5rYyDMMwAmOWh2EYhhEYszwMwzCMwJjyMAzDMAJjysMwDMMIjCkPwzAMIzCmPAzDMIzA/D/Vq1q3B2Zg7gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "OlgOIbzVmEu4",
        "outputId": "e82ca5a7-a0d6-4472-f2fb-9ce0c97e79ec"
      },
      "source": [
        "def smooth_curve(points,factor=0.9):\n",
        "  smoothed_points=[]\n",
        "  for point in points:\n",
        "    if smoothed_points:\n",
        "      previous = smoothed_points[-1]\n",
        "      smoothed_points.append(previous*factor + point*(1-factor))\n",
        "    else:\n",
        "      smoothed_points.append(point)\n",
        "  return smoothed_points\n",
        "\n",
        "plt.clf()\n",
        "kullback_leibler_divergence = history.history['kullback_leibler_divergence']\n",
        "val_kullback_leibler_divergence = history.history['val_kullback_leibler_divergence']\n",
        "\n",
        "smooth_kl_history = smooth_curve(kullback_leibler_divergence)\n",
        "smooth_val_kl_history = smooth_curve(val_kullback_leibler_divergence)\n",
        "epochs = range(1,len(kullback_leibler_divergence)+1)\n",
        "\n",
        "plt.plot(epochs[10:],smooth_kl_history[10:],'ro',label='Training kullback_leibler_divergence')\n",
        "plt.plot(epochs[10:],smooth_val_kl_history[10:],'r',label='Validation kullback_leibler_divergence')\n",
        "plt.title('Training and validation smoothed kullback_leibler_divergence')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Kullback_leibler_divergence')\n",
        "plt.legend()\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAEWCAYAAADPZygPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZwU1bn/8c+XYV9kd2MbUBRRYMAB3IMat7gQFUUkiWii4nLVbCaaRWPiTYz+EhNjFoxbFAWjkYsJLle9LhEjzCio4AY4CK7sMA4IA8/vj1M9U9N0z3QP0/QM87xfr3p1V9WpqlPVVfXUOVVdR2aGc845lw8t8p0B55xzzZcHIeecc3njQcg551zeeBByzjmXNx6EnHPO5Y0HIeecc3nTbIKQpMclndfQafNJUpmkL+dgviZp3+j7nyX9JJO09VjORElP1TefTU1D/l6S7pH0izTjJkn6dz3mWRj9ni2j/uckfWtH5pnBMquWkYtpJJVLGhB9T7vNovH13pezkc3x1Ry0zHcGaiOpPNbbHvgC2Br1X2xmUzOdl5mdlIu0uzozm9wQ85FUCLwPtDKzymjeU4GMf8OmRNI9wHIz+3G+89KcmVnHfOehNg11fDVljToIxXcgSWXAt8zs6eR0klomTmzOObczNbbzjyQBMrNt+c5LJppkdZykMZKWS/qBpE+AuyV1lfRPSSskrYm+945Ns13VgqRborTvSzqpnmn7S3pB0gZJT0u6XdL9afKdSR5/LumlaH5PSeoRG/91SUslrZL0o1q2z2hJn0gqiA07XdLr0fdRkl6WtFbSx5L+IKl1mnnVqMKQ9P1omo8kXZCU9mRJr0laL2mZpOtjo1+IPtdGVSSHJlfxSDpM0lxJ66LPwzLdNkn56BFt27WSVkt6UVKLaFxZtA6vS/pc0p2S9lCogk38hl1j8zpN0oJoXs9JOiA27oBo2NoozWnR8IuAicDV0bo+FsteUbTsdZKmS2obm98pkuZF85staWhs3HBJr0Z5nA60JUOSbo724c5KqhKUdH26/TX1rPSHKO9vSzo2NuJ8SW9F+Vsi6eKkCcdG67Ze0mJJJ6aY+V7Rtvl+pusWTXdBtOw1kp6U1C82LrmKrYek/43y+Xw8bdI82ygc8x9I+lSh2qxdNG67808d+avtmKk6vqJ1OCU2rqXCuWJE1H9ItF+slTRf0phY2uck3SjpJaACGCDpeEnvRL/XH6P1/VZsmrq222RJ70XLu12SYuMvjP3eC2N53FvSI1G+35d0RW3bBgAzaxIdUAZ8Ofo+BqgEbgLaAO2A7sCZhGq7TsDfgRmx6Z8jlKQAJgFbgAuBAuAS4CPC1UO2aV8GbgFaA0cA64H706xDJnlcDOwXrdNzwK+icYOBcuCoaJ1/E22DL6dZ1mLguFj/34EfRt8PBg4hlIQLgbeAq2JpDdg3+n4P8Ivo+4nAp8BBQAfggaS0Y4AhhIuboVHar0bjCqO0LWPLmQT8O/reDVgDfD3K14Sov3td2ybFuv8S+DPQKuqOjP1eZcB/gD2AXsBnwKvAcMKJ/VnguijtfsDnwHHRfK4GFkW/davo+7VR/zHABmD/5O2WtA/PAfaO1vctYHI0bniUl9GE/ey8KH2baP5LgW9Hyx1H2Cd/kWb9JwH/jn6HO4AngfbJx1HUfz3R/pr8G7H9cVAZy8N4YB3QLRp/MrAPIOBLhBPhiGjcqCjtcVGeegGD4ssA+gPvAhdlcC6I52ts9DscQNhvfgzMrmVf3kD1MfQ7ov0vRdrfAjOj36kT8Bjwy3Tnn1ryWtcxcw/Vx9dPgamxaU8G3oq+9wJWAV+JtuFxUX/P2Db5ADgw2g49CeeiM6L+Kwn7TDbb7Z9AF6AvsAI4MRp3FvAhMDL6vfcF+kX5Ko3WozUwAFgCnFDr75mPgFKfju2D0GagbS3pi4A1aXbcScCi2Lj20UbfM5u00Y9TSXSAR+PvJ00QyjCPP471Xwo8EdtBp8XGdYi2Qbog9Avgruh7J8LJtF+atFcBj9Zy4CYOkruInfgJJ+mqtCnmeyvw2+h7IbUHoa8Dc5KmfxmYVNe2SbHcG4D/SZWvaD+aGOt/BPhTrP+/iC4MgJ8AD8XGtSAcfGMIge0ToEVs/IPA9cnbLWnZX4v1/xr4c/T9T8DPk9K/QzihH0XswicaNzt5/knb9RVgerR+rVMdR1H/9WQehJLzMAf4epo8zACujL7/JbEfpEj3HOGCqgyYkOFxE8/X48A3k36jCqJ9ne335fgx1JFwj7lPPC3hxPo5sE8s7aHA+9H3MdRx/olNV+sxQ83ja19CkExcMEwFfhp9/wFwX9K8nwTOi22TG2LjvgG8HOsXsCzL7XZEbPxDVF/EPpn4bZPyMxr4IGnYNcDdtW2jJlkdF1lhZpsSPZLaS/qLQnXVekL1TxfFqqSSfJL4YmYV0dd0NzHTpd0bWB0bBuGHTinDPH4S+14Ry9Pe8Xmb2eeEK6F0HgDOkNSGcDX0qpktjfKxn0J11SdRPv4bSFm1laRGHghX5/H1Gy3p/6Ki+DpgcobzTcx7adKwpYQrwIR02ybZzYSrvKeiqqEfJo3/NPZ9Y4r++DavypOFOvZlUZ72BpZZzXr35Pymkm4d+gHfjao+1kpaC/SJlrM38KFFR3VsWbXZl3C1+zMz21xH2kylysPeAJJOkvQfherPtYQr9sRv34dQik1nIiG4P1yPPPUDfhfbZqsJJ9x0v0P8GCqP0u+dlKYn4WKzNDbfJ6LhCTXOP7Wo9ZiJM7NFhNLxqZLaA6cRjmMI63lW0v5xBLBXqnVLXm70uy2Pjc9ku6XbV9P9nv2AvZPyeC2h1iGtphyELKn/u8D+wGgz241w9Qhhw+bKx0C3aIdJ6FNL+h3J48fxeUfL7J4usZktJOzwJwHnUr0zQ7jqfhsYGOXj2vrkgVASjHuAUIXRx8w6E6rEEvNN/r2SfUTYieP6Ek5OWTGzDWb2XTMbQDiQv6PY/Yss1MhTVCfeJ8rTR0AfRfeaUuS3rvVNtgy40cy6xLr2ZvYgYbv3itfJs/22T/YWcD7wuKT9Y8M/J5xgE/bMIo+p8vBRdKHzCKFaeg8z6wLMovq3X0aoqkvnemAl8EAtF43pLCM8KRvfbu3MbHaa9PFjqCOhuu2jpDQrCRcjB8bm2dlqPmmX6e9b1zGT7EFCVfRYYGEUmCCs531J69nBzH6VJk8fA/H7zYr3k/12i0v3ey4jlBbj8+xkZl+pbWZNOQgl60TYcdZK6gZcl+sFRiWLEuB6Sa0lHQqcmqM8PgycIukIhYcIbqDu3+8BQl3wUYR7QvF8rAfKJQ0i3OfKxEPAJEmDoyCYnP9OhJLhJkmjCMEvYQWwjVBPnMosYD9J50Y3ZMcT7oP9M8O8VVG4wb9vdOCtI1S51OdJoYeAkyUdK6kV4SLiC0JV2CuEq8OrJbWKbhKfCkyLpv2U9Ouayh3A5Kg0KUkdFB706ESolqwEroiWdQbhPkutogB2LfC0pMRJYx5wTjSfYsL9pUztHsvDWYT7CbMI9f9tCL9xpcKDO8fHprsTOD/aji0k9Yr2u4QthPsMHYC/JQX2uvwZuEbSgQAKD1+cVUv6r8SOoZ8D/zGzGrUXUen2DuC3knaP5ttL0glZ5CuhrmMm2TTCtruEmheO9xNKSCdIKpDUVuEBid4p5wL/AoZI+qrC/74uo+YFR7bbLe6vwPckHRztq/sqPNQwB9ig8MBGuyifB0kaWdvMdqUgdCvhhvVKwo3nJ3bScicS6otXEe7DTCecqFKpdx7NbAFhR3qAcJWzhprF61QeJNxTeNbMVsaGf48QIDYQDrbpGebh8WgdniVUdz2blORS4AZJGwj3sB6KTVsB3Ai8FBXVD0ma9yrgFMKJfhXhIYBTkvKdqYHA04QHOV4G/mhm/5ftTMzsHeBrwG2E3+xU4FQz2xxVcZ1KKGmuBP4IfMPM3o4mvxMYHK3rjAyWVUJ4+OUPhN92EeE+DNGyzoj6VxMeCvhHhutwL+GC5VmF/2r9hHAVuwb4GTVPdHV5hbBtVxJ+y3FmtsrMNgBXEH7vNYR9a2YsD3MIpbLfEi4Kniep1Btbxz2AuzINRGb2KOEBgWlR1fKbhN8knQcIgWA14QGdr6VJ9wPCb/CfaL5PE2oxspLBMZOc/mPCPnsYseMyCpRjCRcVKwilju+T5hweHTdnEe47riJc0JUQnZvqsd3i8/474fd/gHAOmUF4QGUr4RguIvwncCUhYHWubX6JJ4ZcA1F4fPZtM8t5Scw55zIRBfXlhIdysr4gy6VdqSSUF5JGStonqmY4kXC1UueVr3PO5VJUddclumeXuO/7nzxnazsehHbcnoTHI8uB3wOXmNlrec2Rc02cwp98U3VH5jtvySRdmyavj+c5a4cSnmJLVCV/1cw25jdL2/PqOOecc3njJSHnnHN506hfYNqQevToYYWFhfnOhnPONRmlpaUrzaxn3Snrr9kEocLCQkpKSvKdDeecazIk1fVmjh3m1XHOOefyxoOQc865vPEg5JxzLm88CDnnnMsbD0LOOefyJudBSNKJCk3MLtL27bokmtGdHo1/JXrJItGbeu+V9IZCM7LXZDpP55xzTUNOg5BC2yC3E97OOhiYIGlwUrJvEloX3Zfwlt2bouFnAW3MbAjhbbcXSyrMcJ7OOeeagFyXhEYRmsZeEr2qfRrhBZ9xY4F7o+8PA8dG7cAY0CFqC6MdoTnd9RnO0znnmi8zWLIE/vY3+O//znduapXrP6v2omaTs8sJ7ZCnTGNmlQrNQncnBKSxhLZz2gPfNrPVkjKZJwCSLgIuAujbt64GDZ1zrokwg3XrYM2a0H32Gbz/fgg8ixbBK6/Axx+HtD17wve/D61a5TfPaTTmNyaMIrSIuTfQFXhR0tPZzMDMpgBTAIqLi3f8Ta2ffw5vvx1+5EWL4IMPYPhwOP102KPWZtRdfVVUQHk5bNwYuoIC6NYNunQJ351rLrZuhRdegIcegn/8IwSeZG3bQmEhHH00HHEEHHkkDB4MLRrvM2i5DkIfUrN99d7RsFRplkdVb50JLQGeCzxhZluAzyS9BBQTSkF1zbNhbNsWirLz54du0aJwBZLQpQtMmQKXXhp+8DPPhDPOgD590s+zqdq2LWyDp56C0lL46KNwpfXZZ9C6Ney2G3TqFL5v2QKVlWFb9egBe+0Fe+4J7dqFA2nr1jB+48YQZDZurJ5m69YQ7D/9FD75JHxPp3PnMN/E/HfbLSyjbVvo0CFcAfbsGfJQWRnmlZj3e++F33PZsjDd7ruHbutWWLEidGvWVOd327awfon59ewZLjz22CMse599YN99oX376vyZwebN0KZN7n8ft+vZsiUca7Nnw0svwYsvhv2yfXs4+WQ45JBwQda1a9gn+/cP+2IjDjip5LQphyiovAscSwgUc4Fzo6aqE2kuA4aY2WRJ5wBnmNnZkn4ADDKz8yV1iKY9B1hY1zxTKS4utnq9O65v33BiHToUhg2Dgw6C/faDAQPCzvDmm/DII6F7880wzahRISCNGxfSNUWbN8O8efCf/4TumWeqr7z23Rd69w4n/z32CAfL+vWwYUOYrlWr0JmFg+aTT0LA+uKLUHopKICWLcP2a9cudK1ahWEFBWH4nntWn+Q7dqxOV1kZgsPq1bBqVXWw+vjjsPxNm0JQ27y59vXbbTcYODD8vuXlYd0++ywsPxG8unULeWrRInTr18PKlaH77LOw/OTjp2/fEKxWrgzjKyvDtjrooNANHw6HHhquVqWc/HQ5sX49vPpq+A0GDAgnvaaU/6Zm1iy48spwoQQhwBx+OIwdC1/5Ss2LnRySVGpmxTldRq7bE5L0FUIb6wXAXWZ2o6QbgBIzmympLXAfMJzQ7vs5ZrZEUkfgbsITcALuNrOb082zrnzUOwh98UXmV7LvvhuKyY88AollFRfD2WfDCSfAAQc02npZ5s+H558Pn/PmhYCaOJHvvTeMGRPW4bjjQvBp7DZvDoFgxYrw2apVOHA7dAgn0IY4iVZWhvl//HE4WbzzTugqKsL8u3cPJ+1334UFC+Ctt8L+BCG4Dh8egmEiX4WFoerkwAPDNt68OaQvL6+ex8KFIcgm5t+1a1i3RHBv3z4E7U6dQpeYd2J4fP/74ovq7bNxY3UAX7+++l7DsmXw8svwxhuhNJjQoUO4GDnwwNAddBAcdljIl6sfs1BC/+534Z//hP33h+uvD8fennvmJUu7RBBqLOodhOpr6VL4+99D/e3cuWFY69bhgB08uPpKu1u3UA20116h69kznCx2RpF6/Xp44AG4445wlQvhJFJUFLrRo0ORv3fv3OelOaisDCfz//wnnNgXLAgBq6IilOLWrKl7Hj16hP1j1aowTbZatw7Tb90abmzXpXPnsB8cdlj43LKl+gZ4IjB+8EF1+qIiOOYYOOqokD6XJ8/KyhCU588PnwsXhnu2lZUh+LdvH46jdetCV14eqmo7dQrBv3PnUKXetWv43qlT2DYdOoQu8b19+1AiTnQtWlRfwLRtW30cZ3KBaRZ+57feChd6b7wRLlyWLQtdRUVY7nXXwRVXhN8rjzwINaCdHoTiysrCSWfevNC9/XaoTiovTz9N+/bhoOjcORwkXbuGA3rAgND17w+9eoVhdZXUNm4Mdcsvvxx2/A8+CDv8Bx+Ek8qQIXDhheF+1t57ezVLvqxeHU5OCxeGasY2bULXrl11qWP33avTb94Ma9dW30vbsiX81uXlIUCVl1ffB0t0iWFSKI3tvnsoUXXoEE6obdqEE3Rin2vbtu58r18Pr78eStLPPBPuYSRKfH36hNqAwsLwvXfv8H2ffcL8M93Xtm4N22bu3NCVloZlbtoUxrdsGarJBw0K2ytxv3HbtnAMJe5ZbtoUts369dVPl61dG7ry8u2rV7Ox224hqHXpEpYZ33abN4f7qMuXh7wldOoUakj69g3bp2/fUHOy9971z0cD8iDUgPIahNLZsiUcBJ9+Gqp0PvooVI8kThYbNoSDI1E18tFHoUvWvXsIWmahk8IVVOvW4artvffCsiCcBBI7fGFhCDwjR3rgcQ1n48ZQsp4zJ3SvvRYueOInXwgn6r32CgFm8+YQTHffvfpCq1WrsO+++274rKgI0+22G4wYAQcfHD6LisL9vR2t6jYLQSoevMvLw3ITQb6yMgS2xHlz06ZQKk1069ZVB7X4fcmWLcO69u4dLh732y9c/PXt26iPPQ9CDahRBqH62LgxlKzef786cH30UTgYpNCZhQNm8+bwud9+oTrl0ENDdZ9zO1uiGmrZsrD/LlkCixeHC7BWrcIFU0FB2KeXLAlpKitDMBo4MOzDI0aEC6b99mtyT4A1VTsjCDXm/wm5VNq1C8X3Aw7Id06cy5xUfe9k2LC602/bFkofjfVBHtdgPAg55xqfxGPxbpfnv7Jzzrm88SDknHMubzwIOeecyxsPQs455/LGg5Bzzrm88SDknHMubzwIOeecyxsPQs455/LGg5Bzzrm88SDknHMubzwIOeecyxsPQs455/LGg5Bzzrm88SDknHMubzwIOeecyxsPQs455/LGg5Bzzrm88SDknHMubzwIOeecyxsPQs455/Im50FI0omS3pG0SNIPU4xvI2l6NP4VSYXR8ImS5sW6bZKKonHjJb0uaYGkm3K9Ds4553IjqyAkqZ2k/bNIXwDcDpwEDAYmSBqclOybwBoz2xf4LXATgJlNNbMiMysCvg68b2bzJHUHbgaONbMDgT0lHZvNejjnnGscMg5Ckk4F5gFPRP1FkmbWMdkoYJGZLTGzzcA0YGxSmrHAvdH3h4FjJSkpzYRoWoABwHtmtiLqfxo4M9P1cM4513hkUxK6nhBU1gKY2Tygfx3T9AKWxfqXR8NSpjGzSmAd0D0pzXjgwej7ImB/SYWSWgJfBfqkWrikiySVSCpZsWJFqiTOOefyKJsgtMXM1iUNs4bMTCqSRgMVZvYmgJmtAS4BpgMvAmXA1lTTmtkUMys2s+KePXvmOqvOOeeylE0QWiDpXKBA0kBJtwGz65jmQ2qWUnpHw1KmiUo2nYFVsfHnUF0KAsDMHjOz0WZ2KPAO8G4W6+Gcc66RyCYI/RdwIPAF8ACh2uyqOqaZCwyU1F9Sa0JASb6PNBM4L/o+DnjWzAxAUgvgbKrvBxEN3z367ApcCvw1i/VwzjnXSLTMNKGZVQA/irpMp6mUdDnwJFAA3GVmCyTdAJSY2UzgTuA+SYuA1YRAlXAUsMzMliTN+neShkXfbzAzLwk551wTpKjQUXdC6X+Bs8xsbdTfFZhmZifkMH8Npri42EpKSvKdDeecazIklZpZcS6XkU11XI9EAIKqBwR2b/gsOeecay6yCULbJPVN9Ejqx054Os4559yuK+N7QoR7Qf+W9Dwg4EjgopzkyjnnXLOQzYMJT0gaARwSDbrKzFbmJlvOOeeag2xKQgBtCE+wtQQGS8LMXmj4bDnnnGsOMg5C0duqxwMLgG3RYAM8CDnnnKuXbEpCXwX2N7MvcpUZ55xzzUs2T8ctAVrlKiPOOeean2xKQhXAPEnPEF7dA4CZXdHguXLOOdcsZBOEZrL9e9+cc865esvmEe17JbUD+prZOznMk3POuWYi1y2rOuecc2ntaMuqA3KQJ+ecc83Ejrasui1lSueccy4D2TyYUKNlVeAK6m5Z1TnnnEurvi2rPgisp+6WVZ1zzrm0ctqyqnPOOVebbN4d9xjbtx+0DigB/mJmmxoyY84553Z92b62pxy4I+rWAxuA/aJ+55xzLivZPJhwmJmNjPU/JmmumY2UtKChM+acc27Xl01JqGNS8959gY5R7+YGzZVzzrlmIZuS0HcIzXsvJjTv3R+4VFIH4N5cZM4559yuLaMgJKkF0AkYCAyKBr8Texjh1hzkzTnn3C4uo+o4M9sGXG1mX5jZ/Kjzp+Gcc87tkGzuCT0t6XuS+kjqluhyljPnnHO7vGyC0HjgMuAFoDTqSuqaSNKJkt6RtEjSD1OMbyNpejT+FUmF0fCJkubFum2SiqJxEyS9Iel1SU9I6pHFejjnnGskMg5CZtY/RVfrW7QlFQC3AycBg4EJkgYnJfsmsMbM9gV+C9wULW+qmRWZWRHwdeB9M5snqSXwO+BoMxsKvA5cnul6OOecazyyaU+ovaQfS5oS9Q+UdEodk40CFpnZEjPbDEwDxialGUv103UPA8dKUlKaCdG0EJ7ME9AhSrcb8FGm6+Gcc67xyKY67m7C/4EOi/o/BH5RxzS9gGWx/uXRsJRpzKyS8Cqg7klpxhNemoqZbQEuAd4gBJ/BwJ2pFi7pIkklkkpWrFhRR1adc87tbNkEoX3M7NfAFqh6oWlyiaXBSRoNVJjZm1F/K0IQGg7sTaiOuybVtGY2xcyKzay4Z8+euc6qc865LGUThDZLakf0ElNJ+xCadajNh0CfWH/vaFjKNNH9ns7Aqtj4c4hKQZEiADNbbGYGPER16cw551wTkm3z3k8AfSRNBZ4Brq5jmrnAQEn9JbUmBJSZSWlmAudF38cBz0bBJfEn2bOpvh8EIWgNlpQo2hwHvJXFejjnnGsksmlP6ClJpcAhhGq4K81sZR3TVEq6HHgSKADuMrMFkm4ASsxsJuF+zn2SFgGrCYEq4ShgmZktic3zI0k/A16QtAVYCkzKdD2cc841HooKHXUnDO0JPQDMNLPPc5qrHCguLraSkjr/1uSccy4iqdTMinO5jGyq424BjgQWSnpY0jhJbXOUL+ecc81ANtVxzwPPR39APQa4ELiL8D8d55xzLmvZNOVA9HTcqYT/7YzAm3Bwzjm3AzIOQpIeIrwB4QngD8Dz0du1nXPOuXrJpiR0JzDBzLbmKjPOOeealzqDkKRjzOxZoAMwNvm1bmb2jxzlzTnn3C4uk5LQl4BnCfeCkhngQcg551y91BmEzOy66PP83GfHOedcc5JJddx3ahtvZr9puOy45mbLli0sX76cTZu8tXjn8qVt27b07t2bVq1a7fRlZ1Id1yn63B8YSfW7304F5uQiU675WL58OZ06daKwsJDtm5FyzuWambFq1SqWL19O//79d/ryM6mO+xmApBeAEWa2Ieq/HvhXTnPndnmbNm3yAORcHkmie/fu5KvNtWxe27MHoVG7hM3RMOd2iAcg5/Irn8dgNv8T+hswR9KjUf9XgXsaPEfOOeeajYxLQmZ2I3A+sCbqzjezXybGS+ra8NlzLsnUqVBYCC1ahM+pU3dodqtWraKoqIiioiL23HNPevXqVdW/efPmWqctKSnhiiuuqHMZhx3WMG0u3nPPPVx++eX1Sn/99ddzyy23ADBmzBga4o3yhYWFrFxZa2suVTp27FhnmsR2eu655zjllFN2eJmZii9v5syZ/OpXv2rQ+bvaZfXuODN7FXg1zehnCO+Tcy43pk6Fiy6CiorQv3Rp6AeYOLFes+zevTvz5s0Dwom6Y8eOfO9736saX1lZScuWqQ+T4uJiiovrfsv97Nmz65W35iYX22nr1q0UFBRknP60007jtNNO2+Hl1rbfuJqyuSdUF6/Yd7n1ox9VB6CEioowvAFNmjSJyZMnM3r0aK6++mrmzJnDoYceyvDhwznssMN45513gJpX0Ndffz0XXHABY8aMYcCAAfz+97+vml+iFPDcc88xZswYxo0bx6BBg5g4cSKJ9rxmzZrFoEGDOPjgg7niiivSlgQS/vWvf3HooYeycuXKGqWDkpISxowZU+c63nfffRQVFXHQQQcxZ054yDXdem7dupXvfe97HHTQQQwdOpTbbrutxrw2btzISSedxB133FHncgFuvvlmRo4cydChQ7nuuuu2204A69ev5+STT2b//fdn8uTJbNu2/Wsq77//fkaNGkVRUREXX3wxW7durZrPd7/7XYYNG8bLL7+cMg9PPPEEgwYNYsSIEfzjH9X/t0+UHtetW0e/fv2qlvv555/Tp08ftmzZwuLFiznxxBM5+OCDOfLII3n77beB7febxYsXc8ghhzBkyBB+/OMf11i/VNugrKyMAw44gAsvvJADDzyQ448/no0bNwKwaNEivvzlLzNs2DBGjBjB4sWLa92WTUlDBqsnBywAACAASURBVKHMWsdzrr4++CC74Ttg+fLlzJ49m9/85jcMGjSIF198kddee40bbriBa6+9NuU0b7/9Nk8++SRz5szhZz/7GVu2bNkuzWuvvcatt97KwoULWbJkCS+99BKbNm3i4osv5vHHH6e0tLTOp5QeffRRfvWrXzFr1ix69OhRr/WrqKhg3rx5/PGPf+SCCy4ASLueU6ZMoaysjHnz5vH6668zMVbqLC8v59RTT2XChAlceOGFdS73qaee4r333mPOnDnMmzeP0tJSXnjhhe3SzZkzh9tuu42FCxeyePHiGoEC4K233mL69Om89NJLzJs3j4KCAqZGVbOff/45o0ePZv78+RxxxBHbzXvTpk1ceOGFPPbYY5SWlvLJJ59sl6Zz584UFRXx/PPPA/DPf/6TE044gVatWnHRRRdx2223UVpayi233MKll15aNV18v7nyyiu58soreeONN+jdu3dG2+C9997jsssuY8GCBXTp0oVHHnkEgIkTJ3LZZZcxf/58Zs+ezV577ZXxtmzsvLzomo6+fUMVXKrhDeyss86qqsZZt24d5513Hu+99x6SUgYXgJNPPpk2bdrQpk0bdt99dz799NMaJx+AUaNGVQ0rKiqirKyMjh07MmDAgKr/aEyYMIEpU6akXMazzz5LSUkJTz31FLvtVv+mvCZMmADAUUcdxfr161m7di0bNmxIuZ5PP/00kydPrqpe6tatW9V8xo4dy9VXX10jMNXmqaee4qmnnmL48OFACGLvvfceRx11VI10o0aNYsCAAVV5/fe//824ceOqxj/zzDOUlpYycuRIIJTGdt99dwAKCgo488wz0+bh7bffpn///gwcOBCAr33taym39/jx45k+fTpHH30006ZN49JLL6W8vJzZs2dz1llnVaX74osvqr7H95uXX36ZGTNmAHDuuedWVfOm2wZ9+/alf//+FBUVAXDwwQdTVlbGhg0b+PDDDzn99NOB8MfSbLZlY9eQQcir41xu3XhjzXtCAO3bh+ENrEOHDlXff/KTn3D00Ufz6KOPUlZWlra6q02bNlXfCwoKqKysrFea2uyzzz4sWbKEd999t+p+VMuWLauqjTJ980TyI7mSMl7PuMMPP5wnnniCc889N6PHfM2Ma665hosvvjjr/CXP57zzzuOXv/wlydq2bZvVfaB0TjvtNK699lpWr15NaWkpxxxzDJ9//jldunSpuo+YLL7fpJNuG5SVlW23fySq47KZT1OTUXWcpAJJb9eR7NgGyI9z6U2cCFOmQL9+IIXPKVPq/VBCptatW0evXr2AcM+goe2///4sWbKEsrIyAKZPn542bb9+/XjkkUf4xje+wYIFC4DwxFhpaSlAVfVNXRLL+Pe//03nzp3p3Llz2vU87rjj+Mtf/lIVMFevXl017oYbbqBr165cdtllGS33hBNO4K677qK8vByADz/8kM8++2y7dHPmzOH9999n27ZtTJ8+fbtqtWOPPZaHH364atrVq1ezNFUpOYVBgwZRVlZWdV/lwQcfTJmuY8eOjBw5kiuvvJJTTjmFgoICdtttN/r378/f//53IASC+fPnp5z+kEMOqfo9pk2blvU2SOjUqRO9e/euKlV98cUXVFRUZD2fxiqjIBS1IfSOpLT1Hma2Ot045xrMxIlQVgbbtoXPHAcggKuvvpprrrmG4cOHZ11yyUS7du344x//WHWzu1OnTnTu3Dlt+kGDBjF16lTOOussFi9ezHXXXceVV15JcXFxxiWAtm3bMnz4cCZPnsydd94JpF/Pb33rW/Tt25ehQ4cybNgwHnjggRrz+t3vfsfGjRu5+uqr61zu8ccfz7nnnsuhhx7KkCFDGDduHBs2bNgu3ciRI7n88ss54IAD6N+/f1VVVMLgwYP5xS9+wfHHH8/QoUM57rjj+PjjjzNe9ylTpnDyySczYsSIqmq8VMaPH8/999/P+PHjq4ZNnTqVO++8k2HDhnHggQfyP//zPymnvfXWW/nNb37D0KFDWbRoUdVvmuk2iLvvvvv4/e9/z9ChQznssMP45JNP6jWfxkiJp3PqTBhe2zOc8L64zxPDzWzHn2fcCYqLi60h/hvhGtZbb73FAQcckO9s5F15eTkdO3bEzLjssssYOHAg3/72t/OdLbcDKioqaNeuHZKYNm0aDz74YNqA1RikOhYllZpZ3f9D2AHZ3BP6Sc5y4Vwzd8cdd3DvvfeyefNmhg8f3uTr+R2UlpZy+eWXY2Z06dKFu+66K99ZapQyLgkBSOoHDDSzpyW1BwoSLzRt7Lwk1Dh5SWjXsWrVKo49dvtbw8888wzdu3fPQ47g9NNP5/33368x7KabbuKEE07IS34as0ZfEpJ0IXAR0A3YB+gF/Bl/IME5R823TzQWjz76aN2JXF5l82fVy4DDgfUAZvYekP6OXkTSiZLekbRI0g9TjG8jaXo0/hVJhdHwiZLmxbptkookdUoavlLSrVmsh3POuUYim3tCX5jZ5sTz+pJaUsdbEiQVALcDxwHLgbmSZprZwliybwJrzGxfSecANwHjzWwqMDWazxBghpklLrOKYssoBWr+ndo551yTkE1J6HlJ1wLtJB0H/B14rI5pRgGLzGyJmW0GpgFjk9KMBe6Nvj8MHKvt//U2IZq2Bkn7EUpjL2axHs455xqJbILQD4EVwBvAxcAs4Md1TNMLWBbrXx4NS5nGzCqBdUDyXczxQKp/lJ0DTLc0T1dIukhSiaSSfLUa6JxzLr1s2hPaZmZ3mNlZZjYu+p7zl5ZKGg1UmNmbKUafQ+rgBICZTTGzYjMr7tmzZ87y6Jquo48+mieffLLGsFtvvZVLLrkk7TTx9ni+8pWvsHbt2u3SxNvvSWfGjBksXFhdM/3Tn/6Up59+Opvsp+TtDnm7Q01JnfeEJL1BLfd+zGxoLZN/CPSJ9feOhqVKszy6z9QZWBUbnzLQSBoGtDSz0lpXwLlaTJgwgWnTptV4ZHfatGn8+te/zmj6WbNm1XvZM2bM4JRTTmHw4MFAeAWOq5u3O7RryaQkdApwai1dbeYCAyX1l9SaEFBmJqWZCZwXfR8HPJsoYUlqAZxNivtBhPtEaUtBrgm66ioYM6Zhu6uuqnWR48aN41//+ldVK6plZWV89NFHHHnkkVxyySUUFxdz4IEHpm2rJX7FfeONN7LffvtxxBFHVLXFA+GPqCNHjmTYsGGceeaZVFRUMHv2bGbOnMn3v/99ioqKWLx4MZMmTeLhhx8Gwn9rhg8fzpAhQ7jggguq3tRcWFjIddddx4gRIxgyZEhVWzbpeLtD3u5QbduyMagzCJnZ0kQHfAEMA4YSnpar9Y2B0T2ey4EngbeAh8xsgaQbJCUuI+4EuktaBHyHcO8p4ShgmZktSTH7s/Eg5HZQt27dGDVqFI8//jgQSkFnn302krjxxhspKSnh9ddf5/nnn+f1119PO5/S0lKmTZvGvHnzmDVrFnPnzq0ad8YZZzB37lzmz5/PAQccwJ133slhhx3Gaaedxs0338y8efPYZ599qtJv2rSJSZMmMX36dN544w0qKyv505/+VDW+R48evPrqq1xyySW1Vvl5u0Pe7lBTaHcomz+rfgv4KfAsodmG2yTdYGa1vovCzGYRHmKID/tp7Psm4Kzk6aJxzwGHpBk3INO8uybi1vz83StRJTd27FimTZtW9ULPhx56iClTplBZWcnHH3/MwoULGTo0de3ziy++yOmnn0779u0BalTVvPnmm/z4xz9m7dq1lJeX1/lv/XfeeYf+/fuz3377AXDeeedx++23c1VUqjvjjDOA0N5M8kk3wdsdCrzdocbf7lA2lZHfB4ab2SoASd2B2YC/EMk1aWPHjuXb3/42r776KhUVFRx88MG8//773HLLLcydO5euXbsyadKkjNvqSTZp0iRmzJjBsGHDuOeee3juued2KL+JNmdqa4/I2x2qno+3O9S42x3K5hHtVUD8PXEbqPkAgXNNUseOHTn66KO54IILqq78169fT4cOHejcuTOffvppVXVdOkcddRQzZsxg48aNbNiwgcceq/4L3YYNG9hrr73YsmVLVVUQhHZiUr16f//996esrIxFixYB4b7Ml770pazWydsdCrzdocbf7lCdQUjSdyR9B1gEvCLpeknXAf8B3s11Bp3bGSZMmMD8+fOrgtCwYcMYPnw4gwYN4txzz+Xwww+vdfoRI0Ywfvx4hg0bxkknnVRV/QPw85//nNGjR3P44YczaNCgquHnnHMON998M8OHD6860UG4Qr/77rs566yzGDJkCC1atGDy5MlZr5O3O+TtDjWFdofqfIt2FHDSMrOfNWiOcsTfot04+Vu0XXPXWNodarRv0W4qQcY555qi5t7uUCZ/Vr3VzK6S9Bgp/rTaVFpWdc41Dt7uUE1HHnlk2vtFzUEmT8fdF33W/g4S5+rJzDJ6YsrtGrzdocZnJ7yBLa1MquNKo8/nJbUD+prZO3VM5lxG2rZty6pVq+jevbsHIufywMxYtWpV1f+KdrZs/qx6KqE01BroL6kIuMGr49yO6N27N8uXL8ffcu5c/rRt27bG2xp2pmz+rHo9oX2g5wDMbJ6k/jnIk2tGWrVqRf/+vhs511xl82fVLWa2LmlY/ioSnXPONXnZlIQWSDoXKJA0ELiC8Noe55xzrl6yKQn9F3Ag4U3aDwLrgdrfk++cc87VIuOSkJlVAD+KOuecc26HZfJn1ZR/Uk3wp+Occ87VVyYlIf+TqnPOuZzI5M+qz2cyI0mPmFn6FqKcc865JNk8mFAXb+nUOedcVhoyCPl/hpxzzmWlIYOQc845l5WGDEL+9knnnHNZyTgISTo4xbBTYr0/aJAcOeecazayKQndIemgRI+kCcBPEv1m9lRDZsw559yuL5t3x40DHo7eH3ck8A3g+JzkyjnnXLOQcUnIzJYA5wD/AM4Ejk/xVu3tSDpR0juSFkn6YYrxbSRNj8a/IqkwGj5R0rxYty1qwwhJrSVNkfSupLcl+f+TnHOuCcrktT1vUPPx625AAfCKJMxsaC3TFgC3A8cBy4G5kmaa2cJYsm8Ca8xsX0nnADcB481sKjA1ms8QYIaZJdoE/hHwmZntJ6lFlCfnnHNNTCbVcafUnSStUcCiqBSFpGnAWCAehMYSGswDeBj4gyRZzUbPJwDTYv0XAIMAzGwbsHIH8uiccy5P6qyOM7OlZrYU2AtYHetfA+xZx+S9gGWx/uXRsJRpzKwSWAd0T0ozntB8BJK6RMN+LulVSX+XtEeqhUu6SFKJpBJvPto55xqfbJ6O+xNQHusvj4bllKTRQIWZvRkNagn0Bmab2QjgZdK8ZNXMpphZsZkV9+zZM9dZdc45l6VsglCNKrKoGqyu6rwPgT6x/t7RsJRpJLUEOgOrYuPPISoFRVYBFYQHJAD+DozIbBWcc841JtkEoSWSrpDUKuquBJbUMc1cYKCk/pJaEwLKzKQ0M4Hzou/jgGcTwS566OBsYveDonGPAWOiQcdS8x6Tc865JiKbIDQZOIxQclkOjAYuqm2C6B7P5cCTwFvAQ2a2QNINkhKN4d0JdJe0CPgOEH+M+yhgWeLBhpgfANdLeh34OvDdLNbDOedcI6GaD6HtuoqLi62kpCTf2XDOuSZDUqmZFedyGRm/MUFSW8J/eg4E2iaGm9kFOciXc865ZiCb6rj7CI9knwA8T3jIYEMuMuWcc655yCYI7WtmPwE+N7N7gZMJ94Wcc865eskmCG2JPtdGb9PuDOze8FlyzjnXXGTzFu0pkroSmm+YCXQk1pSDc845l62Mg5CZ/TX6+jwwIDfZcc4515xk07Jqd0m3Re9rK5V0q6Tkd7w555xzGcvmntA04DNCW0LjCG+unp6LTDnnnGsesrkntJeZ/TzW/wtJ4xs6Q84555qPbEpCT0k6R1KLqDub8Doe55xzrl4yaVl1A6FlVQFXEf60CqF11XLgeznLnXPOuV1anUHIzDrtjIw455xrfjIpCdXaVo+Zvdpw2XHOOdecZPJgwv+rZZwBxzRQXpxzzjUzmVTHHb0zMuKcc675yaQ67ozaxpvZP2ob75xzzqWTSXXcqbWMM8CDkHPOuXrJpDru/J2REeecc81PNi2r/jTVcDO7oeGy45xzrjnJ5o0Jn8e6rcBJQGEO8tR4TJ0KhYXQokX4nDo13zlyzrldisysfhNKbYAnzWxMg+YoR4qLi62kpCTzCaZOhYsugoqK6mESmEG/fnDjjTBxYsNn1DnnGglJpWZWnMtlZFMSStYe6N1QGWl0fvSjmgEIQgACWLoUzj8fevTwUpJzzu2AbO4JvUF4Gg7Ce+N6Arvu/aAPPqh9/JYtsGpV+L50aSg1gZeOnHMuC3WWhCT1ib6eQnhc+1TgeGBvoCxnOcu3vn2zS19RAV/7mpeKnHMuC5lUx/2vpEIzWxrrPgS+Dvwux/nLnxtvhPbts58uUSryQOScc3XKJAh9h9CW0MDEAEnXRMO/VNfEkk6U9I6kRZJ+mGJ8G0nTo/GvSCqMhk+UNC/WbZNUFI17LppnYtzuma1uFiZOhClTwkMIYaGZT1tRAeed5/eLnHOuDnUGITObBVwCPC7pIEm3EqrkjjKz5bVNK6kAuJ3wOPdgYIKkwUnJvgmsMbN9gd8CN0XLnWpmRWZWRCh1vW9m82LTTUyMN7PPMlrbbE2cCGVl4YGE++4LAUmC7t2hdevap926NUznJSPnnEsro6fjzOwZ4HzgOWAAcIyZrclg0lHAIjNbYmabgWnA2KQ0Y4F7o+8PA8dK2xU7JkTT5k8iIG3bBitXwl13VZeS6lJREZ62c845V0MmDyZskLQemAXsBhwLfBYbXptewLJY//JoWMo0ZlYJrAO6J6UZDzyYNOzuqCruJymCViLvF0kqkVSyYsWKOrKapURQuv/+zO4dLV3qVXPOOZckk+q4Tma2W/TZ2sw6xPp3y3UGJY0GKszszdjgiWY2BDgy6r6eJu9TzKzYzIp79uyZmwzG7x1JUFCQPq1XzTnnXA078mfVTHwI9In1946GpUwjqSXQGVgVG38OSaWg6Ok8zGwD8ACh2i9/4lV1995be8nIH+V2zrkquQ5Cc4GBkvpLak0IKDOT0swEzou+jwOetehdQpJaAGcTux8kqaWkHtH3VoT/L71JY5H8VF06XipyzrncBqHoHs/lwJPAW8BDZrZA0g2STouS3Ql0l7SI8Nh3/DHuo4BlZrYkNqwN8KSk14F5hJLUHblcj6wlSkZ1BSJ/lNs518zV+wWmTU3WLzBtCKleglobf0Gqc64RaewvMHV1ybRqLsFfkOqca2Y8COVato9yJyRekOp/eHXO7cI8CO0s2TzKnYo/Veec2wV5ENqZsnmUO52lS+HrXw+BzAOSc66J8yCULzvyglS/d+Sc20V4EMqnHXlBakJt946mTg2ByQOUc66R8ke0G6upU8NLT5curd/03bvDhg2weXP1MH8E3DmXBX9Euzmr71N1CatW1QxAkL4ar0eP1N8LC+HSS7005ZzLGS8JNQXxUlGiNJMv8dLUV74Cs2bBBx+E5tC9dOXcLsVLQi5oiHtHDSVemvrTn8Kn/5fJOVdPHoSamh1pXC/X0v2XyR+QcM6l4dVxu4pU76lr1Qp22y3cH9rZ1XiJ5fkDEs41WV4d5zKX/EaGfv3g7rtDaSkf1XiJgFfXAxJehdf0eMnWNSAPQruSeFVdWVnNEka6arxEUOreffvv/frBJZfU7w+1maqtOYt0J7vk4fEn+JKf7vMTZMNIbHMpvLEjfi8w3ZOWvv1dJsysWXQHH3ywuR10//1m/fqZSeHzkkvCZzgdNUzXvn1Yzv33h+/xcVLNz0y6Vq3MuncP03TvXv09nv9E//33176e2cyjrm2XLl1jlOq3yLRL/FZNbZ2dmZkBJZbjc3Peg8PO6jwI5dCOnKTSdQUFDTu/bE6Y2QS5dPOIB6vu3c1at06dLptAVtdv0JCBMj6Phvot4hcEHpSaBA9CDdh5EMqxxAks1Uk8cfLZ0RN8c+myLT1kexGQaaDcmevZEIHY1dQApW8PQg3YeRDaiWrb+ePj8lHaaWpduurEXJRUGlOXSYCqrdp0Vw5kmZR649sw0SWqurPgQagBOw9CjVAuqvG82zldvDSVrxJUXaXqdIEs+cTdEMEqXWBoqGXVVtOQTdevX1aL9SDUgJ0HoUYqfnCl6hJX+ckHXiYnmHycIHflrqCg7tJtYvvv6MlyZ3b1qRZMXudM97NM74vVd/6ZrGsWPAg1YOdBqJFLVSqKVx/Ut367vlUXmV5F1zaPVCefpnRyTvdbZPOb5uJEujO7VPtBQ/6GuZ5/cuclIQ9Crhb5fIy5IZZdW8BLV3JoyJNPoqSyo4Ey11VWuTzJepe+83tCHoScq1N9Sw+ZnmAyDZS5lqtA7F3Nbgf/p+VBqAE7D0KuScr1De/GKJsAtbOrsxpLl+ox+xw8IbgzgpC/wNQ513Qk2taqqw2r5HTxtq+6dQtpGvLFvomXBa9eXT3/VN+TX+Zbn/nvxLa7dsYLTHNeAgFOBN4BFgE/TDG+DTA9Gv8KUBgNnwjMi3XbgKKkaWcCb2aSDy8JOee2syPVgvWp6srmvlgjeOURTb06DigAFgMDgNbAfGBwUppLgT9H388BpqeYzxBgcdKwM4AHPAg553Ii13+GbQJ/tt0ZQSin1XGSDgWuN7MTov5rAMzsl7E0T0ZpXpbUEvgE6GmxjEn67zCZ/Sjq7wg8AVwEPGRmB9WVF6+Oc8657OwK7Qn1ApbF+pdHw1KmMbNKYB3QPSnNeODBWP/Pgf8HVFALSRdJKpFUsmLFiuxz75xzLqcafXtCkkYDFWb2ZtRfBOxjZo/WNa2ZTTGzYjMr7tmzZ66z6pxzLku5DkIfAn1i/b2jYSnTRNVxnYFVsfHnULMUdChQLKkM+Dewn6TnGjTXzjnndopcB6G5wEBJ/SW1JgSUmUlpZgLnRd/HAc8m7gdJagGcDUxLJDazP5nZ3mZWCBwBvGtmY3K6Fs4553KiZS5nbmaVki4HniQ8KXeXmS2QdAPhqYuZwJ3AfZIWAasJgSrhKGCZmS3JZT6dc87lR7P5s6qkFcDSfOejHnoAK/OdiUbEt0dNvj2259ukph3ZHv3MLKc31JtNEGqqJJXk+hHJpsS3R02+Pbbn26Smxr49Gv3Tcc4553ZdHoScc87ljQehxm9KvjPQyPj2qMm3x/Z8m9TUqLeH3xNyzjmXN14Scs45lzcehJxzzuWNB6FGQlIfSf8naaGkBZKujIZ3k/S/kt6LPrvmO687k6QCSa9J+mfU31/SK5IWSZoevYmj2ZDURdLDkt6W9JakQ5vzPiLp29Hx8qakByW1bW77iKS7JH0m6c3YsJT7hILfR9vmdUkj8pfzwINQ41EJfNfMBgOHAJdJGgz8EHjGzAYCz0T9zcmVwFux/puA35rZvsAa4Jt5yVX+/A54wswGAcMI26ZZ7iOSegFXAMVRcy4FhDeuNLd95B5C46Fx6faJk4CBUXcR8KedlMe0PAg1Emb2sZm9Gn3fQDi59ALGAvdGye4FvpqfHO58knoDJwN/jfoFHAM8HCVpbtujM+FVVncCmNlmM1tLM95HCK8eaxe9/Lg98DHNbB8xsxcIrzyLS7dPjAX+FrVZ9x+gi6S9dk5OU/Mg1AhJKgSGE5o738PMPo5GfQLskads5cOtwNWEpt0htDO1Nmp3ClK3T7Ur6w+sAO6Oqij/KqkDzXQfMbMPgVuADwjBZx1QSvPeRxLS7ROZtPG2U3kQamSiVmMfAa4ys/XxcdHbxZvFM/WSTgE+M7PSfOelEWkJjAD+ZGbDgc9JqnprZvtIV8KVfX9gb6AD21dLNXuNfZ/wINSISGpFCEBTzewf0eBPE8Xl6POzfOVvJzscOC1qN2oaoYrld4Tqg8Tb31O1T7UrWw4sN7NXov6HCUGpue4jXwbeN7MVZrYF+Adhv2nO+0hCun0ikzbedioPQo1EdL/jTuAtM/tNbFS8vaXzgP/Z2XnLBzO7xsx6R+1GnUNoZ2oi8H+EdqegGW0PADP7BFgmaf9o0LHAQprpPkKohjtEUvvo+Elsj2a7j8Sk2ydmAt+InpI7BFgXq7bLC39jQiMh6QjgReANqu+BXEu4L/QQ0JfQFMXZZpZ8E3KXJmkM8D0zO0XSAELJqBvwGvA1M/sin/nbmaLm7f8KtAaWAOcTLiab5T4i6WfAeMLTpa8B3yLc42g2+4ikB4ExhCYbPgWuA2aQYp+IgvUfCNWWFcD5ZlaSj3wneBByzjmXN14d55xzLm88CDnnnMsbD0LOOefyxoOQc865vPEg5JxzLm88CDm3AyRtlTQv1jXYy0MlFcbfjOzcrqhl3Umcc7XYaGZF+c6Ec02Vl4ScywFJZZJ+LekNSXMk7RsNL5T0bNSWyzOS+kbD95D0qKT5UXdYNKsCSXdEbeY8JaldlP6KqO2p1yVNy9NqOrfDPAg5t2PaJVXHjY+NW2dmQwj/UL81GnYbcK+ZDQWmAr+Phv8eeN7MhhHeB7cgGj4QuN3MDgTWAmdGw38IDI/mMzlXK+dcrvkbE5zbAZLKzaxjiuFlwDFmtiR6Me0nZtZd0kpgLzPbEg3/2Mx6SFoB9I6/XiZq0uN/o4bJkPQDoJWZ/ULSE0A54fUsM8ysPMer6lxOeEnIudyxNN+zEX/n2Vaq7+OeDNxOKDXNjb012rkmxYOQc7kzPvb5cvR9NuGt4AATCS+thdAE8yUAkgqiVlRTktQC6GNm/wf8AOgMbFcac64p8Ksn53ZMO0nzYv1PmFniMe2ukl4nlGYmRMP+i9Ay6vcJraSeHw2/Epgi6ZuEEs8lhNZCUykA7o8ClYDfR818O9fk+D0h53IguidUbGYr850X5xozr45zzjmXN14Scs45lzdeEnLOOZc3HoScc87ljQch55xzeeNByDnnub1pZQAAAA9JREFUXN54EHLOOZc3/x8SdlobGoySUgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 622
        },
        "id": "lo_D7A759H10",
        "outputId": "b784eaca-9084-4f5a-b26a-d762b3c21fe2"
      },
      "source": [
        "def smooth_curve(points,factor=0.9):\n",
        "  smoothed_points=[]\n",
        "  for point in points:\n",
        "    if smoothed_points:\n",
        "      previous = smoothed_points[-1]\n",
        "      smoothed_points.append(previous*factor + point*(1-factor))\n",
        "    else:\n",
        "      smoothed_points.append(point)\n",
        "  return smoothed_points\n",
        "\n",
        "plt.clf()\n",
        "kullback_leibler_divergence = history.history['kullback_leibler_divergence']\n",
        "val_kullback_leibler_divergence = history.history['val_kullback_leibler_divergence']\n",
        "kullback_leibler_divergence_baseline = history_baseline.history['kullback_leibler_divergence']\n",
        "val_kullback_leibler_divergence_baseline = history_baseline.history['val_kullback_leibler_divergence']\n",
        "\n",
        "smooth_kl_history = smooth_curve(kullback_leibler_divergence)\n",
        "smooth_val_kl_history = smooth_curve(val_kullback_leibler_divergence)\n",
        "smooth_kl_history_baseline = smooth_curve(kullback_leibler_divergence_baseline)\n",
        "smooth_val_kl_history_baseline = smooth_curve(val_kullback_leibler_divergence_baseline)\n",
        "epochs = range(1,len(kullback_leibler_divergence)+1)\n",
        "\n",
        "plt.plot(epochs[1:],smooth_kl_history[1:],'ro',label='Training kullback_leibler_divergence')\n",
        "plt.plot(epochs[1:],smooth_val_kl_history[1:],'r',label='Validation kullback_leibler_divergence')\n",
        "plt.plot(epochs[1:],smooth_kl_history_baseline[1:],'bo',label='Training kullback_leibler_divergence baseline')\n",
        "plt.plot(epochs[1:],smooth_val_kl_history_baseline[1:],'b',label='Validation kullback_leibler_divergence baseline')\n",
        "plt.title('Training and validation smoothed kullback_leibler_divergence')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Kullback_leibler_divergence')\n",
        "plt.legend()\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-112-f2976af922d4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msmooth_kl_history\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'ro'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Training kullback_leibler_divergence'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msmooth_val_kl_history\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'r'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Validation kullback_leibler_divergence'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msmooth_kl_history_baseline\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'bo'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Training kullback_leibler_divergence baseline'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msmooth_val_kl_history_baseline\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'b'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Validation kullback_leibler_divergence baseline'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Training and validation smoothed kullback_leibler_divergence'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/pyplot.py\u001b[0m in \u001b[0;36mplot\u001b[0;34m(scalex, scaley, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2761\u001b[0m     return gca().plot(\n\u001b[1;32m   2762\u001b[0m         *args, scalex=scalex, scaley=scaley, **({\"data\": data} if data\n\u001b[0;32m-> 2763\u001b[0;31m         is not None else {}), **kwargs)\n\u001b[0m\u001b[1;32m   2764\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2765\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axes/_axes.py\u001b[0m in \u001b[0;36mplot\u001b[0;34m(self, scalex, scaley, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1645\u001b[0m         \"\"\"\n\u001b[1;32m   1646\u001b[0m         \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcbook\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormalize_kwargs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmlines\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLine2D\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1647\u001b[0;31m         \u001b[0mlines\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_lines\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1648\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlines\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1649\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_line\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axes/_base.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    214\u001b[0m                 \u001b[0mthis\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m                 \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 216\u001b[0;31m             \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_plot_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mthis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    217\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_next_color\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axes/_base.py\u001b[0m in \u001b[0;36m_plot_args\u001b[0;34m(self, tup, kwargs)\u001b[0m\n\u001b[1;32m    340\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    341\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 342\u001b[0;31m             raise ValueError(f\"x and y must have same first dimension, but \"\n\u001b[0m\u001b[1;32m    343\u001b[0m                              f\"have shapes {x.shape} and {y.shape}\")\n\u001b[1;32m    344\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: x and y must have same first dimension, but have shapes (99,) and (199,)"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAfUklEQVR4nO3df5DcdZ3n8ecrM8mERAgwGRDya4IENIoEaBLRPUov5xa4W2ZLIyQbEZWVO5U99axSWOrWlV3OYtc78Goprazg5UIWskbUrLUrpbJ7q54iHY1AQHAI5AcEM5mQQJgkk0ne98fn2zdN0zPT87Nn+vt6VH1rur/fT3/n8+1v9/f1/Xy+329/FRGYmVn+TKl3BczMrD4cAGZmOeUAMDPLKQeAmVlOOQDMzHKqud4VGIrZs2dHe3t7vathZjapbNmyZV9EtFWOn1QB0N7eTrFYrHc1zMwmFUk7qo13F5CZWU45AMzMcsoBYGaWUw4AM7OccgCYmeWUA8DMLKccAGZmOZWPALjrLvinf6p3LczMJpTGD4Bjx+DOO2H1anjiiXrXxsxswmj8AJg6Fb7zHZg+Hd77XnjxxXrXyMxsQmj8AACYPx++/W3YsQOuvhp6e+tdIzOzustHAAC8/e3wta/BD34An/kM+FaYZpZzk+rH4Ebsox9NxwG+/GWYPRu+8IV618jMrG7yFQAAt90GXV3wF38Bs2bBpz9d7xqZmdVF/gJgyhRYuxZeeil1Bb3udfAnf1LvWpmZjbv8BQBAczNs2ACHDsHHPpYODn/xiykczMxyIr9bvJYW+O5303GBv/orWLkyBYKZWU7kNwAghcDXvw533JHC4LLLfLGYmeVGvgMAQIJPfQq+/3343e/gkktSKPg0UTNrcA6Akne/G379a3jHO9Jxgauugn376l0rM7Mx4wAod9ZZ8MAD6VTR734X3vxmuP/+etfKzGxM1BQAkq6Q9KSkDkk3VpneImljNv0hSe3Z+KmS1kl6VNITkm6qdZ51M2UKfO5zUCzC3Lnw/venn4/Yu7feNTMzG1WDBoCkJuBO4EpgMbBa0uKKYtcBL0bEucDtwG3Z+A8ALRFxAXAJ8B8ltdc4z/p661vh5z+HW29NPyb3pjfB+vU+NmBm46+nZ0xmW0sLYCnQERHbI6IHuA9YUVFmBbAue7wJWC5JQAAzJTUDJwE9wEs1zrP+pk6FP/sz+NWv4I1vhA99CK68Ep56qt41M7NG1dsLzzyTdjw/+1lYuhTa2sYkBGq5EGwOsKvs+W5gWX9lIqJX0kGglRQGK4A9wAzgMxGxX1It8wRA0vXA9QDz58+vobpjYPFi+PGP4atfhZtuSscGPvEJ+PM/h9bW2ubR05NW6vTpsGDB2NbXzCa+7dth06Z06vn+/emn6p9/Pl2YWvrF4paWFAA33ABHjsC0aaNahbG+EngpcBw4GzgN+LGkHw5lBhGxFlgLUCgU6tf/MmUKfPKT6YKxL3wB/vZvYd06eNe7UkC86U3pB+amT08rbdcu2Lo1nVn0xBNppZ44keZ18cXwgQ+keZ17bt0WaVSdOJFaSo8/Dnv2pOHIETjllDRMn55uznPsWHovzzwzHXRva0vdasePpw/9kSPQ3Q2HD6eypfFdXWmeL7yQfsbj8OFUbto0OP30NMyeneb5+tenYD7ppDTMnJn+T2srNDWl+h47BgcOwNNPw29/m9bPySfDGWekcq+8ko77dHbC0aOpHidOpHnNnp3md8YZaTle//q0jFJ914FNbIcPw8MPw09+kn6evlhM4+fMSZ+500+HQiEdczznnNTrUCik7ckYqSUAngPmlT2fm42rVmZ31t0zC+gC/hj4fkQcA/ZK+ilQIO39DzbPienMM9PPSt9wA3zpS/DLX8L3vlf9HgPNzSkYli2DD34wbez37k2pf9NNabjwwnSgeeXKVHayiEgbzp/9DH74w3T2VGdn3/SZM2HGDHj55bRRHw2nnJLe/1NPTfNubU0tq1LYdnYO/L+ktJEvhUutpkzpG/prhp98Mpx3Hpx/fgqG/fvTacQ9PemLfMEFabjoohSGE93Ro2l5p06td00mv2IRPv/51ItQ+txdcgn89V+nHcH29rpVTTHIQc1sg/4UsJy0kX4Y+OOI2FZW5pPABRHxnyStAt4XEVdJ+jzwxoj4iKSZ2WtXAY8PNs9qCoVCFEupOZH09KQ9yYMH0wboyJG0oVq8uP/03rkznWK6aRP89Kdp3AUXpPRfuTJtTCbSHuUzz6S9l61b057+L36RNnKQ9oZ///fhiivg0kvh7LPTBrGkpye9J1OnpuH48XTR3Z49aSMppT3zpqa0YS/tuU+d2jf+9NNTqAwkIrUO9uxJdTtyJG3sX345/Z+9e9M6mjEjzeuUU2DhQli0KH0Ju7tTmX370o8EtrWlvf3yddjTk1ojnZ2p7AsvpGXZsQOefBJ+85s0vbU1vbapKbUASz8zMnVq+vIvXZrKzJyZhje8IXUtnnVW33qPSPPfti21rI4eTfOcPTvVr7k5zb+lJT0/+eT0d+bMvpZOtfeouzsNpffnwIHU/bBvX1q3P/0pbNmS1tP8+WlvdNEieMtb0nDhhXDaacP6GOVKVxfcfHP68ckzzoAPfzhdZ/T2t9fedTxKJG2JiMJrxg8WANmL3wPcATQBd0fErZJuAYoRsVnSdGA9cBGwH1gVEdslvQ74BulMHwHfiIi/6W+eg9VjwgbASD3/PHzrW7BxY18YnHoqLFmSvmxnntnXzXHmmWkDe9ZZaUM2liFx+HCq19q1ae8F+lo1S5emn85429vSc/+QXv9OnEiBv3VrajH97GdpA9vd/dqype6yo0fTBvro0eH9z1IolIaWlrRB2rs3rdeBXnfppWkj1dKS+qmffjoF24EDqYyUQmz58tQFumxZ+ryOtuPH0/995JEUgI8/noK3tIPQ1JQC/+DBtEylUD/llFSf005LQ/n7MHNm39+WlvR5bm5On18pDSedlDbQtbZ+9u+HRx9N9dy2Le0M7NqV3rueHvjTP+37+fk6GVEATBQNGwDldu6Ef/7ntCf2q1+lD9Qrr1Qv29zct9c3a1bfB/7ss9Me5TnnpAPOc+akvdmB9gqffjqFT7GYPsA7d6Zxhw6lrquPfSxdLT1Qq8aGprc3hcBLL6XutG3b0gbv2LH0Hre0wLx5qWXw5jenjVZXV9pTP3QobSCPH09B8fLLaTh0KH1eyodDh1KZ1ta0J1pqQUyfnobyjeU551RfvxGpZfXoo+n06B/9KAVZqevz/PNTKLS3pzrPm5fmtXBhbV1eEdDRkVqZv/hFX2uzFJJTpqTP4dln9x0nOn48bexnzUob7VdeSe/lSy+lFs2BAwOH3WBOPjm9Z6eemobyFuiRI2nHbffu9L6XnHZaWub589N376MfTaeU15kDYDI7fDh9oLu6UpfAnj3pw/fSS31f/IMH+84k2L07/S3X1JS+KBFpkNIB1GnT0pepqyuVO/nk9AGeNy99md/3PnjnO72Hb6916BA89FDf8Otfp8/e8eN9ZaR0kFxKe8O9van1es456fO1f386rfqpp/o2pDNmpGMll1ySDoJeeGEKmOHseBw9mupZGkqBeOhQCtre3vT3xIm+70bp+9DVlep38GAKk0OH+lrcU6emHau5c9N35S1vSV245V14E4gDIG8OHEhN0F274LnnUmAcONDXzI1IX8ientSSuPTS1D/p7hwbiePH007Kjh193Uc7d6bP1LRp6e/zz6fxzz6bujXPOy8dY7jwwtS1uHhx+kzaqHEAmJnlVH8B4F09M7OccgCYmeWUA8DMLKccAGZmOeUAMDPLKQeAmVlOOQDMzHLKAWBmllMOADOznHIAmJnllAPAzCynHABmZjnlADAzyykHgJlZTjkAzMxyygFgZpZTDgAzs5xyAJiZ5ZQDwMwsp2oKAElXSHpSUoekG6tMb5G0MZv+kKT2bPwaSVvLhhOSlmTTrpb0iKRtkm4bzYUyM7PBDRoAkpqAO4ErgcXAakmLK4pdB7wYEecCtwO3AUTEhohYEhFLgGuAZyJiq6RW4G+A5RHxZuD1kpaP2lKZmdmgamkBLAU6ImJ7RPQA9wErKsqsANZljzcByyWposzq7LUA5wC/jYjO7PkPgfcPtfJmZjZ8tQTAHGBX2fPd2biqZSKiFzgItFaUuRq4N3vcAZwvqV1SM/BHwLxq/1zS9ZKKkoqdnZ3VipiZ2TCMy0FgScuA7oh4DCAiXgQ+DmwEfgw8Cxyv9tqIWBsRhYgotLW1jUd1zcxyoZYAeI5X753PzcZVLZPt0c8Cusqmr6Jv7x+AiPjHiFgWEZcBTwJPDa3qZmY2ErUEwMPAIkkLJU0jbcw3V5TZDFybPV4JPBgRASBpCnAVff3/ZOPPyP6eBnwC+PpwF8LMzIauebACEdEr6QbgAaAJuDsitkm6BShGxGbgLmC9pA5gPykkSi4HdkXE9opZf0XShdnjWyLCLQAzs3GkbEd9UigUClEsFutdDTOzSUXSlogoVI73lcBmZjnlADAzyykHgJlZTjkAzMxyygFgZpZTDgAzs5xyAJiZ5ZQDwMwspxwAZmY55QAwM8spB4CZWU45AMzMcsoBYGaWUw4AM7OccgCYmeWUA8DMLKccAGZmOeUAMDPLKQeAmVlOOQDMzHLKAWBmllMOADOznKopACRdIelJSR2SbqwyvUXSxmz6Q5Las/FrJG0tG05IWpJNWy3pUUmPSPq+pNmjuWBmZjawQQNAUhNwJ3AlsBhYLWlxRbHrgBcj4lzgduA2gIjYEBFLImIJcA3wTERsldQMfAV4V0S8FXgEuGG0FsrMzAZXSwtgKdAREdsjoge4D1hRUWYFsC57vAlYLkkVZVZnrwVQNszMyp0CPD+M+puZ2TDVEgBzgF1lz3dn46qWiYhe4CDQWlHmauDerMwx4OPAo6QN/2Lgrmr/XNL1koqSip2dnTVU18zMajEuB4ElLQO6I+Kx7PlUUgBcBJxN6gK6qdprI2JtRBQiotDW1jYe1TUzy4VaAuA5YF7Z87nZuKplsv79WUBX2fRVZHv/mSUAEfF0RATwD8Dbh1RzMzMbkVoC4GFgkaSFkqaRNuabK8psBq7NHq8EHsw27EiaAlxFX/8/pMBYLKm0S/9u4InhLYKZmQ1H82AFIqJX0g3AA0ATcHdEbJN0C1CMiM2k/vv1kjqA/aSQKLkc2BUR28vm+bykLwL/JukYsAP48GgtlJmZDU7ZjvqkUCgUolgs1rsaZmaTiqQtEVGoHO8rgc3McsoBYGaWUw4AM7OccgCYmeWUA8DMLKccAGZmOeUAMDPLKQeAmVlOOQDMzHLKAWBmllMOADOznHIAmJnllAPAzCynHABmZjnlADAzyykHgJlZTjkAzMxyygFgZpZTDgAzs5xyAJiZ5ZQDwMwspxwAZmY5VVMASLpC0pOSOiTdWGV6i6SN2fSHJLVn49dI2lo2nJC0RNLJFeP3SbpjdBfNzMwG0jxYAUlNwJ3Au4HdwMOSNkfE42XFrgNejIhzJa0CbgOujogNwIZsPhcA34mIrdlrlpT9jy3A/aOxQGZmVptaWgBLgY6I2B4RPcB9wIqKMiuAddnjTcBySaooszp77atIOg84A/jxUCpuZmYjU0sAzAF2lT3fnY2rWiYieoGDQGtFmauBe6vMfxWwMSKi2j+XdL2koqRiZ2dnDdU1M7NajMtBYEnLgO6IeKzK5FVUDwYAImJtRBQiotDW1jZmdTQzy5taAuA5YF7Z87nZuKplJDUDs4CusulVN/KSLgSaI2LLEOpsZmajoJYAeBhYJGmhpGmkjfnmijKbgWuzxyuBB0tdOpKmAFdRpf+fdFyg371/MzMbO4OeBRQRvZJuAB4AmoC7I2KbpFuAYkRsBu4C1kvqAPaTQqLkcmBXRGyvMvurgPeMdCHMzGzo1M+x1wmpUChEsVisdzXMzCYVSVsiolA53lcCm5nllAPAzCynHABmZjnlADAzyykHgJlZTjkAzMxyygFgZpZTDgAzs5xyAJiZ5ZQDwMwspxwAZmY55QAwM8spB4CZWU45AMzMcsoBYGaWU40fABs2QHs7TJmS/m7YUO8amZlNCIPeEWxS27ABrr8eurvT8x070nOANWvqVy8zswmgsVsAN9/ct/Ev6e5O483Mcq6xA2DnzqGNNzPLkcYOgPnzhzbezCxHGjsAbr0VZsx49bgZM9J4M7Oca+wAWLMG1q6FBQtAgtZWOOkkuOYanxFkZrlXUwBIukLSk5I6JN1YZXqLpI3Z9IcktWfj10jaWjackLQkmzZN0lpJT0n6jaT3j+aC/X9r1sCzz8L69XD4MHR1QUTfGUEOATPLqUEDQFITcCdwJbAYWC1pcUWx64AXI+Jc4HbgNoCI2BARSyJiCXAN8ExEbM1eczOwNyLOy+b7f0ZjgfrlM4LMzF6llhbAUqAjIrZHRA9wH7CioswKYF32eBOwXJIqyqzOXlvyUeBLABFxIiL2DbXyQ+IzgszMXqWWAJgD7Cp7vjsbV7VMRPQCB4HWijJXA/cCSDo1G/eXkn4p6ZuSzqz2zyVdL6koqdjZ2VlDdfvR35k/ET4eYGa5NC4HgSUtA7oj4rFsVDMwF/i/EXEx8DPgy9VeGxFrI6IQEYW2trbhV6LaGUElPh5gZjlUSwA8B8wrez43G1e1jKRmYBbQVTZ9Fdnef6YL6Abuz55/E7i45loPR/kZQdX4eICZ5UwtAfAwsEjSQknTSBvzzRVlNgPXZo9XAg9GRABImgJcRVn/fzbtH4F3ZqOWA48PcxlqVzoj6DWHJzI+HmBmOTLoj8FFRK+kG4AHgCbg7ojYJukWoBgRm4G7gPWSOoD9pJAouRzYFRHbK2b9+ew1dwCdwEdGvjg1mj8/dftUG29mlhPKdtQnhUKhEMViceQzqvyVUEitgojURXTrrf61UDNrGJK2REShcnxjXwncn8rjAaWNP/iAsJnlRj4DAPqOByxY0LfxL/EBYTPLgfwGQIkvEDOznHIA+AIxM8spB4AvEDOznHIA+AIxM8spBwD4AjEzyyUHQDkfDzCzHHEAlPPxADPLEQdAOR8PMLMccQBUGux4wI4dMGWKu4TMbNJzAPRnoB+G8z2FzawBOAD6M9DxgBJ3CZnZJOYA6E/58YD+uoMgtQTcHWRmk5ADYCCl4wEnTvR/YBjcHWRmk5IDoFaDdQl1d8MHP+jWgJlNGg6AWg12imiJWwNmNkk4AIai/B4CA3FrwMwmAQfAcNRyhhCk1sA116SDyA4DM5tgHADDUWt3ELz6VpMOAzObQBwAw1XqDrrnntpaA+AwMLMJxQEwUkNpDZTzTejNrM5qCgBJV0h6UlKHpBurTG+RtDGb/pCk9mz8Gklby4YTkpZk0/41m2dp2hmjuWDjajitgXK+otjM6mDQAJDUBNwJXAksBlZLWlxR7DrgxYg4F7gduA0gIjZExJKIWAJcAzwTEVvLXremND0i9o7C8tRXZWtgoCuIK/mKYjMbZ7W0AJYCHRGxPSJ6gPuAFRVlVgDrssebgOXSa7Z+q7PXNrZSayAC1q8fWhi4O8jMxlEtATAH2FX2fHc2rmqZiOgFDgKtFWWuBu6tGPeNrPvnv1YJjMlvOGHgawjMbJyMy0FgScuA7oh4rGz0moi4APh32XBNP6+9XlJRUrGzs3McajtG+guD/rg1YGZjrJYAeA6YV/Z8bjauahlJzcAsoKts+ioq9v4j4rns78vA35O6ml4jItZGRCEiCm1tbTVUdxIYyhXF117rG9CY2ZioJQAeBhZJWihpGmljvrmizGbg2uzxSuDBiHSeo6QpwFWU9f9LapY0O3s8FfhD4DHyppYrio8f77sBja8dMLNRNGgAZH36NwAPAE8A/xAR2yTdIum9WbG7gFZJHcB/AcpPFb0c2BUR28vGtQAPSHoE2EpqQfzdiJdmshnqNQS+kMzMRpGitFGZBAqFQhSLxXpXY2xs2JD6/Lu7h/7aGTNSkKxZM/r1MrNJT9KWiChUjveVwBNF5R3Imppqf63PHDKzYXAATCTldyBbt27oVxX7zCEzGwIHwEQ13KuKS62B2bPT4DOIzKwfDoCJbCRXFXd1pcFnEJlZPxwAk8VQLySrVO3XRzdsSIHgVoJZLvksoMlsJGcOQWoRlK//0vMFC9I1Cj6ryKwh+CygRjTcexGUVIZ/eSvhIx/pO4ZQfjyhv8duQZhNOg6AyW6k9yLoz7FjfccQyo8n9Pe4/DiDg8FsUnAANIrK6whaW9MAQ7svwUiUWhA+AG02KTgAGkn5dQT79qVhuAeNR1O1n7BwK8Gs7hwAeTBQN1GpdVDvVkL5MQcHgtm4cADkSWU30YIFqXVQ3koodR9Nmza+dSs/5jCSbqPyU1vdyhg5v5+NLSImzXDJJZeEjZN77olYsCBCimhtTUN/jyE9T5vv0R9mzEj1GaxOg9WjNJ+BlnXBgr4yQx3fKErLN9j7WZrWiO9BgwGKUWWbWveN+lAGB8AEVm3jPNrBMFrzqgyPadOq/5/K/zfY+PL5DhQYH//4xAuQWjf6g62b/t4DqysHgNXHSDcsk3noLzBGa+M5lFZatfAZjxbc1Kn918XhMG4cAFZ/A22wKvfCPVQPhvHqdhvP5esvDPr7vDRKeAwnwIe53A4Am9jy3FLw8NqwG+xzMJ5dTmMRRPfck45JDfV96u841iAcADZ5lIfBaG1MPIzu+zlRQ7q8VVGtu2ugDfdwjmPVciC8v/kOd1iwYMhfKQeATT7V9pIG6hbp78s92N5WqZ+62pd9qAeEG2WotXtmsr4HYxFq1Y53jMX7Iw35q+QAsMlptE65rLUZP5JTQgf60lfbM51oG8/hntbpYzvjO7gFYDbB1RpcIz19drAWUS1nAY13//lQls/Dq4dRPgbg+wGYTUQbNsDNN8POnXD66Wnc/v2vfjx//uS9b0Np+XbseO19KcqVppV+2LCra+Dyk1Vp+SrX8Sit7/7uB1DTnjdwBfAk0AHcWGV6C7Axm/4Q0J6NXwNsLRtOAEsqXrsZeKyWergFYNaAhnqWTT1aFWN1IHyYe/RDxXC7gIAm4GngHGAa8GtgcUWZTwBfyx6vAjZWmc8FwNMV494H/L0DwMxGbKArrsfi9NLKIBroeEedr5QeSQBcBjxQ9vwm4KaKMg8Al2WPm4F9ZLebLCvz34Bby56/DvgJsNgBYGbjZqwuMJvAF671FwDNNXQfzQF2lT3fDSzrr0xE9Eo6CLRmQVByNbCi7PlfAv8dGOYNbc3MhmHNmrE5bjJW8x1D4/Jz0JKWAd0R8Vj2fAnwhoj4dg2vvV5SUVKxs7NzrKtqZpYbtQTAc8C8sudzs3FVy0hqBmYBXWXTVwH3lj2/DChIepbUDXSepH+t9s8jYm1EFCKi0NbWVkN1zcysFrUEwMPAIkkLJU0jbcw3V5TZDFybPV4JPJj1OyFpCnAVcF+pcER8NSLOjoh24PeApyLinSNZEDMzG5pBjwFkffo3kA70NgF3R8Q2SbeQDixsBu4C1kvqAPaTQqLkcmBXRGwf/eqbmdlw+UIwM7MG19+FYJMqACR1AjsGKTabV599lBde7nzxcufLSJd7QUS85iDqpAqAWkgqVku6Ruflzhcvd76M1XKPy2mgZmY28TgAzMxyqhEDYG29K1AnXu588XLny5gsd8MdAzAzs9o0YgvAzMxq4AAwM8uphgkASVdIelJSh6Qb612fsSJpnqR/kfS4pG2SPpWNP13SDyT9Nvt7Wr3rOhYkNUn6laTvZc8XSnooW+8bs58raSiSTpW0SdJvJD0h6bIcre/PZJ/zxyTdK2l6I65zSXdL2ivpsbJxVdexkv+ZLf8jki4e7v9tiACQ1ATcCVxJur/AakmL61urMdMLfDYiFgNvAz6ZLeuNwI8iYhHwo+x5I/oU8ETZ89uA2yPiXOBF4Lq61GpsfQX4fkS8EbiQtPwNv74lzQH+M1CIiLeQfopmFY25zv8X6c6L5fpbx1cCi7LheuCrw/2nDREAwFKgIyK2R0QP6YfnVgzymkkpIvZExC+zxy+TNgZzSMu7Liu2Dvij+tRw7EiaC/wB8PXsuYB/D2zKijTcckuaRfo9rbsAIqInIg6Qg/WdaQZOyn5leAawhwZc5xHxb6TfUSvX3zpeAfzv7F4vPwdOlXTWcP5vowRAtZvWzKlTXcaNpHbgItJ9mM+MiD3ZpBeAM+tUrbF0B/A50r2lId106EBE9GbPG3G9LwQ6gW9kXV9flzSTHKzviHgO+DKwk7ThPwhsofHXeUl/63jUtneNEgC5I+l1wLeAT0fES+XTsp/ibqjzeyX9IbA3IrbUuy7jrBm4GPhqRFwEvEJFd08jrm+ArM97BSkEzwZm8tpuklwYq3XcKAFQy01rGoakqaSN/4aIuD8b/btSMzD7u7de9Rsj7wDem91E6D5SN8BXSM3f0s+aN+J63w3sjoiHsuebSIHQ6Osb4D8Az0REZ0QcA+4nfQ4afZ2X9LeOR2171ygBUMtNaxpC1u99F/BERPyPsknlN+W5FvjueNdtLEXETRExN7uJ0CrSTYfWAP9CugkRNOZyvwDsknR+Nmo58DgNvr4zO4G3SZqRfe5Ly97Q67xMf+t4M/Ch7GygtwEHy7qKhqbaneIn4wC8B3gKeBq4ud71GcPl/D1SU/ARYGs2vIfUH/4j4LfAD4HT613XMXwP3gl8L3t8DvALoAP4JtBS7/qNwfIuAYrZOv8OcFpe1jfwReA3wGPAeqClEdc56Za5e4BjpFbfdf2tY0Cksx6fBh4lnSU1rP/rn4IwM8upRukCMjOzIXIAmJnllAPAzCynHABmZjnlADAzyykHgJlZTjkAzMxy6v8B00Gn5NsQVnAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WjmnD4ojDGGa"
      },
      "source": [
        "### Final fit"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yXmGJbdYuA3T",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b268bc9f-d4a7-444f-ea32-dcc4e3907fa0"
      },
      "source": [
        "model_baseline_final = get_model_baseline()\n",
        "model_baseline_final.compile(optimizer=keras.optimizers.RMSprop(),loss= 'kullback_leibler_divergence', metrics=['acc','kullback_leibler_divergence'])\n",
        "history_baseline_final = model_baseline_final.fit(X_train,y_train,batch_size=256,epochs=200,validation_data=(X_test,y_test),verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/200\n",
            "8/8 [==============================] - 1s 24ms/step - loss: 0.0837 - acc: 0.1970 - kullback_leibler_divergence: 0.0837 - val_loss: 0.0830 - val_acc: 0.2320 - val_kullback_leibler_divergence: 0.0830\n",
            "Epoch 2/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0811 - acc: 0.2389 - kullback_leibler_divergence: 0.0811 - val_loss: 0.0830 - val_acc: 0.2320 - val_kullback_leibler_divergence: 0.0830\n",
            "Epoch 3/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0822 - acc: 0.2527 - kullback_leibler_divergence: 0.0822 - val_loss: 0.0829 - val_acc: 0.2840 - val_kullback_leibler_divergence: 0.0829\n",
            "Epoch 4/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0820 - acc: 0.2851 - kullback_leibler_divergence: 0.0820 - val_loss: 0.0829 - val_acc: 0.2880 - val_kullback_leibler_divergence: 0.0829\n",
            "Epoch 5/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0815 - acc: 0.2662 - kullback_leibler_divergence: 0.0815 - val_loss: 0.0828 - val_acc: 0.2840 - val_kullback_leibler_divergence: 0.0828\n",
            "Epoch 6/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0824 - acc: 0.2736 - kullback_leibler_divergence: 0.0824 - val_loss: 0.0828 - val_acc: 0.2840 - val_kullback_leibler_divergence: 0.0828\n",
            "Epoch 7/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0823 - acc: 0.2620 - kullback_leibler_divergence: 0.0823 - val_loss: 0.0828 - val_acc: 0.2720 - val_kullback_leibler_divergence: 0.0828\n",
            "Epoch 8/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0834 - acc: 0.2527 - kullback_leibler_divergence: 0.0834 - val_loss: 0.0827 - val_acc: 0.3320 - val_kullback_leibler_divergence: 0.0827\n",
            "Epoch 9/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0814 - acc: 0.2904 - kullback_leibler_divergence: 0.0814 - val_loss: 0.0826 - val_acc: 0.2520 - val_kullback_leibler_divergence: 0.0826\n",
            "Epoch 10/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0813 - acc: 0.2917 - kullback_leibler_divergence: 0.0813 - val_loss: 0.0825 - val_acc: 0.2560 - val_kullback_leibler_divergence: 0.0825\n",
            "Epoch 11/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0818 - acc: 0.2582 - kullback_leibler_divergence: 0.0818 - val_loss: 0.0824 - val_acc: 0.3080 - val_kullback_leibler_divergence: 0.0824\n",
            "Epoch 12/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0826 - acc: 0.2812 - kullback_leibler_divergence: 0.0826 - val_loss: 0.0823 - val_acc: 0.3460 - val_kullback_leibler_divergence: 0.0823\n",
            "Epoch 13/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0818 - acc: 0.3266 - kullback_leibler_divergence: 0.0818 - val_loss: 0.0823 - val_acc: 0.3220 - val_kullback_leibler_divergence: 0.0823\n",
            "Epoch 14/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0821 - acc: 0.3082 - kullback_leibler_divergence: 0.0821 - val_loss: 0.0821 - val_acc: 0.3560 - val_kullback_leibler_divergence: 0.0821\n",
            "Epoch 15/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0825 - acc: 0.3130 - kullback_leibler_divergence: 0.0825 - val_loss: 0.0819 - val_acc: 0.2880 - val_kullback_leibler_divergence: 0.0819\n",
            "Epoch 16/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0799 - acc: 0.3139 - kullback_leibler_divergence: 0.0799 - val_loss: 0.0819 - val_acc: 0.3420 - val_kullback_leibler_divergence: 0.0819\n",
            "Epoch 17/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0817 - acc: 0.3204 - kullback_leibler_divergence: 0.0817 - val_loss: 0.0816 - val_acc: 0.3540 - val_kullback_leibler_divergence: 0.0816\n",
            "Epoch 18/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0804 - acc: 0.3075 - kullback_leibler_divergence: 0.0804 - val_loss: 0.0814 - val_acc: 0.3520 - val_kullback_leibler_divergence: 0.0814\n",
            "Epoch 19/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0801 - acc: 0.3316 - kullback_leibler_divergence: 0.0801 - val_loss: 0.0813 - val_acc: 0.3320 - val_kullback_leibler_divergence: 0.0813\n",
            "Epoch 20/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0795 - acc: 0.3183 - kullback_leibler_divergence: 0.0795 - val_loss: 0.0811 - val_acc: 0.3080 - val_kullback_leibler_divergence: 0.0811\n",
            "Epoch 21/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0800 - acc: 0.3360 - kullback_leibler_divergence: 0.0800 - val_loss: 0.0810 - val_acc: 0.2900 - val_kullback_leibler_divergence: 0.0810\n",
            "Epoch 22/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0811 - acc: 0.3271 - kullback_leibler_divergence: 0.0811 - val_loss: 0.0808 - val_acc: 0.2920 - val_kullback_leibler_divergence: 0.0808\n",
            "Epoch 23/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0810 - acc: 0.3226 - kullback_leibler_divergence: 0.0810 - val_loss: 0.0804 - val_acc: 0.3440 - val_kullback_leibler_divergence: 0.0804\n",
            "Epoch 24/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0790 - acc: 0.3368 - kullback_leibler_divergence: 0.0790 - val_loss: 0.0802 - val_acc: 0.3560 - val_kullback_leibler_divergence: 0.0802\n",
            "Epoch 25/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0785 - acc: 0.3373 - kullback_leibler_divergence: 0.0785 - val_loss: 0.0800 - val_acc: 0.3260 - val_kullback_leibler_divergence: 0.0800\n",
            "Epoch 26/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0797 - acc: 0.3195 - kullback_leibler_divergence: 0.0797 - val_loss: 0.0798 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0798\n",
            "Epoch 27/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0782 - acc: 0.3410 - kullback_leibler_divergence: 0.0782 - val_loss: 0.0799 - val_acc: 0.2860 - val_kullback_leibler_divergence: 0.0799\n",
            "Epoch 28/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0798 - acc: 0.3166 - kullback_leibler_divergence: 0.0798 - val_loss: 0.0795 - val_acc: 0.2980 - val_kullback_leibler_divergence: 0.0795\n",
            "Epoch 29/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0791 - acc: 0.3216 - kullback_leibler_divergence: 0.0791 - val_loss: 0.0791 - val_acc: 0.3220 - val_kullback_leibler_divergence: 0.0791\n",
            "Epoch 30/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0791 - acc: 0.3209 - kullback_leibler_divergence: 0.0791 - val_loss: 0.0788 - val_acc: 0.3220 - val_kullback_leibler_divergence: 0.0788\n",
            "Epoch 31/200\n",
            "8/8 [==============================] - 0s 27ms/step - loss: 0.0780 - acc: 0.3511 - kullback_leibler_divergence: 0.0780 - val_loss: 0.0787 - val_acc: 0.3360 - val_kullback_leibler_divergence: 0.0787\n",
            "Epoch 32/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0788 - acc: 0.3497 - kullback_leibler_divergence: 0.0788 - val_loss: 0.0784 - val_acc: 0.3340 - val_kullback_leibler_divergence: 0.0784\n",
            "Epoch 33/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0757 - acc: 0.3337 - kullback_leibler_divergence: 0.0757 - val_loss: 0.0781 - val_acc: 0.3320 - val_kullback_leibler_divergence: 0.0781\n",
            "Epoch 34/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0773 - acc: 0.3493 - kullback_leibler_divergence: 0.0773 - val_loss: 0.0780 - val_acc: 0.3320 - val_kullback_leibler_divergence: 0.0780\n",
            "Epoch 35/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0766 - acc: 0.3478 - kullback_leibler_divergence: 0.0766 - val_loss: 0.0783 - val_acc: 0.3060 - val_kullback_leibler_divergence: 0.0783\n",
            "Epoch 36/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0771 - acc: 0.3344 - kullback_leibler_divergence: 0.0771 - val_loss: 0.0777 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0777\n",
            "Epoch 37/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0751 - acc: 0.3419 - kullback_leibler_divergence: 0.0751 - val_loss: 0.0774 - val_acc: 0.3540 - val_kullback_leibler_divergence: 0.0774\n",
            "Epoch 38/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0770 - acc: 0.3306 - kullback_leibler_divergence: 0.0770 - val_loss: 0.0775 - val_acc: 0.3320 - val_kullback_leibler_divergence: 0.0775\n",
            "Epoch 39/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0769 - acc: 0.3588 - kullback_leibler_divergence: 0.0769 - val_loss: 0.0773 - val_acc: 0.3420 - val_kullback_leibler_divergence: 0.0773\n",
            "Epoch 40/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0751 - acc: 0.3585 - kullback_leibler_divergence: 0.0751 - val_loss: 0.0779 - val_acc: 0.3100 - val_kullback_leibler_divergence: 0.0779\n",
            "Epoch 41/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0770 - acc: 0.3449 - kullback_leibler_divergence: 0.0770 - val_loss: 0.0769 - val_acc: 0.3320 - val_kullback_leibler_divergence: 0.0769\n",
            "Epoch 42/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0764 - acc: 0.3649 - kullback_leibler_divergence: 0.0764 - val_loss: 0.0767 - val_acc: 0.3320 - val_kullback_leibler_divergence: 0.0767\n",
            "Epoch 43/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0760 - acc: 0.3452 - kullback_leibler_divergence: 0.0760 - val_loss: 0.0765 - val_acc: 0.3420 - val_kullback_leibler_divergence: 0.0765\n",
            "Epoch 44/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0756 - acc: 0.3377 - kullback_leibler_divergence: 0.0756 - val_loss: 0.0767 - val_acc: 0.3220 - val_kullback_leibler_divergence: 0.0767\n",
            "Epoch 45/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0768 - acc: 0.3489 - kullback_leibler_divergence: 0.0768 - val_loss: 0.0764 - val_acc: 0.3340 - val_kullback_leibler_divergence: 0.0764\n",
            "Epoch 46/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0765 - acc: 0.3431 - kullback_leibler_divergence: 0.0765 - val_loss: 0.0763 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0763\n",
            "Epoch 47/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0749 - acc: 0.3548 - kullback_leibler_divergence: 0.0749 - val_loss: 0.0770 - val_acc: 0.3280 - val_kullback_leibler_divergence: 0.0770\n",
            "Epoch 48/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0779 - acc: 0.3433 - kullback_leibler_divergence: 0.0779 - val_loss: 0.0761 - val_acc: 0.3460 - val_kullback_leibler_divergence: 0.0761\n",
            "Epoch 49/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0744 - acc: 0.3552 - kullback_leibler_divergence: 0.0744 - val_loss: 0.0760 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0760\n",
            "Epoch 50/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0750 - acc: 0.3465 - kullback_leibler_divergence: 0.0750 - val_loss: 0.0759 - val_acc: 0.3320 - val_kullback_leibler_divergence: 0.0759\n",
            "Epoch 51/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0768 - acc: 0.3500 - kullback_leibler_divergence: 0.0768 - val_loss: 0.0763 - val_acc: 0.3500 - val_kullback_leibler_divergence: 0.0763\n",
            "Epoch 52/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0775 - acc: 0.3515 - kullback_leibler_divergence: 0.0775 - val_loss: 0.0762 - val_acc: 0.3300 - val_kullback_leibler_divergence: 0.0762\n",
            "Epoch 53/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0763 - acc: 0.3469 - kullback_leibler_divergence: 0.0763 - val_loss: 0.0769 - val_acc: 0.3240 - val_kullback_leibler_divergence: 0.0769\n",
            "Epoch 54/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0751 - acc: 0.3587 - kullback_leibler_divergence: 0.0751 - val_loss: 0.0758 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0758\n",
            "Epoch 55/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0758 - acc: 0.3550 - kullback_leibler_divergence: 0.0758 - val_loss: 0.0756 - val_acc: 0.3480 - val_kullback_leibler_divergence: 0.0756\n",
            "Epoch 56/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0753 - acc: 0.3624 - kullback_leibler_divergence: 0.0753 - val_loss: 0.0755 - val_acc: 0.3560 - val_kullback_leibler_divergence: 0.0755\n",
            "Epoch 57/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0759 - acc: 0.3491 - kullback_leibler_divergence: 0.0759 - val_loss: 0.0752 - val_acc: 0.3380 - val_kullback_leibler_divergence: 0.0752\n",
            "Epoch 58/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0761 - acc: 0.3475 - kullback_leibler_divergence: 0.0761 - val_loss: 0.0751 - val_acc: 0.3520 - val_kullback_leibler_divergence: 0.0751\n",
            "Epoch 59/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0747 - acc: 0.3622 - kullback_leibler_divergence: 0.0747 - val_loss: 0.0751 - val_acc: 0.3480 - val_kullback_leibler_divergence: 0.0751\n",
            "Epoch 60/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0744 - acc: 0.3592 - kullback_leibler_divergence: 0.0744 - val_loss: 0.0755 - val_acc: 0.3540 - val_kullback_leibler_divergence: 0.0755\n",
            "Epoch 61/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0748 - acc: 0.3437 - kullback_leibler_divergence: 0.0748 - val_loss: 0.0749 - val_acc: 0.3420 - val_kullback_leibler_divergence: 0.0749\n",
            "Epoch 62/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0761 - acc: 0.3479 - kullback_leibler_divergence: 0.0761 - val_loss: 0.0749 - val_acc: 0.3420 - val_kullback_leibler_divergence: 0.0749\n",
            "Epoch 63/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0737 - acc: 0.3636 - kullback_leibler_divergence: 0.0737 - val_loss: 0.0750 - val_acc: 0.3620 - val_kullback_leibler_divergence: 0.0750\n",
            "Epoch 64/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0739 - acc: 0.3615 - kullback_leibler_divergence: 0.0739 - val_loss: 0.0751 - val_acc: 0.3580 - val_kullback_leibler_divergence: 0.0751\n",
            "Epoch 65/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0760 - acc: 0.3549 - kullback_leibler_divergence: 0.0760 - val_loss: 0.0750 - val_acc: 0.3600 - val_kullback_leibler_divergence: 0.0750\n",
            "Epoch 66/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0749 - acc: 0.3569 - kullback_leibler_divergence: 0.0749 - val_loss: 0.0746 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0746\n",
            "Epoch 67/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0736 - acc: 0.3599 - kullback_leibler_divergence: 0.0736 - val_loss: 0.0747 - val_acc: 0.3520 - val_kullback_leibler_divergence: 0.0747\n",
            "Epoch 68/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0748 - acc: 0.3459 - kullback_leibler_divergence: 0.0748 - val_loss: 0.0778 - val_acc: 0.3380 - val_kullback_leibler_divergence: 0.0778\n",
            "Epoch 69/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0753 - acc: 0.3461 - kullback_leibler_divergence: 0.0753 - val_loss: 0.0745 - val_acc: 0.3440 - val_kullback_leibler_divergence: 0.0745\n",
            "Epoch 70/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0741 - acc: 0.3569 - kullback_leibler_divergence: 0.0741 - val_loss: 0.0747 - val_acc: 0.3540 - val_kullback_leibler_divergence: 0.0747\n",
            "Epoch 71/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0751 - acc: 0.3487 - kullback_leibler_divergence: 0.0751 - val_loss: 0.0745 - val_acc: 0.3580 - val_kullback_leibler_divergence: 0.0745\n",
            "Epoch 72/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0755 - acc: 0.3576 - kullback_leibler_divergence: 0.0755 - val_loss: 0.0747 - val_acc: 0.3480 - val_kullback_leibler_divergence: 0.0747\n",
            "Epoch 73/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0743 - acc: 0.3523 - kullback_leibler_divergence: 0.0743 - val_loss: 0.0749 - val_acc: 0.3560 - val_kullback_leibler_divergence: 0.0749\n",
            "Epoch 74/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0759 - acc: 0.3469 - kullback_leibler_divergence: 0.0759 - val_loss: 0.0748 - val_acc: 0.3640 - val_kullback_leibler_divergence: 0.0748\n",
            "Epoch 75/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0753 - acc: 0.3648 - kullback_leibler_divergence: 0.0753 - val_loss: 0.0751 - val_acc: 0.3480 - val_kullback_leibler_divergence: 0.0751\n",
            "Epoch 76/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0735 - acc: 0.3635 - kullback_leibler_divergence: 0.0735 - val_loss: 0.0742 - val_acc: 0.3620 - val_kullback_leibler_divergence: 0.0742\n",
            "Epoch 77/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0735 - acc: 0.3596 - kullback_leibler_divergence: 0.0735 - val_loss: 0.0744 - val_acc: 0.3680 - val_kullback_leibler_divergence: 0.0744\n",
            "Epoch 78/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0743 - acc: 0.3652 - kullback_leibler_divergence: 0.0743 - val_loss: 0.0740 - val_acc: 0.3580 - val_kullback_leibler_divergence: 0.0740\n",
            "Epoch 79/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0734 - acc: 0.3660 - kullback_leibler_divergence: 0.0734 - val_loss: 0.0750 - val_acc: 0.3760 - val_kullback_leibler_divergence: 0.0750\n",
            "Epoch 80/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0762 - acc: 0.3346 - kullback_leibler_divergence: 0.0762 - val_loss: 0.0740 - val_acc: 0.3600 - val_kullback_leibler_divergence: 0.0740\n",
            "Epoch 81/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0756 - acc: 0.3539 - kullback_leibler_divergence: 0.0756 - val_loss: 0.0745 - val_acc: 0.3640 - val_kullback_leibler_divergence: 0.0745\n",
            "Epoch 82/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0749 - acc: 0.3645 - kullback_leibler_divergence: 0.0749 - val_loss: 0.0748 - val_acc: 0.3740 - val_kullback_leibler_divergence: 0.0748\n",
            "Epoch 83/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0743 - acc: 0.3571 - kullback_leibler_divergence: 0.0743 - val_loss: 0.0738 - val_acc: 0.3580 - val_kullback_leibler_divergence: 0.0738\n",
            "Epoch 84/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0736 - acc: 0.3743 - kullback_leibler_divergence: 0.0736 - val_loss: 0.0739 - val_acc: 0.3720 - val_kullback_leibler_divergence: 0.0739\n",
            "Epoch 85/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0735 - acc: 0.3710 - kullback_leibler_divergence: 0.0735 - val_loss: 0.0740 - val_acc: 0.3740 - val_kullback_leibler_divergence: 0.0740\n",
            "Epoch 86/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0749 - acc: 0.3858 - kullback_leibler_divergence: 0.0749 - val_loss: 0.0768 - val_acc: 0.3420 - val_kullback_leibler_divergence: 0.0768\n",
            "Epoch 87/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0747 - acc: 0.3598 - kullback_leibler_divergence: 0.0747 - val_loss: 0.0738 - val_acc: 0.3620 - val_kullback_leibler_divergence: 0.0738\n",
            "Epoch 88/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0741 - acc: 0.3566 - kullback_leibler_divergence: 0.0741 - val_loss: 0.0737 - val_acc: 0.3840 - val_kullback_leibler_divergence: 0.0737\n",
            "Epoch 89/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0738 - acc: 0.3853 - kullback_leibler_divergence: 0.0738 - val_loss: 0.0737 - val_acc: 0.3700 - val_kullback_leibler_divergence: 0.0737\n",
            "Epoch 90/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0751 - acc: 0.3550 - kullback_leibler_divergence: 0.0751 - val_loss: 0.0741 - val_acc: 0.3720 - val_kullback_leibler_divergence: 0.0741\n",
            "Epoch 91/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0742 - acc: 0.3597 - kullback_leibler_divergence: 0.0742 - val_loss: 0.0736 - val_acc: 0.3800 - val_kullback_leibler_divergence: 0.0736\n",
            "Epoch 92/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0736 - acc: 0.3891 - kullback_leibler_divergence: 0.0736 - val_loss: 0.0746 - val_acc: 0.3340 - val_kullback_leibler_divergence: 0.0746\n",
            "Epoch 93/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0749 - acc: 0.3363 - kullback_leibler_divergence: 0.0749 - val_loss: 0.0737 - val_acc: 0.3740 - val_kullback_leibler_divergence: 0.0737\n",
            "Epoch 94/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0721 - acc: 0.3659 - kullback_leibler_divergence: 0.0721 - val_loss: 0.0738 - val_acc: 0.3740 - val_kullback_leibler_divergence: 0.0738\n",
            "Epoch 95/200\n",
            "8/8 [==============================] - 0s 22ms/step - loss: 0.0748 - acc: 0.3631 - kullback_leibler_divergence: 0.0748 - val_loss: 0.0735 - val_acc: 0.3740 - val_kullback_leibler_divergence: 0.0735\n",
            "Epoch 96/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0745 - acc: 0.3764 - kullback_leibler_divergence: 0.0745 - val_loss: 0.0746 - val_acc: 0.3700 - val_kullback_leibler_divergence: 0.0746\n",
            "Epoch 97/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0744 - acc: 0.3633 - kullback_leibler_divergence: 0.0744 - val_loss: 0.0734 - val_acc: 0.3640 - val_kullback_leibler_divergence: 0.0734\n",
            "Epoch 98/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0740 - acc: 0.3721 - kullback_leibler_divergence: 0.0740 - val_loss: 0.0738 - val_acc: 0.3720 - val_kullback_leibler_divergence: 0.0738\n",
            "Epoch 99/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0725 - acc: 0.3686 - kullback_leibler_divergence: 0.0725 - val_loss: 0.0737 - val_acc: 0.3620 - val_kullback_leibler_divergence: 0.0737\n",
            "Epoch 100/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0728 - acc: 0.3802 - kullback_leibler_divergence: 0.0728 - val_loss: 0.0732 - val_acc: 0.3800 - val_kullback_leibler_divergence: 0.0732\n",
            "Epoch 101/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0714 - acc: 0.3886 - kullback_leibler_divergence: 0.0714 - val_loss: 0.0733 - val_acc: 0.3900 - val_kullback_leibler_divergence: 0.0733\n",
            "Epoch 102/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0733 - acc: 0.3660 - kullback_leibler_divergence: 0.0733 - val_loss: 0.0758 - val_acc: 0.3460 - val_kullback_leibler_divergence: 0.0758\n",
            "Epoch 103/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0760 - acc: 0.3574 - kullback_leibler_divergence: 0.0760 - val_loss: 0.0733 - val_acc: 0.3800 - val_kullback_leibler_divergence: 0.0733\n",
            "Epoch 104/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0715 - acc: 0.3816 - kullback_leibler_divergence: 0.0715 - val_loss: 0.0740 - val_acc: 0.3640 - val_kullback_leibler_divergence: 0.0740\n",
            "Epoch 105/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0725 - acc: 0.3699 - kullback_leibler_divergence: 0.0725 - val_loss: 0.0731 - val_acc: 0.3780 - val_kullback_leibler_divergence: 0.0731\n",
            "Epoch 106/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0736 - acc: 0.3636 - kullback_leibler_divergence: 0.0736 - val_loss: 0.0735 - val_acc: 0.3840 - val_kullback_leibler_divergence: 0.0735\n",
            "Epoch 107/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0738 - acc: 0.3651 - kullback_leibler_divergence: 0.0738 - val_loss: 0.0747 - val_acc: 0.3660 - val_kullback_leibler_divergence: 0.0747\n",
            "Epoch 108/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0731 - acc: 0.3642 - kullback_leibler_divergence: 0.0731 - val_loss: 0.0738 - val_acc: 0.3880 - val_kullback_leibler_divergence: 0.0738\n",
            "Epoch 109/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0749 - acc: 0.3681 - kullback_leibler_divergence: 0.0749 - val_loss: 0.0729 - val_acc: 0.3800 - val_kullback_leibler_divergence: 0.0729\n",
            "Epoch 110/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0736 - acc: 0.3662 - kullback_leibler_divergence: 0.0736 - val_loss: 0.0730 - val_acc: 0.3800 - val_kullback_leibler_divergence: 0.0730\n",
            "Epoch 111/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0731 - acc: 0.3855 - kullback_leibler_divergence: 0.0731 - val_loss: 0.0731 - val_acc: 0.3860 - val_kullback_leibler_divergence: 0.0731\n",
            "Epoch 112/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0711 - acc: 0.3518 - kullback_leibler_divergence: 0.0711 - val_loss: 0.0730 - val_acc: 0.3700 - val_kullback_leibler_divergence: 0.0730\n",
            "Epoch 113/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0748 - acc: 0.3718 - kullback_leibler_divergence: 0.0748 - val_loss: 0.0728 - val_acc: 0.3740 - val_kullback_leibler_divergence: 0.0728\n",
            "Epoch 114/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0732 - acc: 0.3651 - kullback_leibler_divergence: 0.0732 - val_loss: 0.0742 - val_acc: 0.3780 - val_kullback_leibler_divergence: 0.0742\n",
            "Epoch 115/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0719 - acc: 0.3667 - kullback_leibler_divergence: 0.0719 - val_loss: 0.0728 - val_acc: 0.3620 - val_kullback_leibler_divergence: 0.0728\n",
            "Epoch 116/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0744 - acc: 0.3716 - kullback_leibler_divergence: 0.0744 - val_loss: 0.0727 - val_acc: 0.3960 - val_kullback_leibler_divergence: 0.0727\n",
            "Epoch 117/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0732 - acc: 0.3633 - kullback_leibler_divergence: 0.0732 - val_loss: 0.0781 - val_acc: 0.3460 - val_kullback_leibler_divergence: 0.0781\n",
            "Epoch 118/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0746 - acc: 0.3577 - kullback_leibler_divergence: 0.0746 - val_loss: 0.0727 - val_acc: 0.3740 - val_kullback_leibler_divergence: 0.0727\n",
            "Epoch 119/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0734 - acc: 0.3523 - kullback_leibler_divergence: 0.0734 - val_loss: 0.0735 - val_acc: 0.3660 - val_kullback_leibler_divergence: 0.0735\n",
            "Epoch 120/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0739 - acc: 0.3733 - kullback_leibler_divergence: 0.0739 - val_loss: 0.0727 - val_acc: 0.3760 - val_kullback_leibler_divergence: 0.0727\n",
            "Epoch 121/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0752 - acc: 0.3699 - kullback_leibler_divergence: 0.0752 - val_loss: 0.0728 - val_acc: 0.3900 - val_kullback_leibler_divergence: 0.0728\n",
            "Epoch 122/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0723 - acc: 0.3609 - kullback_leibler_divergence: 0.0723 - val_loss: 0.0726 - val_acc: 0.3800 - val_kullback_leibler_divergence: 0.0726\n",
            "Epoch 123/200\n",
            "8/8 [==============================] - 0s 10ms/step - loss: 0.0751 - acc: 0.3531 - kullback_leibler_divergence: 0.0751 - val_loss: 0.0726 - val_acc: 0.3920 - val_kullback_leibler_divergence: 0.0726\n",
            "Epoch 124/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0737 - acc: 0.3810 - kullback_leibler_divergence: 0.0737 - val_loss: 0.0727 - val_acc: 0.3640 - val_kullback_leibler_divergence: 0.0727\n",
            "Epoch 125/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0737 - acc: 0.3791 - kullback_leibler_divergence: 0.0737 - val_loss: 0.0725 - val_acc: 0.3860 - val_kullback_leibler_divergence: 0.0725\n",
            "Epoch 126/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0731 - acc: 0.3732 - kullback_leibler_divergence: 0.0731 - val_loss: 0.0727 - val_acc: 0.3800 - val_kullback_leibler_divergence: 0.0727\n",
            "Epoch 127/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0754 - acc: 0.3693 - kullback_leibler_divergence: 0.0754 - val_loss: 0.0727 - val_acc: 0.3880 - val_kullback_leibler_divergence: 0.0727\n",
            "Epoch 128/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0739 - acc: 0.3618 - kullback_leibler_divergence: 0.0739 - val_loss: 0.0725 - val_acc: 0.3840 - val_kullback_leibler_divergence: 0.0725\n",
            "Epoch 129/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0723 - acc: 0.3754 - kullback_leibler_divergence: 0.0723 - val_loss: 0.0725 - val_acc: 0.3700 - val_kullback_leibler_divergence: 0.0725\n",
            "Epoch 130/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0734 - acc: 0.3706 - kullback_leibler_divergence: 0.0734 - val_loss: 0.0726 - val_acc: 0.3880 - val_kullback_leibler_divergence: 0.0726\n",
            "Epoch 131/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0753 - acc: 0.3894 - kullback_leibler_divergence: 0.0753 - val_loss: 0.0724 - val_acc: 0.3920 - val_kullback_leibler_divergence: 0.0724\n",
            "Epoch 132/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0731 - acc: 0.3751 - kullback_leibler_divergence: 0.0731 - val_loss: 0.0738 - val_acc: 0.3800 - val_kullback_leibler_divergence: 0.0738\n",
            "Epoch 133/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0732 - acc: 0.3736 - kullback_leibler_divergence: 0.0732 - val_loss: 0.0724 - val_acc: 0.3880 - val_kullback_leibler_divergence: 0.0724\n",
            "Epoch 134/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0723 - acc: 0.3848 - kullback_leibler_divergence: 0.0723 - val_loss: 0.0722 - val_acc: 0.3900 - val_kullback_leibler_divergence: 0.0722\n",
            "Epoch 135/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0732 - acc: 0.3746 - kullback_leibler_divergence: 0.0732 - val_loss: 0.0724 - val_acc: 0.3880 - val_kullback_leibler_divergence: 0.0724\n",
            "Epoch 136/200\n",
            "8/8 [==============================] - 0s 6ms/step - loss: 0.0712 - acc: 0.3726 - kullback_leibler_divergence: 0.0712 - val_loss: 0.0728 - val_acc: 0.3780 - val_kullback_leibler_divergence: 0.0728\n",
            "Epoch 137/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0730 - acc: 0.3833 - kullback_leibler_divergence: 0.0730 - val_loss: 0.0732 - val_acc: 0.3760 - val_kullback_leibler_divergence: 0.0732\n",
            "Epoch 138/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0713 - acc: 0.3844 - kullback_leibler_divergence: 0.0713 - val_loss: 0.0727 - val_acc: 0.3920 - val_kullback_leibler_divergence: 0.0727\n",
            "Epoch 139/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0730 - acc: 0.3780 - kullback_leibler_divergence: 0.0730 - val_loss: 0.0726 - val_acc: 0.3860 - val_kullback_leibler_divergence: 0.0726\n",
            "Epoch 140/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0723 - acc: 0.3803 - kullback_leibler_divergence: 0.0723 - val_loss: 0.0720 - val_acc: 0.3860 - val_kullback_leibler_divergence: 0.0720\n",
            "Epoch 141/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0741 - acc: 0.3901 - kullback_leibler_divergence: 0.0741 - val_loss: 0.0720 - val_acc: 0.3900 - val_kullback_leibler_divergence: 0.0720\n",
            "Epoch 142/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0721 - acc: 0.3667 - kullback_leibler_divergence: 0.0721 - val_loss: 0.0720 - val_acc: 0.3880 - val_kullback_leibler_divergence: 0.0720\n",
            "Epoch 143/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0725 - acc: 0.3876 - kullback_leibler_divergence: 0.0725 - val_loss: 0.0723 - val_acc: 0.3920 - val_kullback_leibler_divergence: 0.0723\n",
            "Epoch 144/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0737 - acc: 0.3934 - kullback_leibler_divergence: 0.0737 - val_loss: 0.0720 - val_acc: 0.3700 - val_kullback_leibler_divergence: 0.0720\n",
            "Epoch 145/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0725 - acc: 0.3708 - kullback_leibler_divergence: 0.0725 - val_loss: 0.0731 - val_acc: 0.3840 - val_kullback_leibler_divergence: 0.0731\n",
            "Epoch 146/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0739 - acc: 0.3875 - kullback_leibler_divergence: 0.0739 - val_loss: 0.0718 - val_acc: 0.3940 - val_kullback_leibler_divergence: 0.0718\n",
            "Epoch 147/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0730 - acc: 0.3793 - kullback_leibler_divergence: 0.0730 - val_loss: 0.0721 - val_acc: 0.3500 - val_kullback_leibler_divergence: 0.0721\n",
            "Epoch 148/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0724 - acc: 0.3696 - kullback_leibler_divergence: 0.0724 - val_loss: 0.0719 - val_acc: 0.3860 - val_kullback_leibler_divergence: 0.0719\n",
            "Epoch 149/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0715 - acc: 0.3961 - kullback_leibler_divergence: 0.0715 - val_loss: 0.0721 - val_acc: 0.3760 - val_kullback_leibler_divergence: 0.0721\n",
            "Epoch 150/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0728 - acc: 0.3626 - kullback_leibler_divergence: 0.0728 - val_loss: 0.0728 - val_acc: 0.3780 - val_kullback_leibler_divergence: 0.0728\n",
            "Epoch 151/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0742 - acc: 0.3698 - kullback_leibler_divergence: 0.0742 - val_loss: 0.0717 - val_acc: 0.4000 - val_kullback_leibler_divergence: 0.0717\n",
            "Epoch 152/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0722 - acc: 0.3682 - kullback_leibler_divergence: 0.0722 - val_loss: 0.0724 - val_acc: 0.3340 - val_kullback_leibler_divergence: 0.0724\n",
            "Epoch 153/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0726 - acc: 0.3676 - kullback_leibler_divergence: 0.0726 - val_loss: 0.0717 - val_acc: 0.3940 - val_kullback_leibler_divergence: 0.0717\n",
            "Epoch 154/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0729 - acc: 0.3821 - kullback_leibler_divergence: 0.0729 - val_loss: 0.0718 - val_acc: 0.3960 - val_kullback_leibler_divergence: 0.0718\n",
            "Epoch 155/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0728 - acc: 0.3728 - kullback_leibler_divergence: 0.0728 - val_loss: 0.0715 - val_acc: 0.4000 - val_kullback_leibler_divergence: 0.0715\n",
            "Epoch 156/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0721 - acc: 0.3931 - kullback_leibler_divergence: 0.0721 - val_loss: 0.0727 - val_acc: 0.4020 - val_kullback_leibler_divergence: 0.0727\n",
            "Epoch 157/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0738 - acc: 0.3723 - kullback_leibler_divergence: 0.0738 - val_loss: 0.0719 - val_acc: 0.3860 - val_kullback_leibler_divergence: 0.0719\n",
            "Epoch 158/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0717 - acc: 0.3801 - kullback_leibler_divergence: 0.0717 - val_loss: 0.0716 - val_acc: 0.3980 - val_kullback_leibler_divergence: 0.0716\n",
            "Epoch 159/200\n",
            "8/8 [==============================] - 0s 24ms/step - loss: 0.0721 - acc: 0.3886 - kullback_leibler_divergence: 0.0721 - val_loss: 0.0720 - val_acc: 0.3920 - val_kullback_leibler_divergence: 0.0720\n",
            "Epoch 160/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0733 - acc: 0.3736 - kullback_leibler_divergence: 0.0733 - val_loss: 0.0726 - val_acc: 0.3920 - val_kullback_leibler_divergence: 0.0726\n",
            "Epoch 161/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0715 - acc: 0.3789 - kullback_leibler_divergence: 0.0715 - val_loss: 0.0720 - val_acc: 0.3580 - val_kullback_leibler_divergence: 0.0720\n",
            "Epoch 162/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0736 - acc: 0.3846 - kullback_leibler_divergence: 0.0736 - val_loss: 0.0723 - val_acc: 0.3940 - val_kullback_leibler_divergence: 0.0723\n",
            "Epoch 163/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0718 - acc: 0.3924 - kullback_leibler_divergence: 0.0718 - val_loss: 0.0714 - val_acc: 0.3980 - val_kullback_leibler_divergence: 0.0714\n",
            "Epoch 164/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0720 - acc: 0.3817 - kullback_leibler_divergence: 0.0720 - val_loss: 0.0716 - val_acc: 0.3960 - val_kullback_leibler_divergence: 0.0716\n",
            "Epoch 165/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0707 - acc: 0.3998 - kullback_leibler_divergence: 0.0707 - val_loss: 0.0713 - val_acc: 0.3900 - val_kullback_leibler_divergence: 0.0713\n",
            "Epoch 166/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0721 - acc: 0.3865 - kullback_leibler_divergence: 0.0721 - val_loss: 0.0729 - val_acc: 0.3680 - val_kullback_leibler_divergence: 0.0729\n",
            "Epoch 167/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0745 - acc: 0.3799 - kullback_leibler_divergence: 0.0745 - val_loss: 0.0715 - val_acc: 0.3980 - val_kullback_leibler_divergence: 0.0715\n",
            "Epoch 168/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0710 - acc: 0.3847 - kullback_leibler_divergence: 0.0710 - val_loss: 0.0719 - val_acc: 0.3560 - val_kullback_leibler_divergence: 0.0719\n",
            "Epoch 169/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0735 - acc: 0.3656 - kullback_leibler_divergence: 0.0735 - val_loss: 0.0727 - val_acc: 0.3820 - val_kullback_leibler_divergence: 0.0727\n",
            "Epoch 170/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0727 - acc: 0.3767 - kullback_leibler_divergence: 0.0727 - val_loss: 0.0715 - val_acc: 0.3920 - val_kullback_leibler_divergence: 0.0715\n",
            "Epoch 171/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0697 - acc: 0.3895 - kullback_leibler_divergence: 0.0697 - val_loss: 0.0716 - val_acc: 0.3780 - val_kullback_leibler_divergence: 0.0716\n",
            "Epoch 172/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0716 - acc: 0.3901 - kullback_leibler_divergence: 0.0716 - val_loss: 0.0760 - val_acc: 0.3360 - val_kullback_leibler_divergence: 0.0760\n",
            "Epoch 173/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0747 - acc: 0.3660 - kullback_leibler_divergence: 0.0747 - val_loss: 0.0721 - val_acc: 0.3980 - val_kullback_leibler_divergence: 0.0721\n",
            "Epoch 174/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0734 - acc: 0.3788 - kullback_leibler_divergence: 0.0734 - val_loss: 0.0716 - val_acc: 0.3880 - val_kullback_leibler_divergence: 0.0716\n",
            "Epoch 175/200\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0738 - acc: 0.3800 - kullback_leibler_divergence: 0.0738 - val_loss: 0.0710 - val_acc: 0.4040 - val_kullback_leibler_divergence: 0.0710\n",
            "Epoch 176/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0741 - acc: 0.3911 - kullback_leibler_divergence: 0.0741 - val_loss: 0.0716 - val_acc: 0.3400 - val_kullback_leibler_divergence: 0.0716\n",
            "Epoch 177/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0722 - acc: 0.3656 - kullback_leibler_divergence: 0.0722 - val_loss: 0.0711 - val_acc: 0.3920 - val_kullback_leibler_divergence: 0.0711\n",
            "Epoch 178/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0731 - acc: 0.3873 - kullback_leibler_divergence: 0.0731 - val_loss: 0.0711 - val_acc: 0.3980 - val_kullback_leibler_divergence: 0.0711\n",
            "Epoch 179/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0716 - acc: 0.3919 - kullback_leibler_divergence: 0.0716 - val_loss: 0.0714 - val_acc: 0.3800 - val_kullback_leibler_divergence: 0.0714\n",
            "Epoch 180/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0713 - acc: 0.3720 - kullback_leibler_divergence: 0.0713 - val_loss: 0.0717 - val_acc: 0.3820 - val_kullback_leibler_divergence: 0.0717\n",
            "Epoch 181/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0718 - acc: 0.3885 - kullback_leibler_divergence: 0.0718 - val_loss: 0.0731 - val_acc: 0.3660 - val_kullback_leibler_divergence: 0.0731\n",
            "Epoch 182/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0732 - acc: 0.3740 - kullback_leibler_divergence: 0.0732 - val_loss: 0.0708 - val_acc: 0.4000 - val_kullback_leibler_divergence: 0.0708\n",
            "Epoch 183/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0719 - acc: 0.3950 - kullback_leibler_divergence: 0.0719 - val_loss: 0.0723 - val_acc: 0.3900 - val_kullback_leibler_divergence: 0.0723\n",
            "Epoch 184/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0712 - acc: 0.3904 - kullback_leibler_divergence: 0.0712 - val_loss: 0.0711 - val_acc: 0.4020 - val_kullback_leibler_divergence: 0.0711\n",
            "Epoch 185/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0725 - acc: 0.3860 - kullback_leibler_divergence: 0.0725 - val_loss: 0.0709 - val_acc: 0.3940 - val_kullback_leibler_divergence: 0.0709\n",
            "Epoch 186/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0711 - acc: 0.3938 - kullback_leibler_divergence: 0.0711 - val_loss: 0.0709 - val_acc: 0.3660 - val_kullback_leibler_divergence: 0.0709\n",
            "Epoch 187/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0719 - acc: 0.3757 - kullback_leibler_divergence: 0.0719 - val_loss: 0.0708 - val_acc: 0.3940 - val_kullback_leibler_divergence: 0.0708\n",
            "Epoch 188/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0703 - acc: 0.3991 - kullback_leibler_divergence: 0.0703 - val_loss: 0.0706 - val_acc: 0.3840 - val_kullback_leibler_divergence: 0.0706\n",
            "Epoch 189/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0716 - acc: 0.3604 - kullback_leibler_divergence: 0.0716 - val_loss: 0.0753 - val_acc: 0.3580 - val_kullback_leibler_divergence: 0.0753\n",
            "Epoch 190/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0733 - acc: 0.3737 - kullback_leibler_divergence: 0.0733 - val_loss: 0.0718 - val_acc: 0.3880 - val_kullback_leibler_divergence: 0.0718\n",
            "Epoch 191/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0726 - acc: 0.3855 - kullback_leibler_divergence: 0.0726 - val_loss: 0.0724 - val_acc: 0.3540 - val_kullback_leibler_divergence: 0.0724\n",
            "Epoch 192/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0719 - acc: 0.3676 - kullback_leibler_divergence: 0.0719 - val_loss: 0.0705 - val_acc: 0.3940 - val_kullback_leibler_divergence: 0.0705\n",
            "Epoch 193/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0721 - acc: 0.3905 - kullback_leibler_divergence: 0.0721 - val_loss: 0.0710 - val_acc: 0.3800 - val_kullback_leibler_divergence: 0.0710\n",
            "Epoch 194/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0719 - acc: 0.3817 - kullback_leibler_divergence: 0.0719 - val_loss: 0.0714 - val_acc: 0.3900 - val_kullback_leibler_divergence: 0.0714\n",
            "Epoch 195/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0707 - acc: 0.3832 - kullback_leibler_divergence: 0.0707 - val_loss: 0.0715 - val_acc: 0.3800 - val_kullback_leibler_divergence: 0.0715\n",
            "Epoch 196/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0727 - acc: 0.3770 - kullback_leibler_divergence: 0.0727 - val_loss: 0.0707 - val_acc: 0.4100 - val_kullback_leibler_divergence: 0.0707\n",
            "Epoch 197/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0726 - acc: 0.3981 - kullback_leibler_divergence: 0.0726 - val_loss: 0.0712 - val_acc: 0.3800 - val_kullback_leibler_divergence: 0.0712\n",
            "Epoch 198/200\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0724 - acc: 0.3794 - kullback_leibler_divergence: 0.0724 - val_loss: 0.0713 - val_acc: 0.3840 - val_kullback_leibler_divergence: 0.0713\n",
            "Epoch 199/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0725 - acc: 0.3800 - kullback_leibler_divergence: 0.0725 - val_loss: 0.0717 - val_acc: 0.3880 - val_kullback_leibler_divergence: 0.0717\n",
            "Epoch 200/200\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0727 - acc: 0.3803 - kullback_leibler_divergence: 0.0727 - val_loss: 0.0703 - val_acc: 0.3920 - val_kullback_leibler_divergence: 0.0703\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uYn9ScaG0g6M",
        "outputId": "32a6ed52-f489-4069-bce8-858eaae8597b"
      },
      "source": [
        "y_pred_baseline = model_baseline_final.predict(X_test)\n",
        "eval_baseline_sll = clf_eval(y_sl_test,get_single_label(y_pred_baseline),y_pred_baseline)\n",
        "eval_baseline_mll = clf_eval(y_ml_test,get_multi_label(y_pred_baseline),y_pred_baseline)\n",
        "eval_baseline = ldl_eval(y_test,y_pred_baseline)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "acc: 0.388\n",
            "f1 score(micro): 0.39960629921259844\n",
            "precision score(micro): 0.406\n",
            "recall score(micro): 0.39341085271317827\n",
            "AUC score(micro): 0.636922817660937\n",
            "hamming loss: 0.20333333333333334\n",
            "label ranking loss: 0.2928\n",
            "confusion matrix: [[[283 101]\n",
            "  [ 51  65]]\n",
            "\n",
            " [[418   0]\n",
            "  [ 82   0]]\n",
            "\n",
            " [[207 142]\n",
            "  [ 59  92]]\n",
            "\n",
            " [[447   0]\n",
            "  [ 53   0]]\n",
            "\n",
            " [[475   0]\n",
            "  [ 25   0]]\n",
            "\n",
            " [[357  54]\n",
            "  [ 43  46]]]\n",
            "acc: 0.07\n",
            "f1 score(micro): 0.4349775784753363\n",
            "precision score(micro): 0.39551478083588176\n",
            "recall score(micro): 0.48318804483188044\n",
            "AUC score(micro): 0.6066372631988259\n",
            "hamming loss: 0.336\n",
            "label ranking loss: 0.30896111111111113\n",
            "confusion matrix: [[[ 87 277]\n",
            "  [ 10 126]]\n",
            "\n",
            " [[324  14]\n",
            "  [147  15]]\n",
            "\n",
            " [[ 72 218]\n",
            "  [ 25 185]]\n",
            "\n",
            " [[369   1]\n",
            "  [129   1]]\n",
            "\n",
            " [[435   8]\n",
            "  [ 55   2]]\n",
            "\n",
            " [[317  75]\n",
            "  [ 49  59]]]\n",
            "Kullback–Leibler divergence:  0.07031856124670693\n",
            "Euclidean distance:  0.15271856057490102\n",
            "MSE:  0.004760165856459085\n",
            "Chebyshev distance:  0.12354125316248106\n",
            "Clark distance:  0.3811317111167079\n",
            "Canberra distance:  0.8183049335778682\n",
            "Cosine similarity:  0.9311610319904229\n",
            "Intersection similarity:  0.8531860243543377\n",
            "Kurtosis_signed_offset:  -0.6288520563510083\n",
            "Kurtosis_abs_offset:  1.1264648478343997\n",
            "Laloss:  0.877658452092704\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aeVCM9prt06p",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bdd5c107-71fa-4ec4-9d7f-10c11844dfc8"
      },
      "source": [
        "model_final = model_baseline_final\n",
        "model_final.compile(optimizer=keras.optimizers.RMSprop(learning_rate=0.0001),loss= label_ambiguity_loss, metrics=['acc','kullback_leibler_divergence'])\n",
        "history_final = model_final.fit(X_train,y_train,batch_size=256,epochs=50,validation_data=(X_test,y_test),verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/50\n",
            "8/8 [==============================] - 1s 32ms/step - loss: 0.0806 - acc: 0.3863 - kullback_leibler_divergence: 0.0732 - val_loss: 0.0765 - val_acc: 0.3940 - val_kullback_leibler_divergence: 0.0709\n",
            "Epoch 2/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0752 - acc: 0.3961 - kullback_leibler_divergence: 0.0714 - val_loss: 0.0750 - val_acc: 0.3920 - val_kullback_leibler_divergence: 0.0708\n",
            "Epoch 3/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0768 - acc: 0.3803 - kullback_leibler_divergence: 0.0724 - val_loss: 0.0752 - val_acc: 0.3980 - val_kullback_leibler_divergence: 0.0710\n",
            "Epoch 4/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0720 - acc: 0.3809 - kullback_leibler_divergence: 0.0710 - val_loss: 0.0751 - val_acc: 0.3860 - val_kullback_leibler_divergence: 0.0712\n",
            "Epoch 5/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0745 - acc: 0.3837 - kullback_leibler_divergence: 0.0731 - val_loss: 0.0753 - val_acc: 0.3820 - val_kullback_leibler_divergence: 0.0719\n",
            "Epoch 6/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0742 - acc: 0.3647 - kullback_leibler_divergence: 0.0721 - val_loss: 0.0752 - val_acc: 0.3760 - val_kullback_leibler_divergence: 0.0721\n",
            "Epoch 7/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0704 - acc: 0.3788 - kullback_leibler_divergence: 0.0713 - val_loss: 0.0750 - val_acc: 0.3760 - val_kullback_leibler_divergence: 0.0722\n",
            "Epoch 8/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0734 - acc: 0.3713 - kullback_leibler_divergence: 0.0732 - val_loss: 0.0752 - val_acc: 0.3640 - val_kullback_leibler_divergence: 0.0726\n",
            "Epoch 9/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0717 - acc: 0.3881 - kullback_leibler_divergence: 0.0737 - val_loss: 0.0750 - val_acc: 0.3540 - val_kullback_leibler_divergence: 0.0732\n",
            "Epoch 10/50\n",
            "8/8 [==============================] - 0s 25ms/step - loss: 0.0705 - acc: 0.3643 - kullback_leibler_divergence: 0.0743 - val_loss: 0.0749 - val_acc: 0.3520 - val_kullback_leibler_divergence: 0.0731\n",
            "Epoch 11/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0724 - acc: 0.3885 - kullback_leibler_divergence: 0.0740 - val_loss: 0.0747 - val_acc: 0.3440 - val_kullback_leibler_divergence: 0.0735\n",
            "Epoch 12/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0696 - acc: 0.3798 - kullback_leibler_divergence: 0.0731 - val_loss: 0.0744 - val_acc: 0.3540 - val_kullback_leibler_divergence: 0.0736\n",
            "Epoch 13/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0679 - acc: 0.3732 - kullback_leibler_divergence: 0.0723 - val_loss: 0.0738 - val_acc: 0.3580 - val_kullback_leibler_divergence: 0.0730\n",
            "Epoch 14/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0734 - acc: 0.3756 - kullback_leibler_divergence: 0.0752 - val_loss: 0.0725 - val_acc: 0.3560 - val_kullback_leibler_divergence: 0.0729\n",
            "Epoch 15/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0720 - acc: 0.3721 - kullback_leibler_divergence: 0.0726 - val_loss: 0.0725 - val_acc: 0.3580 - val_kullback_leibler_divergence: 0.0729\n",
            "Epoch 16/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0725 - acc: 0.3784 - kullback_leibler_divergence: 0.0739 - val_loss: 0.0729 - val_acc: 0.3540 - val_kullback_leibler_divergence: 0.0733\n",
            "Epoch 17/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0702 - acc: 0.3757 - kullback_leibler_divergence: 0.0716 - val_loss: 0.0727 - val_acc: 0.3520 - val_kullback_leibler_divergence: 0.0736\n",
            "Epoch 18/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0732 - acc: 0.3745 - kullback_leibler_divergence: 0.0756 - val_loss: 0.0728 - val_acc: 0.3480 - val_kullback_leibler_divergence: 0.0739\n",
            "Epoch 19/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0687 - acc: 0.3781 - kullback_leibler_divergence: 0.0731 - val_loss: 0.0733 - val_acc: 0.3480 - val_kullback_leibler_divergence: 0.0736\n",
            "Epoch 20/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0735 - acc: 0.3725 - kullback_leibler_divergence: 0.0753 - val_loss: 0.0730 - val_acc: 0.3560 - val_kullback_leibler_divergence: 0.0739\n",
            "Epoch 21/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0706 - acc: 0.3768 - kullback_leibler_divergence: 0.0746 - val_loss: 0.0727 - val_acc: 0.3600 - val_kullback_leibler_divergence: 0.0731\n",
            "Epoch 22/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0695 - acc: 0.3924 - kullback_leibler_divergence: 0.0730 - val_loss: 0.0721 - val_acc: 0.3500 - val_kullback_leibler_divergence: 0.0734\n",
            "Epoch 23/50\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0682 - acc: 0.3890 - kullback_leibler_divergence: 0.0719 - val_loss: 0.0723 - val_acc: 0.3540 - val_kullback_leibler_divergence: 0.0735\n",
            "Epoch 24/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0682 - acc: 0.3970 - kullback_leibler_divergence: 0.0731 - val_loss: 0.0722 - val_acc: 0.3500 - val_kullback_leibler_divergence: 0.0734\n",
            "Epoch 25/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0727 - acc: 0.3757 - kullback_leibler_divergence: 0.0757 - val_loss: 0.0727 - val_acc: 0.3480 - val_kullback_leibler_divergence: 0.0736\n",
            "Epoch 26/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0702 - acc: 0.3830 - kullback_leibler_divergence: 0.0732 - val_loss: 0.0725 - val_acc: 0.3460 - val_kullback_leibler_divergence: 0.0738\n",
            "Epoch 27/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0690 - acc: 0.3728 - kullback_leibler_divergence: 0.0735 - val_loss: 0.0726 - val_acc: 0.3440 - val_kullback_leibler_divergence: 0.0741\n",
            "Epoch 28/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0696 - acc: 0.3762 - kullback_leibler_divergence: 0.0743 - val_loss: 0.0725 - val_acc: 0.3460 - val_kullback_leibler_divergence: 0.0737\n",
            "Epoch 29/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0715 - acc: 0.3849 - kullback_leibler_divergence: 0.0749 - val_loss: 0.0721 - val_acc: 0.3480 - val_kullback_leibler_divergence: 0.0736\n",
            "Epoch 30/50\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0687 - acc: 0.3715 - kullback_leibler_divergence: 0.0737 - val_loss: 0.0724 - val_acc: 0.3500 - val_kullback_leibler_divergence: 0.0736\n",
            "Epoch 31/50\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0689 - acc: 0.3675 - kullback_leibler_divergence: 0.0740 - val_loss: 0.0719 - val_acc: 0.3540 - val_kullback_leibler_divergence: 0.0735\n",
            "Epoch 32/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0709 - acc: 0.3741 - kullback_leibler_divergence: 0.0739 - val_loss: 0.0721 - val_acc: 0.3560 - val_kullback_leibler_divergence: 0.0736\n",
            "Epoch 33/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0710 - acc: 0.3794 - kullback_leibler_divergence: 0.0751 - val_loss: 0.0723 - val_acc: 0.3460 - val_kullback_leibler_divergence: 0.0738\n",
            "Epoch 34/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0700 - acc: 0.3660 - kullback_leibler_divergence: 0.0735 - val_loss: 0.0722 - val_acc: 0.3520 - val_kullback_leibler_divergence: 0.0743\n",
            "Epoch 35/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0690 - acc: 0.3683 - kullback_leibler_divergence: 0.0735 - val_loss: 0.0719 - val_acc: 0.3520 - val_kullback_leibler_divergence: 0.0738\n",
            "Epoch 36/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0709 - acc: 0.3608 - kullback_leibler_divergence: 0.0751 - val_loss: 0.0719 - val_acc: 0.3520 - val_kullback_leibler_divergence: 0.0741\n",
            "Epoch 37/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0685 - acc: 0.3732 - kullback_leibler_divergence: 0.0732 - val_loss: 0.0714 - val_acc: 0.3580 - val_kullback_leibler_divergence: 0.0739\n",
            "Epoch 38/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0702 - acc: 0.3780 - kullback_leibler_divergence: 0.0746 - val_loss: 0.0713 - val_acc: 0.3680 - val_kullback_leibler_divergence: 0.0739\n",
            "Epoch 39/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0692 - acc: 0.3705 - kullback_leibler_divergence: 0.0740 - val_loss: 0.0710 - val_acc: 0.3740 - val_kullback_leibler_divergence: 0.0735\n",
            "Epoch 40/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0656 - acc: 0.3914 - kullback_leibler_divergence: 0.0713 - val_loss: 0.0713 - val_acc: 0.3640 - val_kullback_leibler_divergence: 0.0734\n",
            "Epoch 41/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0708 - acc: 0.3925 - kullback_leibler_divergence: 0.0749 - val_loss: 0.0707 - val_acc: 0.3760 - val_kullback_leibler_divergence: 0.0734\n",
            "Epoch 42/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0680 - acc: 0.3870 - kullback_leibler_divergence: 0.0730 - val_loss: 0.0710 - val_acc: 0.3700 - val_kullback_leibler_divergence: 0.0737\n",
            "Epoch 43/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0707 - acc: 0.3775 - kullback_leibler_divergence: 0.0745 - val_loss: 0.0706 - val_acc: 0.3800 - val_kullback_leibler_divergence: 0.0731\n",
            "Epoch 44/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0670 - acc: 0.3927 - kullback_leibler_divergence: 0.0728 - val_loss: 0.0706 - val_acc: 0.3760 - val_kullback_leibler_divergence: 0.0732\n",
            "Epoch 45/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0698 - acc: 0.3763 - kullback_leibler_divergence: 0.0753 - val_loss: 0.0707 - val_acc: 0.3760 - val_kullback_leibler_divergence: 0.0732\n",
            "Epoch 46/50\n",
            "8/8 [==============================] - 0s 7ms/step - loss: 0.0686 - acc: 0.3710 - kullback_leibler_divergence: 0.0740 - val_loss: 0.0709 - val_acc: 0.3760 - val_kullback_leibler_divergence: 0.0736\n",
            "Epoch 47/50\n",
            "8/8 [==============================] - 0s 9ms/step - loss: 0.0693 - acc: 0.3647 - kullback_leibler_divergence: 0.0742 - val_loss: 0.0712 - val_acc: 0.3660 - val_kullback_leibler_divergence: 0.0738\n",
            "Epoch 48/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0721 - acc: 0.3729 - kullback_leibler_divergence: 0.0766 - val_loss: 0.0714 - val_acc: 0.3560 - val_kullback_leibler_divergence: 0.0739\n",
            "Epoch 49/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0662 - acc: 0.3850 - kullback_leibler_divergence: 0.0713 - val_loss: 0.0709 - val_acc: 0.3740 - val_kullback_leibler_divergence: 0.0734\n",
            "Epoch 50/50\n",
            "8/8 [==============================] - 0s 8ms/step - loss: 0.0681 - acc: 0.3771 - kullback_leibler_divergence: 0.0720 - val_loss: 0.0704 - val_acc: 0.3820 - val_kullback_leibler_divergence: 0.0731\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2VzPv8US0de_",
        "outputId": "406b7d4f-e6af-4fd9-fad1-e7cbfa0abdc4"
      },
      "source": [
        "y_pred = model_final.predict(X_test)\n",
        "eval_sll = clf_eval(y_sl_test,get_single_label(y_pred),y_pred)\n",
        "eval_mll = clf_eval(y_ml_test,get_multi_label(y_pred),y_pred)\n",
        "eval = ldl_eval(y_test,y_pred)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "acc: 0.378\n",
            "f1 score(micro): 0.3937007874015748\n",
            "precision score(micro): 0.4\n",
            "recall score(micro): 0.3875968992248062\n",
            "AUC score(micro): 0.6334119761824515\n",
            "hamming loss: 0.20533333333333334\n",
            "label ranking loss: 0.32820000000000005\n",
            "confusion matrix: [[[366  18]\n",
            "  [ 90  26]]\n",
            "\n",
            " [[418   0]\n",
            "  [ 82   0]]\n",
            "\n",
            " [[142 207]\n",
            "  [ 31 120]]\n",
            "\n",
            " [[447   0]\n",
            "  [ 53   0]]\n",
            "\n",
            " [[475   0]\n",
            "  [ 25   0]]\n",
            "\n",
            " [[336  75]\n",
            "  [ 35  54]]]\n",
            "acc: 0.11\n",
            "f1 score(micro): 0.44332210998877664\n",
            "precision score(micro): 0.4034729315628192\n",
            "recall score(micro): 0.4919053549190536\n",
            "AUC score(micro): 0.6130441658527903\n",
            "hamming loss: 0.33066666666666666\n",
            "label ranking loss: 0.3286055555555556\n",
            "confusion matrix: [[[294  70]\n",
            "  [ 75  61]]\n",
            "\n",
            " [[316  22]\n",
            "  [139  23]]\n",
            "\n",
            " [[ 79 211]\n",
            "  [ 29 181]]\n",
            "\n",
            " [[247 123]\n",
            "  [ 65  65]]\n",
            "\n",
            " [[377  66]\n",
            "  [ 53   4]]\n",
            "\n",
            " [[300  92]\n",
            "  [ 47  61]]]\n",
            "Kullback–Leibler divergence:  0.07312496478522547\n",
            "Euclidean distance:  0.15293117601968456\n",
            "MSE:  0.004929100720677145\n",
            "Chebyshev distance:  0.12519112990550932\n",
            "Clark distance:  0.3788563824672573\n",
            "Canberra distance:  0.8219818198764313\n",
            "Cosine similarity:  0.9288942473491081\n",
            "Intersection similarity:  0.852513090432815\n",
            "Kurtosis_signed_offset:  -0.09315283974677543\n",
            "Kurtosis_abs_offset:  0.9622838342911059\n",
            "Laloss:  0.5277183370189406\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 622
        },
        "id": "scV9w0OcuKPS",
        "outputId": "839379f1-8d69-4def-c209-41b889a7dcff"
      },
      "source": [
        "def smooth_curve(points,factor=0.9):\n",
        "  smoothed_points=[]\n",
        "  for point in points:\n",
        "    if smoothed_points:\n",
        "      previous = smoothed_points[-1]\n",
        "      smoothed_points.append(previous*factor + point*(1-factor))\n",
        "    else:\n",
        "      smoothed_points.append(point)\n",
        "  return smoothed_points\n",
        "\n",
        "plt.clf()\n",
        "kullback_leibler_divergence = history_final.history['kullback_leibler_divergence']\n",
        "val_kullback_leibler_divergence = history_final.history['val_kullback_leibler_divergence']\n",
        "kullback_leibler_divergence_baseline = history_baseline_final.history['kullback_leibler_divergence']\n",
        "val_kullback_leibler_divergence_baseline = history_baseline_final.history['val_kullback_leibler_divergence']\n",
        "\n",
        "smooth_kl_history = smooth_curve(kullback_leibler_divergence)\n",
        "smooth_val_kl_history = smooth_curve(val_kullback_leibler_divergence)\n",
        "smooth_kl_history_baseline = smooth_curve(kullback_leibler_divergence_baseline)\n",
        "smooth_val_kl_history_baseline = smooth_curve(val_kullback_leibler_divergence_baseline)\n",
        "epochs = range(1,len(kullback_leibler_divergence)+1)\n",
        "\n",
        "plt.plot(epochs[10:],smooth_kl_history[10:],'ro',label='Training kullback_leibler_divergence')\n",
        "plt.plot(epochs[10:],smooth_val_kl_history[10:],'r',label='Validation kullback_leibler_divergence')\n",
        "plt.plot(epochs[10:],smooth_kl_history_baseline[10:],'bo',label='Training kullback_leibler_divergence baseline')\n",
        "plt.plot(epochs[10:],smooth_val_kl_history_baseline[10:],'b',label='Validation kullback_leibler_divergence baseline')\n",
        "plt.title('Training and validation smoothed kullback_leibler_divergence')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Kullback_leibler_divergence')\n",
        "plt.legend()\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-121-3312b0a78c04>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msmooth_kl_history\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'ro'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Training kullback_leibler_divergence'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msmooth_val_kl_history\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'r'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Validation kullback_leibler_divergence'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msmooth_kl_history_baseline\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'bo'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Training kullback_leibler_divergence baseline'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msmooth_val_kl_history_baseline\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'b'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Validation kullback_leibler_divergence baseline'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Training and validation smoothed kullback_leibler_divergence'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/pyplot.py\u001b[0m in \u001b[0;36mplot\u001b[0;34m(scalex, scaley, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2761\u001b[0m     return gca().plot(\n\u001b[1;32m   2762\u001b[0m         *args, scalex=scalex, scaley=scaley, **({\"data\": data} if data\n\u001b[0;32m-> 2763\u001b[0;31m         is not None else {}), **kwargs)\n\u001b[0m\u001b[1;32m   2764\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2765\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axes/_axes.py\u001b[0m in \u001b[0;36mplot\u001b[0;34m(self, scalex, scaley, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1645\u001b[0m         \"\"\"\n\u001b[1;32m   1646\u001b[0m         \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcbook\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormalize_kwargs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmlines\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLine2D\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1647\u001b[0;31m         \u001b[0mlines\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_lines\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1648\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlines\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1649\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_line\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axes/_base.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    214\u001b[0m                 \u001b[0mthis\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m                 \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 216\u001b[0;31m             \u001b[0;32myield\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_plot_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mthis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    217\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_next_color\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axes/_base.py\u001b[0m in \u001b[0;36m_plot_args\u001b[0;34m(self, tup, kwargs)\u001b[0m\n\u001b[1;32m    340\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    341\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 342\u001b[0;31m             raise ValueError(f\"x and y must have same first dimension, but \"\n\u001b[0m\u001b[1;32m    343\u001b[0m                              f\"have shapes {x.shape} and {y.shape}\")\n\u001b[1;32m    344\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: x and y must have same first dimension, but have shapes (40,) and (190,)"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD4CAYAAAAHHSreAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3debxVdb3/8deHSQJNBdFIFETsKqlhkkQq5hQgIimo6ImwNMprpXY1McxKRMX6OXDxagQqCY6oiUMiIpqWmgdkFJVRAZEplRAFDufz++O7TmzOtPc5e1h7eD8fj/XYe6+19tqfs84567O/4zJ3R0REJFGTuAMQEZH8o+QgIiI1KDmIiEgNSg4iIlKDkoOIiNTQLO4AMmGfffbxTp06xR2GiEhBmTVr1gZ3b1fbtqJIDp06daK8vDzuMERECoqZvVfXNlUriYhIDUoOIiJSg5KDiIjUoOQgIiI1KDmIiEgNSg4iuTB5MnTqBE2ahMfJkwvn8+OOXWKh5CCSKXVdRCdPhmHD4L33wD08DhuWu4tsOp8fd+wSH3cv+OXoo492kVhNmuTeqpV7uISGpVWrsL5jx13XVy0dO+76/o4d3c3C46RJmYst2efX99mpxC4FCyj3Oq6rsV/YM7EoOUjs6ruImtW+zSy8t77Ekqr6LvD1fX6yz04WuxS0+pJDStVKZtbHzN4xsyVmNryW7buZ2UPR9tfNrFO0vszM5iQslWbWrdp7p5rZgoTXbcxsupktjh73TqdkJJIT779f9/oDD6x9W9X6ESNgy5Zdt23ZEtanIlnVT32fn+yzk8UuRStpcjCzpsAdQF+gK3CemXWtttuFwEfu3gW4FRgN4O6T3b2bu3cDhgDL3X1OwrHPAjZXO9ZwYIa7HwLMiF6LxK++htn6LqKjRkGrVruub9UqrIf6E0sqn53sAl/f5yf77GSxJ4tNClddRYqqBegJTEt4fTVwdbV9pgE9o+fNgA2AVdvnBmBUwuvdgVcICWdBwvp3gPbR8/bAO8liVLWSZF2y6pdUtje2Xj8TVT91fX667SGZqBKT2JBOmwMwCBif8HoIMLbaPguADgmvlwL7VNtnKXB4wutbgTOBTtWSw8cJzy3xdV2LkoNkXTYblZNdYJN9djqNxule3NVgXdDqSw456cpqZj2ALe6+IHrdDTjY3R+v731R8F7HMYeZWbmZla9fvz7jMYvsIpWqn7IyWLECKivDY1lZascuK4Nx46BjRzALj+PG7Xx/Jqp+GvvZyaRbJZbKdolHXVmjaiED1UqEUsKvEl5fDHwArABWAduAF13VSpKv4vyGHHdX2HRiS7c6TrKKNKuVmgHLgIOAFsBc4KvV9rkEuCt6Phh4OGFbE2A10LmO43di12ql3wPDo+fDgZuTxajkIBmRr3Xr+XwBzXaVWLpJL66kWSDSSg7h/ZwGvEtoNxgRrbsOOCN63hJ4BFgC/DMxEQDfBl6r59jVk0NbQi+lxcDzQJtk8Sk5SNpSuQDHeaHJ54tcY8dYJNue7u8kn5NqnqgvOVjYXti6d+/uuhOcpKVTpzA+oLqOHUP7gTROsvNa33ao/71V4zsSu/G2arWzzUS/06TMbJa7d69tm+ZWEoHUGlal4ZI1lqczBiPZ+I5MNJaXMCUHEdBI4GxJ1huqvu3JfifJLv7J3q9JBetXV31TIS1qc5C0qX46/6Tb2J3u+0sAcY9zEMl76fb3l8xL9jtJVmWV7viRUldX1iikRSUHSUk+9/iRxknnd5rtkkOy2PLg7xFN2S0lT9VGUl0muso2thttnvw9KjmIqH5ZatPYC3yc82FlUH3JQeMcpDQ0aRL+/aozC3MhiVSXzhiMZH9vqfw9Tp4cuuVW3RNk1KiMt4FpnIOIuqpKQ9XXYJ1uN9oC6Gar5CClIZ2ZS6U01XcBT3ZxT2fwH6R2d8BsD+Crq76pkBa1OUhK8qB3iBSQdNocqt7f2N5KubjvuNff5hD7hT0Ti5KDuLsu/pJ5je2tlK4cNWjXlxzUIC3FIdkkbCKFJNnfc4Y6WKhBWopHXfWsqdTRihSKZKO7c9DBolnGjiSSbdW/TVX14ABNhSDFp6ys7lLvqFG1lywy2MFCJQcpHPWVDtRVVUpJDuYCU3KQ/FJf97z6SgfqqiqlpqwsDLirrAyPGW5bU3KQ/JFs4E99pQPNqiqSUUoOklv1lQySNSqnMkVzFr9JiZQSJQfJnWQlg2SNyiodiOSMkoM0TDpD9pOVDFJpVFbpQCQnlBwkdelOBpasZKBGZZG8oeQgqUt3MrBkJQNVG4nkDSUHSV2yb/7JShaplAxUbSSSF5QcJHXJvvknK1moZCBSMJQcJHXJvvmnMoWFSgbFxx2WLYN582DTprijkQxRcpCa6mo3yIPJwCRPfPgh3H8/XHghHHQQHHwwfO1rsOee0LYtHH00DBoEV14Jd9wBzzwDGzbEHbU0gKbsll2lM/W1ps0uXps3w4wZO5e33grr99oLTjwRTj4Z9t03lAaXL9+5rFgBW7eGfc3gW9+C/v3DcthhYZ3Epr4pu5UcZFf13VR9xYrk78/BTdElR/79b3jqKXjkEfjrX+Hzz0OyP/54OOmkkBC6dYOmTes+RmUlrF0LS5aEpDJ1Krz5ZtjWufPORNGrFzRvnpufS/5DyUFSl6GbiEiB+ve/4ckndyaErVvhy1+GgQPhrLOgZ0/Ybbf0PmPVqpB0nnwyJIytW+GLX4RTT4XevcOiqsicUHKQ1KVbcpDCNGcO/O53uyaEQYPg7LNDVVCTLDVPfvopPP98SBbPPhsSB4Qqp969oU+fUKr4whfC+ooKWLkyVFktW7ZzadkSrr02lEYkZUoOkjq1G5SW7dvhxhth5MjQflBWFhJCz57ZSwh1cYdFi0KSmDYNXnopJKqWLUP11bp14YvLjh0739OsWfjisnZt+FmGD4errtqZTKReSg7SMGo3KA3z58PQoaEN4PzzYcyY0NMoX2zZAn/7W0gUs2fD/vuHnlGdO+9c9t8/JIgPPoArroAHHgj7jBkDp58e90+Q95QcRGSnigoYPTpUI+29N9x1F5x5ZtxRZcbMmXDJJaEE0r8/3HabqprqUV9y0DiHUpTOzKpS2BYuDFVG11wTGpgXLiyexAChW+3cufD734dE0bVrSIKffRZ3ZDWtXQu//jV8//sh3mnTQgkoX76wu3vBL0cffbRLiiZNcm/Vyj38CYalVauwXorXqlXu11/v3qKF+z77uD/ySNwRZd+qVe6DB4e/8f32cx8xwn3Firijcl+61P3ii913283dzL19+13/H9u2dT/xRPdLL3UfPz78HFkClHsd19XYL+yZWJQcGqBjx13/EKuWjh3jjkwyacsW92nT3P/nf9wPP3zn73ngQPe1a+OOLrdmznTv39+9SZNwMe7Xz/3JJ90rKnIbx5tvhmTVpElI0j/6kfu774ZtGze6v/ii+5gxYX2PHju/xLVo4f7jH7svX57xkNJODkAf4B1gCTC8lu27AQ9F218HOkXry4A5CUsl0C3a9iwwF1gI3AU0jdb/Flid8J7TksWn5NAAZrUnB7O4I5N0LV3qfsst7r17u7dsufPCcvLJ7jff7D53rntlZdxRxue999x//eud39QPPNB95Ej3Dz7I3mdWVoaLfp8+4TP32MP9l79M7TN37HBfuND9Jz8Jv8dmzdx/8AP3xYszFl5ayQFoCiwFOgMtogt612r7/DdwV/R8MPBQLcc5Alia8PqL0aMBjwKDfWdyuCJZXImLkkMDqORQnF54YWdCOPTQUCXxzDPumzfHHVn+2bbNfcoU91NOCeerWTP3QYPCOcxU8ly0yP23v3U/7LDwGfvu637DDe4ffdS4461c6f6zn4XfcZMm7mVl7m+9lXaY6SaHnsC0hNdXA1dX22ca0DN63gzYQNQTKmGfG4BRtRy/OfAkcK4rOWSf2hyKzz/+4d66tftXvxpKD5K6d98NVW9t2uxMrLff3riL+NKlIQF87Ws7S+MnnOA+blyo5suENWtCvK1aheOffbb7ggWNPly6yWEQMD7h9RBgbLV9FgAdEl4vBfapts9S4PBq66YBHwH3V6tWWgHMA+4G9q4jrmFAOVB+4IEHNvrklKRJk0JJwSw8KjEUrtmz3ffc071Ll+xWjxS7LVvcJ04Mdf1VX5guush91qya+37+eaiiev1196lTQ5XdN76x88tWz57ut93mvnp19uJdt8796qtDNdXEiY0+TH3JIek4BzMbBPRx94ui10OAHu7+04R9FkT7rIpeL4322RC97hElmCNqOX5LYDKhWmq6me0XlTwcGAm0d/cf1hejxjlISXrrLTjhhDCC/eWXNR9RpsyeDXfeGaYk37IlTD++xx5hmvIPP4SPP675nqOPhnPPhXPOCSO2c+Vf/wrzUjVr1qi31zfOIZUjrgYOSHjdIVpX2z6rzKwZsCewMWH7YOCB2g7u7p+b2RPAAGC6u69NCPxPwFMpxCiJNMK5+C1ZAqecEmYynTFDiSGTvv51+NOfwtiDP/85jLquqIDDDw/nfL/94EtfCst++0GHDtC+fTyxtmmTtUOnkhzeAA4xs4MISWAwcH61faYCQ4FXCdVQL0RFFsysCXAOcHzVzma2O7CHu6+Jkkk/4OVoW3t3XxPteiahykpSVX1upKr7OIMSRLF4//0wXfa2bWF6iS5d4o6oOO21F/z852EpQUlHSLt7BfBTQvvAIuBhd19oZteZ2RnRbhOAtma2BPgFMDzhEL2Ale6+LGFda2Cqmc0jdFddR+jOCnCzmc2Ptp0IXN74H68EJbuPsxS2NWtCYvjkE5g+PYwAFskCza1UbHQ/huK1YQN8+9th6vTp08M0GCJp0NxKpUT3cS4+O3bAxInQvTssXRrufaDEIFmm5FBsRo0KvVcStWoV1kthqayEhx8ODaEXXBCm054+PZQeRLJMyaHYlJWFG/N07Biqkjp21I16Co17KB18/euhe2STJvDoo1BeDscdF3d0UiIa1zlW8ltZmZJBoZoxI0yn/dprcPDBMGkSDB4MTZvGHZmUGCUHkTh99BG8+GK4j/Lzz8O774Z+8+PGhaqk5s3jjlBKlJKDSC59/jn84x+hhPD886GqqLISWrcOo50vvzwkhZYt445USpySg0imbdoEy5fXvixZAlu3hmqib34z3AnslFPgmGOgRYu4Ixf5DyUHkXR99lnoRfTYY/DXv8K6dbtu32OPcNP7Ll2gb9/Q26hXr7BeJE8pOYg0xqZN8MwzISE88wx8+mmYbqFfPzjyyJAMqpY2bULPMZECouRQiDSxXjzWroUnn4THHw/tBdu2hcnXhgyBs84KJQI1IEuRUHIoNJpYL7fefhueeCIsr70WxiAcdBD87GchIXzzm2EcgkiR0dxKhaZTp5AQquvYMcy5I+nZsQNefTUkg6lTQ9dSCPP1n3EGDBgQqo1UTSRFIN37OUg+ef/9hq2X1LiHhHDllaFHUfPmcOKJcOmlISl06BB3hCI5peRQaA48sPaSgybWa7y5c8P4gpkz4bDDQtVdv36w555xRyYSG1WWFhpNrJc5a9eG9pqjjoJ582Ds2PB4/vlKDFLylBwKjSbWS9/nn8Po0XDIIXDPPaHUsHgxXHJJo+/FK1Js9J9QiDSxXuO9+CL88IdhtPIZZ4T7BH/lK3FHJZJ3VHLIR5Mnh15JTZqEx8mT446oOLz+emhLaNEijFN44gklBpE6qOSQbzSOITsWLYLTToP27eGll2C//eKOSCSvqeSQb0aM2JkYqmzZEtZL46xcCb17h+6pzz2nxCCSApUc8o3GMWTWxo0hMXzySSgxdO4cd0QiBUElh3xT13gFjWNouE8/DW0My5aF0c7dusUdkUjBUHLINxrHkBnbtsHAgfDGG/Dgg+FGOiKSMiWHfKNxDOmrrIQf/ACmTYM//hG++924IxIpOGpzyEcax9B47vCLX8D998MNN8BFF8UdkUhBUslBiseOHWG08+23hwnzhg+POyKRgqWSgxSHTZtg8OBwm85LL4VbbtG02iJpUMkhLhoFnTnLl0PPnuE+znfdBbfdphvwiKRJJYc4aBR05rzyCpx5ZqhSmjYNTjop7ohEioK+XsVBo6Az4957QzJo0ybcwlOJQSRjlBzioFHQ6amshKuuCt1Ve/UKiUET6IlklJJDHDQKuvE+/hjOOgtuvhl+8pPQAL333nFHJVJ0lBzioFHQjTNjBhxxBDz1FIwZA//3f2EyPRHJOCWHbKmvN5JGQTfMZ5+F7qmnnAKtW8Orr8LPfqauqiJZpN5K2ZBKbySNgk5NeTkMGQJvvw0//znceGPNUpeIZJxKDtmg3kjp274drrsujF/YvDmMYbj9diUGkRxRySEb1BspPe+8E0oLb7wB558PY8eq0Vkkx1IqOZhZHzN7x8yWmFmNCWvMbDczeyja/rqZdYrWl5nZnISl0sy6RdueNbO5ZrbQzO4ys6bR+jZmNt3MFkePhXdVUG+kxlm3Di67DI48EpYuhYceClV0SgwiOZc0OUQX7TuAvkBX4Dwz61pttwuBj9y9C3ArMBrA3Se7ezd37wYMAZa7+5zoPee4+9eAw4F2wNnR+uHADHc/BJgRvS4s6o3UMB9/DNdcE+7S9r//G0oN8+fDOefEHZlIyUql5HAMsMTdl7n7NuBBYEC1fQYAE6PnU4CTzWp0JTkvei8A7r4petoMaAF4LceaCOTnZPzqjZS+LVtg9OiQFEaNCndte+stGD8evvzluKMTKWmptDnsD6xMeL0K6FHXPu5eYWafAG2BDQn7nEu1pGJm0wjJ56+EpAKwn7uviZ5/CNR6N3gzGwYMAzgw19U16o2Unm3bQgIYORI+/BBOOw2uvx6OOiruyEQkkpPeSmbWA9ji7gsS17t7b6A9sBtQY2Icd3d2liiqbxvn7t3dvXu7du2yEHU91Bup8WbOhMMPh0sugUMOgZdfhqefVmIQyTOpJIfVwAEJrztE62rdx8yaAXsCGxO2DwYeqO3g7v458AQ7SxVrzax9dKz2wLoUYswt9UZquI0bw1xIJ50U5kZ6+ml46SU47ri4IxORWqSSHN4ADjGzg8ysBeFCP7XaPlOBodHzQcAL0bd+zKwJcA4J7Q1mtntCAmgG9APeruVYQwmJI7+oN1Lq3GHSJDj00PB49dWhsfm00zTCWSSPJU0O7l4B/BSYBiwCHnb3hWZ2nZmdEe02AWhrZkuAX7BrD6NewEp3X5awrjUw1czmAXMIpYO7om03Aaea2WLglOh1flFvpNQsXQq9e4feR126wOzZ4b7OX/hC3JGJSBIWfcEvaN27d/fy8vLcfujkyaGN4f33Q4lh1Cg1QFfZvh3+8Icwwrl5c7jpJvjxj6Fp07gjE5EEZjbL3bvXtk0jpBtLvZFq98orcPHFsGBBmFp7zBjYf/+4oxKRBtLcSpIZGzbAhRfC8cfDpk3wxBPw6KNKDCIFSslB0lNZCRMmhAbnP/8ZfvnLMJDtjDOSv1dE8paSQ13qGwEtwfz54TadF10EXbvCm2+GEc+tW8cdmYikSW0OtUllBHQpW7UqtCXceivstRfccw8MHaquqSJFRL2VatOpU0gI1XXsCCtWZO5zCsXmzWHA2vTp8NxzsGhRWH/RRaEnUtu28cYnIo2i3koNpRHQMHduGMU8fTr8/e+he2rLlnDCCSEp9O0Lhx0Wd5QikiVKDrU58MDaSw6lMgL6vvvg+98Pz486Ci6/HL7zHTj22JAgRKToKTnUZtSoXdscoHRGQM+fHwasnXACPPww7Ltv3BGJSAzUW6k2pXo/hk2bYOBA2HNPePBBJQaREqaSQ11KbQS0exjEtmwZvPACfOlLcUckIjFScpBgzBiYMgVuvjmMXRCRkqZqJYFXX4UrroABA8KjiJQ8JYdSt349nHNO6Il1770ayCYigKqVStuOHaFdZf36UHrYa6+4IxKRPFG6JQfNnQQjR4ZBbmPH6h7OIrKL0iw5aO4kePbZcDOeCy4IvZRERBKU5txKpT530qJFcNxx4V4Lr71W85anIlIS6ptbqTSrlUp57qT33gtTYTRvDo8/rsQgIrUqzeRQ1xxJxT530rp1cOqpYZbV556Dgw+OOyIRyVOlmRxGjar5jbnY50765BPo0yfci+Gpp+DII+OOSETyWGkmh1KbO+mzz6B/f1iwAB57LMyuKiJSj9LsrQSlM3fS9u1w9tnwyivwwAOh9CAikkTpJodSUFkZuqo+/TTcdRece27cEYlIgSjNaqVS4A6XXgr33w833hju0SAikiIlh2J1/fVh5PMVV8BVV8UdjYgUGCWHYvTmm/Db38L3vhem4NZkeiLSQEoOxaayEv77v6Ft23CPBiUGEWkENUgXm3vuCVNiTJwIe+8ddzQiUqBUcigmGzeG9oXjj4chQ+KORkQKmJJDMbn6avj4Y7jjDlUniUhalByKxWuvwfjxcNllcMQRcUcjIgVOyaEY7NgRGqHbt4ff/CbuaESkCKhBuhjceWfovvrQQ7DHHnFHIyJFQCWHQrd2LVxzTZiK++yz445GRIqEkkOhu/LKMOvq2LFqhBaRjEkpOZhZHzN7x8yWmNnwWrbvZmYPRdtfN7NO0foyM5uTsFSaWTcza2VmT5vZ22a20MxuSjjWBWa2PuE9F2Xqhy06L70E990XEsRXvhJ3NCJSRJImBzNrCtwB9AW6AueZWddqu10IfOTuXYBbgdEA7j7Z3bu5ezdgCLDc3edE7/mDux8KHAUca2Z9E473UNX73H18Oj9g0dq+HS65JNwP+1e/ijsaESkyqZQcjgGWuPsyd98GPAgMqLbPAGBi9HwKcLJZjTqO86L34u5b3H1m9HwbMBvo0LgfoUTdcgssXBimyNB9oEUkw1JJDvsDKxNer4rW1bqPu1cAnwBtq+1zLvBA9YOb2V5Af2BGwuqBZjbPzKaY2QEpxFhaZs+GX/8azjor3OFNRCTDctIgbWY9gC3uvqDa+maEhDHG3ZdFq58EOrn7kcB0dpZIqh9zmJmVm1n5+vXrsxh9nvn0UzjvPNh333BrUxGRLEglOawGEr+9d4jW1bpPdMHfE9iYsH0wtZQagHHAYne/rWqFu290963Ry/HA0bUF5e7j3L27u3dv165dCj9Gkbj0Uli8ODREt61eOBMRyYxUksMbwCFmdpCZtSBc6KdW22cqMDR6Pgh4wd0dwMyaAOcQtTdUMbPrCUnksmrr2ye8PANYlNqPUgIeeQQmTAhzKJ14YtzRiEgRSzpC2t0rzOynwDSgKXC3uy80s+uAcnefCkwA7jOzJcC/CAmkSi9gZUK1EWbWARgBvA3Mjtqux0Y9k35uZmcAFdGxLkj/xywC770HP/oR9OgRbuQjIpJFFn3BL2jdu3f38vLyuMPInooK+Pa3Yd48mDMHOneOOyIRKQJmNsvdu9e2TXMrFYLrr4e//x0mTVJiEJGc0PQZ+e7ll2HkyHDznrKyuKMRkRKh5JDPPvooJISDDgo38BERyRFVK+Urdxg2DNasgX/8Q1Nxi0hOKTnkq7FjYcoUuOkm+MY34o5GREqMqpXy0eOPh8Fu/fuHGVdFRHJMySHf/P3vcP75cMwx8OCD0ES/IhHJPV158smiRaG0cMAB8NRTmm1VRGKj5JAv1qyBvn2heXN49lnYZ5+4IxKREqYG6XywaROcdhps2BDu7qaBbiISMyWHuG3bBgMHwvz5oSrp6FonoRURySklhzi5w4UXwvPPwz33QJ8+cUckIgKozSFev/pVmC9p5Ei44IK4oxER+Q+VHOKwdStcfjnceSf8+McwYkTcEYmI7ELJIdeWL4ezz4ZZs+CKK+DGGyHcz0JEJG8oOeTS1KkwdGhoa/jLX2DAgLgjEhGpldoccqGiAq66KiSDzp1h9mwlBhHJayo5ZNsHH8DgweG+DD/5Cdx6K7RsGXdUIiL1UnLIphdegPPOg82bQ68k3axHRAqEqpWyZfx4OPVUaNsW3nhDiUFECoqSQ6a5w+9+Bz/6EfTuDf/8J3TtGndUIiINomqlTKqogIsvDqWGH/wA/vjHMJGeiEiBUckhUz79FM48MySGa66BCROUGESkYKnkkAkbNsDpp4e2hTvvDL2SREQKmJJDupYvD20LK1fCo4/Cd78bd0QiImlTckjH7NnhPgzbtoWZVY89Nu6IREQyQsmhsebPh5NPhi9+EWbOhMMOizsiEZGMUXJojBUrQlVS69bwt79Bx45xRyQiklFKDg21fn1IDJ99FqbEUGIQkSKk5NAQmzdDv37w/vuhjeHww+OOSEQkK5QcUrVtG5x1VmiE/stf1PgsIkVNySEVlZXhNp7Tp4d7PZ9+etwRiYhklUZIJ+Mebun5wAMwerTu9SwiJUHJIZkbb4QxY+AXv4Arr4w7GhGRnFByqM9998GIEfC978Hvf697PYtIyVByqMuWLXDFFaHh+e67oYlOlYiUDjVI12XcOFi3LsyXpNlVRaTEpPR12Mz6mNk7ZrbEzIbXsn03M3so2v66mXWK1peZ2ZyEpdLMuplZKzN72szeNrOFZnZTsmPl1GefhcbnE0+E447L+ceLiMQtaXIws6bAHUBfoCtwnplVv7XZhcBH7t4FuBUYDeDuk929m7t3A4YAy919TvSeP7j7ocBRwLFm1re+Y+XUhAnw4Yfwm9/k/KNFRPJBKiWHY4Al7r7M3bcBDwIDqu0zAJgYPZ8CnGxWo/X2vOi9uPsWd58ZPd8GzAY6NOBY2bN1K9x0E/TqBSeckLOPFRHJJ6kkh/2BlQmvV0Xrat3H3SuAT4C21fY5F3ig+sHNbC+gPzCjAcfCzIaZWbmZla9fvz6FHyNFd98Nq1fDtddm7pgiIgUmJ11wzKwHsMXdF1Rb34yQMMa4+7KGHNPdx7l7d3fv3q5du8wEum1bGNfwrW/BSSdl5pgiIgUoleSwGjgg4XWHaF2t+0QX/D2BjQnbB1NLqQEYByx299sacKzsmTgx3NHt2ms1pkFESloqyeEN4BAzO8jMWhAu9FOr7TMVGBo9HwS84O4OYGZNgHOI2huqmNn1hAv/ZakeK6u2b4cbboAePeA738n6x4mI5LOk4xzcvcLMfgpMA5oCd7v7QjO7DkHj6rEAAAU9SURBVCh396nABOA+M1sC/IuQQKr0AlYmVhuZWQdgBPA2MDtqbx7r7uOTHCt77rsv3MTnjjtUahCRkme5+FKebd27d/fy8vLGH6CiAv7rv2DvveGNN5QcRKQkmNksd+9e2zaNkAa4/35YtgyeeEKJQUQEza0EO3bA9ddDt27Qv3/c0YiI5AWVHB58EBYvhsceU6lBRCRS2iWHqlLDEUfAgOqDvkVESldplxymTIG334aHH9aU3CIiCUr7irj77qHEMHBg3JGIiOSV0i459OsXFhER2UVplxxERKRWSg4iIlKDkoOIiNSg5CAiIjUoOYiISA1KDiIiUoOSg4iI1KDkICIiNRTF/RzMbD3wXh2b9wE25DCchsrn+BRb4yi2xlFsjZNObB3dvV1tG4oiOdTHzMrruplFPsjn+BRb4yi2xlFsjZOt2FStJCIiNSg5iIhIDaWQHMbFHUAS+RyfYmscxdY4iq1xshJb0bc5iIhIw5VCyUFERBpIyUFERGooquRgZneb2TozW5Cwro2ZTTezxdHj3nkU22/NbLWZzYmW02KK7QAzm2lmb5nZQjO7NFof+7mrJ7bYz52ZtTSzf5rZ3Ci230XrDzKz181siZk9ZGYt8ii2e81secJ565br2BJibGpmb5rZU9Hr2M9bPbHlxXkzsxVmNj+KoTxal5X/06JKDsC9QJ9q64YDM9z9EGBG9DoO91IzNoBb3b1btDyT45iqVAD/4+5dgW8Cl5hZV/Lj3NUVG8R/7rYCJ7n714BuQB8z+yYwOoqtC/ARcGEexQZwZcJ5mxNDbFUuBRYlvM6H81alemyQP+ftxCiGqrENWfk/Lark4O5/A/5VbfUAYGL0fCLw3ZwGFakjtrzg7mvcfXb0/N+Ef4r9yYNzV09ssfNgc/SyebQ4cBIwJVof13mrK7a8YGYdgH7A+Oi1kQfnrbbYCkBW/k+LKjnUYT93XxM9/xDYL85gavFTM5sXVTvFUuWVyMw6AUcBr5Nn565abJAH5y6qfpgDrAOmA0uBj929ItplFTEls+qxuXvVeRsVnbdbzWy3OGIDbgN+CVRGr9uSJ+eNmrFVyYfz5sBzZjbLzIZF67Lyf1oKyeE/PPTbzZtvT8CdwMGEYv8a4P/FGYyZ7Q48Clzm7psSt8V97mqJLS/OnbvvcPduQAfgGODQOOKoTfXYzOxw4GpCjN8A2gBX5TouMzsdWOfus3L92cnUE1vs5y1ynLt/HehLqGLtlbgxk/+npZAc1ppZe4DocV3M8fyHu6+N/oErgT8RLi6xMLPmhIvvZHd/LFqdF+euttjy6dxF8XwMzAR6AnuZWbNoUwdgdWyBsUtsfaJqOnf3rcA9xHPejgXOMLMVwIOE6qTbyY/zViM2M5uUJ+cNd18dPa4DHo/iyMr/aSkkh6nA0Oj5UOCJGGPZRdUvNHImsKCufbMchwETgEXufkvCptjPXV2x5cO5M7N2ZrZX9PwLwKmENpGZwKBot7jOW22xvZ1wETFC3XTOz5u7X+3uHdy9EzAYeMHdy8iD81ZHbN/Lh/NmZq3NbI+q58B3ojiy83/q7kWzAA8Qqhi2E+osLyTUZc4AFgPPA23yKLb7gPnAvOgX3D6m2I4jFEXnAXOi5bR8OHf1xBb7uQOOBN6MYlgAXBut7wz8E1gCPALslkexvRCdtwXAJGD3OP7mEuL8NvBUvpy3emKL/bxF52dutCwERkTrs/J/qukzRESkhlKoVhIRkQZSchARkRqUHEREpAYlBxERqUHJQUREalByEBGRGpQcRESkhv8P8GtsVQa0Ed8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 249
        },
        "id": "tP4bVifIw-cs",
        "outputId": "e6a7b8c3-1ce2-4747-ba2c-353c6ffc1f03"
      },
      "source": [
        "def smooth_curve(points,factor=0.9):\n",
        "  smoothed_points=[]\n",
        "  for point in points:\n",
        "    if smoothed_points:\n",
        "      previous = smoothed_points[-1]\n",
        "      smoothed_points.append(previous*factor + point*(1-factor))\n",
        "    else:\n",
        "      smoothed_points.append(point)\n",
        "  return smoothed_points\n",
        "\n",
        "plt.clf()\n",
        "la_loss = history_final.history['laloss']\n",
        "val_la_loss = history_final.history['val_laloss']\n",
        "la_loss_baseline = history_baseline_final.history['laloss']\n",
        "val_la_loss_baseline = history_baseline_final.history['val_laloss']\n",
        "\n",
        "smooth_laloss_history = smooth_curve(la_loss)\n",
        "smooth_val_laloss_history = smooth_curve(val_la_loss)\n",
        "smooth_laloss_history_baseline = smooth_curve(la_loss_baseline)\n",
        "smooth_val_laloss_history_baseline = smooth_curve(val_la_loss_baseline)\n",
        "epochs = range(1,len(la_loss)+1)\n",
        "\n",
        "plt.plot(epochs[10:],smooth_laloss_history[10:],'ro',label='Training laloss')\n",
        "plt.plot(epochs[10:],smooth_val_laloss_history[10:],'r',label='Validation laloss')\n",
        "plt.plot(epochs[10:],smooth_laloss_history_baseline[10:],'bo',label='Training laloss baseline')\n",
        "plt.plot(epochs[10:],smooth_val_laloss_history_baseline[10:],'b',label='Validation laloss baseline')\n",
        "plt.title('Training and validation smoothed laloss')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('laloss')\n",
        "plt.legend()\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-122-da97f72375c5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mla_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhistory_final\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'laloss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0mval_la_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhistory_final\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val_laloss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0mla_loss_baseline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhistory_baseline_final\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'laloss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyError\u001b[0m: 'laloss'"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 0 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 249
        },
        "id": "SU_iONL9AI3S",
        "outputId": "38e5f675-4505-40d9-876c-566b9935bf93"
      },
      "source": [
        "def smooth_curve(points,factor=0.9):\n",
        "  smoothed_points=[]\n",
        "  for point in points:\n",
        "    if smoothed_points:\n",
        "      previous = smoothed_points[-1]\n",
        "      smoothed_points.append(previous*factor + point*(1-factor))\n",
        "    else:\n",
        "      smoothed_points.append(point)\n",
        "  return smoothed_points\n",
        "\n",
        "plt.clf()\n",
        "la_loss = history_final.history['new_laloss']\n",
        "val_la_loss = history_final.history['val_new_laloss']\n",
        "la_loss_baseline = history_baseline_final.history['new_laloss']\n",
        "val_la_loss_baseline = history_baseline_final.history['val_new_laloss']\n",
        "\n",
        "smooth_laloss_history = smooth_curve(la_loss)\n",
        "smooth_val_laloss_history = smooth_curve(val_la_loss)\n",
        "smooth_laloss_history_baseline = smooth_curve(la_loss_baseline)\n",
        "smooth_val_laloss_history_baseline = smooth_curve(val_la_loss_baseline)\n",
        "epochs = range(1,len(la_loss)+1)\n",
        "\n",
        "plt.plot(epochs[10:],smooth_laloss_history[10:],'ro',label='Training laloss')\n",
        "plt.plot(epochs[10:],smooth_val_laloss_history[10:],'r',label='Validation laloss')\n",
        "plt.plot(epochs[10:],smooth_laloss_history_baseline[10:],'bo',label='Training laloss baseline')\n",
        "plt.plot(epochs[10:],smooth_val_laloss_history_baseline[10:],'b',label='Validation laloss baseline')\n",
        "plt.title('Training and validation smoothed new laloss')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('new laloss')\n",
        "plt.legend()\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-123-7664dfa453cd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mla_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhistory_final\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'new_laloss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0mval_la_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhistory_final\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val_new_laloss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0mla_loss_baseline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhistory_baseline_final\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'new_laloss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyError\u001b[0m: 'new_laloss'"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 0 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FilxWTv_xNfM"
      },
      "source": [
        "## VGG16"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RKFRmxgNxTG9"
      },
      "source": [
        "from keras.applications import VGG16\n",
        "\n",
        "conv_base = VGG16(weights='imagenet',\n",
        "          include_top=False,\n",
        "          input_shape=(150,150,3))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gvLKrZ0EBA10"
      },
      "source": [
        "## Multilayer perceptron"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZPaeT5zWdHPF"
      },
      "source": [
        "Define network structure\n",
        "\n",
        "Simple multi-layer perceptron, input -> two hidden layers -> output."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MnHLdjJqAoov"
      },
      "source": [
        "def perceptron(feature_dim,label_dim):\n",
        "    model = Sequential()\n",
        "    print(\"create model. feature_dim ={}, label_dim ={}\".format(feature_dim, label_dim))\n",
        "    model.add(Dropout(0.2, input_shape=(feature_dim,)))\n",
        "    model.add(Dense(500, activation='relu' ,kernel_regularizer=regularizers.l2(0.0001)))\n",
        "    model.add(Dropout(0.5))\n",
        "    model.add(Dense(200, activation='relu' ,kernel_regularizer=regularizers.l2(0.003)))\n",
        "    model.add(Dense(label_dim, activation='sigmoid',kernel_regularizer=regularizers.l2(0.0001)))\n",
        "    model.compile(optimizer='adam',loss='binary_crossentropy', metrics=['acc', precision, recall, f1measure]) \n",
        "    return model\n",
        "\n",
        "\n",
        "def train_perceptron(X_train,y_train,X_test,y_test):\n",
        "    feature_dim = X_train.shape[1]\n",
        "    label_dim = y_train.shape[1]\n",
        "    model = perceptron(feature_dim,label_dim)\n",
        "    model.summary()\n",
        "    fit_history = model.fit(X_train, y_train, batch_size=128, epochs=100, validation_data=(X_test,y_test), verbose=0)\n",
        "    return model,fit_history"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sYoimGBLVbFC"
      },
      "source": [
        "Define metrics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7PMmP5LzVc0f"
      },
      "source": [
        "def precision(y_true, y_pred):\n",
        "    # Calculates the precision\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
        "    precision = true_positives / (predicted_positives + K.epsilon())\n",
        "    return precision\n",
        "\n",
        "\n",
        "def recall(y_true, y_pred):\n",
        "    # Calculates the recall\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
        "    recall = true_positives / (possible_positives + K.epsilon())\n",
        "    return recall\n",
        "\n",
        "def fbeta_score(y_true, y_pred, beta=1):\n",
        "    # Calculates the F score, the weighted harmonic mean of precision and recall.\n",
        "\n",
        "    if beta < 0:\n",
        "        raise ValueError('The lowest choosable beta is zero (only precision).')     \n",
        "    p = precision(y_true, y_pred)\n",
        "    r = recall(y_true, y_pred)\n",
        "    bb = beta ** 2\n",
        "    fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())\n",
        "    return fbeta_score\n",
        "\n",
        "def f1measure(y_true, y_pred):\n",
        "    # Calculates the f-measure, the harmonic mean of precision and recall.\n",
        "    return fbeta_score(y_true, y_pred, beta=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "52nvNwqKeX-0"
      },
      "source": [
        " Load a multi-label dataset, if not done before."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SFM0iRskebSH",
        "outputId": "1251c8d0-5e79-4b7d-c4ad-da6a16ebb2f0"
      },
      "source": [
        "data = spio.loadmat(data_path['slashdot'])\n",
        "X = data['features']\n",
        "y = data['labels']\n",
        "X_train,X_test,y_train,y_test = model_selection.train_test_split(X,y,test_size=0.3)\n",
        "print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(2647, 1079) (1135, 1079) (2647, 22) (1135, 22)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pA8kZAUZdLQz"
      },
      "source": [
        "Fit the model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BOvIpT98As9h",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b2a99815-8548-4ce1-820d-aedde0409bee"
      },
      "source": [
        "[clf_nn, clf_nn_history] = train_perceptron(X_train,y_train,X_test,y_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "create model. feature_dim =1079, label_dim =22\n",
            "Model: \"sequential_4\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "dropout_16 (Dropout)         (None, 1079)              0         \n",
            "_________________________________________________________________\n",
            "dense_16 (Dense)             (None, 500)               540000    \n",
            "_________________________________________________________________\n",
            "dropout_17 (Dropout)         (None, 500)               0         \n",
            "_________________________________________________________________\n",
            "dense_17 (Dense)             (None, 200)               100200    \n",
            "_________________________________________________________________\n",
            "dense_18 (Dense)             (None, 22)                4422      \n",
            "=================================================================\n",
            "Total params: 644,622\n",
            "Trainable params: 644,622\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xc5e-Ki5dt9B",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "ede5d219-c777-481d-e176-275a68946a77"
      },
      "source": [
        "# plot history\n",
        "plt.plot(clf_nn_history.history['acc'], label='train')\n",
        "plt.plot(clf_nn_history.history['val_acc'], label='test')\n",
        "plt.ylabel('acc')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3yV5f3/8deHJBD2RmQGWYqKIAEFpOIGB24rTqwWW2dbR+Xb1lZbW+uq2jp+DlDrLi5QVFABZQlhiOw9AgIh7BGSc87n98d9kpyEAwTIIZC8n4/HeST3dd/3Odd97uT63Nf1uYe5OyIiIsVVKusKiIjI4UkBQkRE4lKAEBGRuBQgREQkLgUIERGJK7msK1BaGjRo4GlpaWVdDRGRI8rUqVPXu3vDePPKTYBIS0sjIyOjrKshInJEMbPle5qnISYREYlLAUJEROJSgBARkbgUIEREJC4FCBERiUsBQkRE4lKAEBGRuBQgROSItW1XiKFTM9kVCpd1VcqlcnOhnEhFtDM3TOXkSiRVsrKuyiHn7tw/9AdG/LiGtVtyuP2MNmVdpXJHPQiRI9SG7bmc/dRYrn1lEqFwpKyrc8i9PmEZI35cQ6OaVXh+9CLWbc0p6yqVOwoQclgaOjWTvwybjZ54GF8k4vz2vRms2ZLDpCUbeHLUgrKu0kFzd14bv5SHhs/e57IzVm7ikRFzOfu4Rrx3a3dywxGe/HLf30H2tl3c/vY03puyojSqXO4pQMhhZ2dumL+PmMtrE5Yxev66Q/75ExavZ+WGHYf8c/fmqzlrmbVqc8H0C2MXM3ZBFg/1O57+3VrwwpjFfDNvLRA0tOMXrWfaio0H9Fk5eWE+m/kTuaGD75VMXrqB5dnb97lcOOI8+Mls/jJ8DkPGL2PO6i17XHbzjjxuf2sajWqm8sSVJ9GqQXVu7J7G+1NXFvmOilu2fjuXvzCBz2b+xO8/+JHHv5yX8AOQlRt28MmMVYQjR+aBjnIQckjsCoXZsjNUMN2gRmXM4o+bfzAtkw3bc6lbLYW/fTaXXm0bkpK072OZXaEwVZKTDriO7s4zXy/k6a8W0ql5HT66rcce61jS99uwPZf8tqFa5SSqV9n/f7lPZ67mjrenA9CtVT3OOrYRT46cT7+TmnDtKS3YFYrww8pN/Pa9H7j7rLa8M3kFC9dto0aVZEbf25uGNavs8b3zwpHdvtsnR87n5e+WclqbBjx/3cnUSk0pmLc/3/GUZRvo//IkGtWswmd39aJe9cpxl9uZG+aud6czas5arj+1Je9NWcnQqZk82KRD3OWfGDmfNVtyGPqr7tSpFrznnWe15YNpmfz10zm8O/DU3fbb9BUbufn1DNyd//2qOx9OW8VzoxezelMO/7y8I5WTS/dYeeryDbw6bilfzFpDxGFLTojrT21ZonXzwhGSK9lB/e2VFvUgJOGmrdhIz0e/oesjXxW8rnhxIhu25+62bDjivDpuKSc1r8PjV5zEkqztvDlpjzebLPDO5BV0ePBLfvlGBpOWZO/3kWFeOMLvP5jJ018tpG2jGsxYuYmM5Qd2BJ4bivDhtEwu/Pc4uvytcJs7PzyKD6dl7td7LV2/nQc++JHOLerwf+cfy6qNO/nH5/NIa1Cdv192ImZGakoSz197MuGI8/Cnc0hOqsQfzj+OnLwwT+1h6GlXKMxv3p1O10e+YnHWtoLyZeu389qEZZzUrDaTlmRz1YsT+WnzTiYuzuaW1zM47k9flGh4JnvbLu54exqNa6WSvS2X3743g0ixo+jNO/J4cexiznxyDF/NXctD/Y7nr5ecwNkdGvHxjFVxezAL127l7ckruPaUFnRuUbegvHbVFH53Tju+X7qBa17+nq/mrCUScX7M3Mxv35vBVf9vIjVTk/nwtp50TavH3y89gfvOa89H01cxYMhktuTklXSX7FU44vzp41lc/sJExi1czy9/dgzpLevyr1EL2Lxz758x96ct3Pu/Hzj+wS/p+8x3vD9lJTl5ZXt2liWyi2VmfYBngCTgFXd/NM4yVwF/ARz4wd2viZZ/AZwKjHP3C/f1Wenp6a7bfRcViTjPjV5ExOHOM9tQKeZMl/emrGDKso27/aPtydL123lhzCLuPbc9jWqllrgOI2ev4a53p9OoZiq39GqFmbFlZx7Pfr2QJnWq8tpNXWlZv3rB8l/MWsOv3pzKc9eczPknNub6Vyfz46rNjL2vd8HRYnGzVm3msucncEzD6qzdksPGHXm0bVSj4Mi5SnIl7j67HZ2a14m7/rZdIW5/axpjF2Rx11lt+dXpx9Dj0W/omlaPl29IL/G2btiey1uTlvPGpOVkbd1F20Y1uLxLs4Jew4iZPzFxSTb3ntuO289os88jxJy8MJc+P4E1m3fy2V29aFKnKqFwhLELsjj26Fo0rVO1yPKzV29m+64wXdPqYmY8NHx2kMi9uxfHNq5VsNzmHXkM/G8G3y/dQLXKSbSoV42PbutJ1cpJDHwjg/GL1jP6vt7MX7OVX785jdxQhNxwhHrVK9OwRhWWZm/no9t6cHyT2gBs2pHL3z6byzENq3NNtxbUTE1hwJDJfL90Ax/d1oPpKzbxx49ncd957bn9jDYsXb+dIeOXMnRqJjtyw3Q/pj53ntmGHm0aADB63jpuem0KL17XhT4nNC6yjTcOnsy0FRsZe98Zu/VIwhFn8LilDB6/lJ8251CvemU2bM+leuUkrkxvzp1ntqF+jaK9qQ+nZXL/0Jm0aVSDITd15ejaVVm0bhuvT1hWJHC2rF+NG7qncdzRwfe4ftsu3pq0gpUbd3Bll2Z0a1WPnLwId74zna/mruWW01rx23PaUb1KMrNWbeai/4zjltNa8YcLdu8V5eSF+fWbUxk9P4uqKUlcdNLRzMzczLw1W6lfvTKDzj+OK7o02+vfysEws6nuHvcPPWEBwsySgAXAOUAmMAXo7+5zYpZpC7wPnOnuG82skbuvi847C6gG3KoAsW9Tl2+kRpVk2jeuCQR/dPf87wc+m/kTABeceDRPXnUSlZMq8diX83lx7GKSKhnhiNO5RR1uPq0VfY5vTHKcoZyduWEueW4889du5ZRW9XjrllPiLpeTF2bM/HUFQ0krN+7gudGLOLFZHV69MZ0GMf+cU5dv4ObXM0gy49/XdKZH66BxuPyFCazbmsPoe3qTnFSJeWu2cP4z39HnhMb0btcIgDrVUujdvhGVkyuxJSePC58dR24owoi7e1GtchIfTV/FpzNXFxyBLsvewdacPP7T/2TO7nBUkTqv25LDTa9NYd6arTxyyQlc3a0FEAyz/Gf0Ir763em0blhjr9/9grVbGTxuKR9NX8WuUITT2zXk5tNa0attgyJBIDcU4f6hP/DxjNVc3bU5f7jgOGrGDN/EcncGffgj705ZyZCbunJG+0Z7rUM8m3bkcvrjYzihaS3evPkUzIxF64JGf1n2dp648iRqV01hwJApXJXejEs6N+Wal78vaMgB5qzewlOjFnD2cY24pHNTtu8KccGz40hNqcTwO09j0448BgyZzLLsHYQjTtWUJDo2q833Szfwj8tOpH+3Frg7d787g09nrqZH6waMX7yelEqV6NepCb/o2YoOTWoVqXcoHKHHo9/QsVltXrmxa0H56PnruGnIFP54wXHc0uuYPW53XjjCF7PW8NnMn0hPq8tVXZsXGSYrbtzC9fzqzakF/z9jF2RRObkSJzatTSUDd5i9egs788L0aF2fJnWqMuyH4O+rRpVktu0KcULTWhjGrNWbeajf8dzQPa3IZ9z3vx/4eMYqRv32dNIaVC8y7z/fLOSJkQu455x23NA9jdrVUnB3Ji7O5l9fLWDKso3cfVZbfnN2W8wMd2fq8o1USU7ixGa1S/rnsEdlFSC6A39x9/Oi04MA3P0fMcs8Bixw91f28B69gXsVIPbule+W8MiIubhDzzb1uf7Ulgwev4zJSzcwqO+xVDLjkRFz6ZpWl8a1qzL8h9Vce0oL7u9zLB9Ny2TIhGUsz95B0zpVubFHS37etQW1qxb+Q933vx8YOi2T/t1a8Pb3K7j9jNbcd96xBfOztu7ire+X8+ak5azfVnTY6OzjjuLZ/p2oVnn3sfclWdsYMGQKKzbsoFPzOpx5bCOeGrWAv1zUgQE9WxUs99dP5/DquKVF1j2qVhVu6J7GzMxNfDV3He8NPJX0tHpxv5+srbu4+fUpzFq1mYcvPoEr04OjsWXrd/CL16awcUcuz117cpFGOGvrLnr+8xuu6NKMv1964m7v6e6MXZDFq+OW8t3C9VRJrsRlJzfjFz3TaHtUzT3uK3fn8S/n8/yYxdSoksxV6c25qWcazetVK1gmFI7w4LDZvP39Cm7r3Zr7+xy7x/fbl9fGL+Uvw+dwzzntmLZiI6PnZ1EzNZn/d32XgqD8xJdBMKxXvTJVU5L4+p7TSU3Zc55hyrINXP3SJLofU595a7aSGwrz8g3p1KqawpDxS/l4xmou7Hg0T155UkGA3LYrxKXPjWfD9lyuPbUl153agkY199wTffTzebz83RImDjqTRjVTyQtH6PvMd4Qjzpe/+Vmp5wzm/rSFm4ZMIRRxbujekmtPaVGkt7FpRy7vTF7J6xOWsWlnLld0acZNPVvRpHZVPpq+isHjl5K5cQfPXN2Z845vvNv7r9uSQ+8nxtCrbQP+3/Xp+yzPlxeOMOjDHxk6NbOgpzJ4/DLm/hQk8fd1cFcSZRUgrgD6uPst0enrgVPc/Y6YZT4m6GX0JBiG+ou7fxEzvzd7CRBmNhAYCNCiRYsuy5fve6y6rEQizpgF63h13FJmZm6m30lNuKlnK1o3rM7U5RsZPH4pY+dnccaxjbj5tFZ0blGXBWu3MmT8Uj794Sc6t6zLL3qmcXq7hgX/dJGI87fP5jJ4/FL6HN+Yjs1r88aE5azZkkPlpEo8fmVHLu7UFIDhP6zmnvd/IDcc4f4+7fn16a0L3icccb6Zt45Xxy1h0pJg2OHK6D/A5GUbuH/oTO48sw33nNue3w+dyXsZwVFt41qpDB63lE+iR1NnHtuIAT3SaN0oOOJOMuOoWlX2OpSyfVeID6ZlMmT8Mpau307tqilMHHRmkYDi7qzZklOQ7J2/ZgtDxi/ju4XrARjU91huPb31Xr//Hbkh7nh7Ot/MK3pWVIMaVRgyoGvcI7FBH87kg2mrmPDAmQW9n525YT6cHtR30bptNKpZhRt7pNG/W4s9JmHjmZm5iVfHLeWzmT8RcefcDo35xWmtOKFpLe58ezpfz1vHbb1bc++57YsMDe6vvHCEPk9/y+Ks7TSoUZnrTm3Jtae0LJK4DoUjXPfq90xasoFn+3em30lN9vm+L45dzKOfz6NpdJgwNijuyA2Rmpy0W713hcJUMivRCQeLs7Zx1pNjeaDvsbRtVIOXvl3C90s38NL1XTg3TgNcGnLywiRV2nv9QuEIoYjvFkAjEWdnXnivJyHk9xT+evHxXB/tYeT3LL763elFhlpjuTv/+mohz369EID2R9Xkpp5p5OSFCw7uTmxam2F39DygxPbhHCA+BfKAq4BmwLfAie6+KTq/N4e4B/HYF/NYkrXv0/L214K1W1myfjuNa6XSpWVdRs1dS24oQot61VixYQe1UpM5vX0jxsxfx9acEC3rV2N59g6qJFfi7A5HMWXpBtZt3UXrhtVp2yj4Z1yzJYcZKzdxU880/nhBB5IqGXnhCKPmrKVJnaq7jbn/mLmZjTty+Vm7uI+fBYJx7MHjljHsh1WEIk5yJSO9ZT3evOUUkioZOXnBcNOSrO3khiNUTUni8i5No8Fu70MxexOJOGMXZlErNZkuLeP3BIqbv2Yrs1dv5pJOTUvUiIbCEYZOzSQ7mhxPqmRcdFKT3cby8y1at42znxpLx2a1aVK7KhF3Ji/bwKYdeZzQtBY3n9aKC05sclBHs2s25/DGxGW8PXkFm3bkUTM1me27Qjx08QklPutlXxat28bcn7ZwToej9tgz2LQjl0lLsjnv+MYlamQiEWf4zNV0b11/rz2Bg3HZ8+OZtmITAEfXTmXgz45hQI+0w+LsngORkxfmtrem8c28dfy6d2vOP+Fo+j03jl/2Oob/O/+4fa7/zby1VElOokfr+kUO7r6eu5YtOaEDzlMczkNMLwLfu/uQ6PTXwAPuPiU63ZtDGCBC4Qht/vA5DWpUpn71PZ8aeCDqVk+hf7cWnH/i0aQkVSpIck1aks35Jzbm8i7NqFY5GM8cmrGSL2av4bQ2DbjmlJbUq16Z3FCET2eu5t0pK9m8Izgbwgyu7tq8yHBMaVm3NYc3Jy5n+spNPHnVSUUagSVZ2/jDR7P4WbuG9O/WfI/J4/Lgkc/m8O2C9QXTrRtVZ0CPVgWJ4NKyMzfMB9MyGf7Dan7Z65jdciUV0ej56xgyfhlXdGlG3xMal6jncbiLHT6sUSWZysmVGHNf773mSBKtrAJEMsHw0VnAKoIk9TXuPjtmmT4EiesbzawBMB3o5O7Z0fm9OYQBIicvzLF/+oL7+7Tntt66r4uIlD535/kxi3n8y/kFifyytLcAkbAL5dw9ZGZ3AF8S5BcGu/tsM3sYyHD3YdF555rZHCAM3BcTHL4DjgVqmFkmcLO7f5mo+gKEooPcyRXwxmcicmiYGbef0YZrT2lx2Pe+E3oltbuPAEYUK3sw5ncHfhd9FV+3VyLrFk84nB8gjvyurIgc3g734AC6krqIUCQ4bz45ST0IEREFiBj5Q0wV8d76IiLFKUDEUA5CRKSQAkQM5SBERAqpJYyhHISISCEFiBhh5SBERAooQMTICysHISKSTwEiRmEPQl+LiIhawhjKQYiIFFKAiBHWaa4iIgUUIGLoQjkRkUIKEDFCug5CRKSAWsIY+TkI9SBERBQgisjPQaQoSS0iogARSzkIEZFCChAxCs9i0tciIqKWMEZeWDkIEZF8ChAxdB2EiEghBYgYBc+DUJJaREQBIpZyECIihdQSxggpByEiUkABIoYeOSoiUkgBIkZYOQgRkQIKEDFCykGIiBRQSxhDjxwVESmkABEj/0I55SBEjhDukLUg/rzsxRC9AaccmIQGCDPrY2bzzWyRmT2wh2WuMrM5ZjbbzN6OKb/RzBZGXzcmsp75whHHDCopQIgcGWa+D891hfmfFy1fOQX+3QVmvlc29SonEhYgzCwJeA7oC3QA+ptZh2LLtAUGAT3d/XjgN9HyesCfgVOAbsCfzaxuouqaLxRxUpR/EDkyRMLw7WPB72P/GfQm8n37GOCwZExZ1KzcSGRr2A1Y5O5L3D0XeBe4uNgyvwSec/eNAO6+Llp+HjDK3TdE540C+iSwrkDQg1D+QYoI5wUNkZS9cF7R6dkfQfYiaNcXVk+HxV8H5aunw8KRkFQFlo8/9PUsbscG2LomeO3YUNa12S+JDBBNgZUx05nRsljtgHZmNt7MJplZn/1Yt9SFwq78gxSKRGDwefDxbWVdE8kYAv9sBUvGBtORCHz3JDQ8Fq58DWo1g7GPB72Ib5+A1Npw+n2weSVsWlF29V78DTzWCp5sH7wei9mGI0BZj6ckA22B3kB/4GUzq1PSlc1soJllmFlGVlbWQVcmFImQpGsgJN+Cz2HVVJjzCeRuL+vaVFx5OTDmUcjdCu9cDcsnwPzPYN0c6HUvpKTCab+BlZNg8ksw71M45VdBzwKC5cvKD+8FwerCfwWvmk2C4bAjRCIDxCqgecx0s2hZrExgmLvnuftSYAFBwCjJurj7S+6e7u7pDRs2POgKhyLqQUiUO4x9DFKqQ2gnLBxV1jWquKb/F7atgctfhdrN4K0rYdSDUO8YOP7SYJnO10ONxvD5/VC5RhAgGnUIGudl48qm3qHcIHne/gJI/0XwOu03wbDXssNg6KsEEhkgpgBtzayVmVUGrgaGFVvmY4LeA2bWgGDIaQnwJXCumdWNJqfPjZYlVDjsukjucDf+GXi0JfyjefD6dxdYO7v0P2fR1/DTDDjvEahWH+YW/9OVfXKHd66BEfeX7HRTdxj5J3j1XNgWHREI5cK4p6H5KXDC5XDDMKjRCDYsgV73QFJysFxKKvS8K/i92y+hWj2oVAla9DiwHsSwu+Cze/Z/vVhLv4Vdm6FDv8Kyk2+A6o0Kk+vF5W6HF3oW/n3/owVMevHg6nEQkhP1xu4eMrM7CBr2JGCwu882s4eBDHcfRmEgmAOEgfvcPRvAzP5KEGQAHnb3hGd3QkpSH97GPxscOR5zBjQ6Liib/RG8cTEMGAEN25XO57gH/8C1m0Ona4Ok56wPgqGOlNTS+YyKYMEXwVAQQDg3GGKxPfx/ucPIP8LE/4BViu7TT2HucNiSCRc9E6xb62gY8BnM/RQ6/rzoe6TfHCSy028qLEvrGQwVbl0DNRuXrN4/zYRprwe/d72l8G9tf839JOjNHHNGYVlKVehxJ4z6U3AqbvOuRdeZ+hqsnQUn3wiVqwe9nzH/gE7XQGqtA6vHQUjo4bK7j3D3du7e2t0fiZY9GA0OeOB37t7B3U9093dj1h3s7m2iryGJrGe+cCSi+zAdria9GPxTHX8pXDsU+vwjeN04HDB4/aLgwqjSsOw7WPk99LwbkisHR4C522DJ6NJ5/4rAHb59HOq0gB53wdQh8Pnvi56KGrvs1w8HwaHbrXDdB8HZSW9cDOOegiadoc1ZhcvXagKnDISklKLvk5+LSK1dWNayR/Bzf85m+vZxqFIraNy/faLk68UKh2DeZ9Cuz+4HFem/gKr1gs+JlZcTHASl9YJ+zwZ/3xc9DTmbIOPVPX+We8IuCExYD+KIsXMjvHkFAHdnb2dnbhhePvSR+ohTuRqc+wgc3bGwbFsWfPqb4GitNHkEVk+DYy+Ey14uHFYAaNAWbhwGr10Ar5wdjEsXl1IVzn4ImnUpLNuxAT79LWzO3H35zSuD8ezO1wfTaT8LGp05w6B9NPE54T9B76U0VK4G5/wVmnQqLNu+HobfHf+7NAuGKk6+obAsEoHRf4OkynD67/d8pL4nS8YEZwr1/WfJj7T3ZvE3QYL/wqehywCIhGDS80HAPeevRes39p9BIOgyIPh8M7j6LXj3mqDncd7f93978jU+KWjol08Ihqh2bQ32+4alwXyzYD93iV6Lu25uMJzY616I5MGEf0PvQdCgTTB/yqvwwzvxA13tpsH2VqsHKybAjuyiw0v5qtSA7rfBN3+Dpd9Bq15BeX6u5bKXCpdt2gVanxX8vXUbGPQqYrnDF4Ng1xbo959gWK0UKUBgBUccO8zYWSlc9AhE4ls7q3AY4Kjjgwb3jYuDseH8o7bSdMqv4ZyHdz9qhGAI4IZhQVc8b+fu89fNhf9eCjd+EhyN5myGNy8LchdppwHFGp/U2sEwRf6RX3JlaH8+zB8RDGGMfzr45z66U5CfOFhrZ8F/L4EbP4XGJ0S/y0sgeyG07Ln78lvXwLA7g0Y3/RdBIzHi3sKjzF1b4dy/lbxRXfotvP1zCOUE39WAz6DGQZ708e0TUKtpMDRiFjTy4dygwU2qAmf9KVjuu6eiQyjXwgUxQ1Btz4H+7wR1yz8b6UAkJQf5i+UTgvH9t66ElZPhmNMBg21rYfhdQTDoektw6mxKdTj1NvAwfP9SELwueR4mvQBfPACNTwzyCEV4kJDetAJu+CQ4mEiuCm3Ojl+vbrfCjHeCIHj9x8F7jn8mqGurnxVd9vT7g9Otp74eBJaCj/RgyPX7F4L/jwMNonuhAFG1Dlz/IQD/ej2D1Zt2MuL6XmVcqSPAhiUw5PwgKFz9dpDQy14E17wHrc/Y9/qlrfEJwVFnPJtWwmvnB0Gi/7vBWPeaWfDzN6F9Ca+/PK5fcOQ49BfBEWbHq4NGo1LSwdd9w9LC77L/OzDiPli/AK55F1qfufvyoVx477rgSDipcrAtGa8GQ2J5O4OhmqTKcNaD+240lk8MgkPdNDjjD/DhwMLAX63egW3PsnHBEXTfxyC5SlBmBn0fD4LEd08E5SnV4OuH4MQrod+/dz/6bXP2nhvY/dGyB3zzV/jvZZA5Ga4YXHj2UygX3r8h+PvduibINXW/HapHA3+XAcGpszWPDup97IXBdRfxDlQWfAnvXgtvXh78zbU9e/cj/nyptYLh0SF94c1LgwC5eWX8PE2LU4Nhp/HPBAcE+Qcuox+BCc8GuZc+/0hIgDCP11U6AqWnp3tGRsZBvcdNQyaTvT2XYXecVkq1KufWLwwatu3rggbp6reDI7/D0cZlQV23rAJLgqteh+MuKvn6eTnweOsgF3H8ZbsPdR2s9YuCILZtLVRKCb7LdufuvT7v9g+GciA4guzzj+Co8rPfBsnOdn2Cse498iDZW7Nx0GuoeRQsHl0YMJp22X2VlNQgENVNKywLRRv9TdFrWzOnBL2038wMhvdiRSLwyW1BsAXocDFcPrh0v8vilk+EIX0AC4ZvOl5VdH5oV3Akv+grSE6Fu2cG3wXAltXwzElBYGvXB676b9Cj3JO5w+H9G4Pex2WvQMcr9163jcuD4dHNK4Me6cAx8Rv6JWPhjX7BAUONxsHQ+ILPg2HGC585qKElM5vq7ulx5ylAFLr+1e/ZtivER7fF6dZLfOvmBWPlp/2mcHz+cJW9GD65A065FY6/ZP/X//bxoMHo+1j8I8iDlTU/OL2y591w7Pn7Xj5vJ3z8a6jTEs7+S2HDEonAyD8Ejf++1G4GV7waJH7zLfwKvvy/+MN127OgekO4aQTUaR4MuQ29KWgYazcHLBixO/330Pm6+J8ZCQe9pNCuIAmbiO8yVigX/jcgyAecdHX8ZfJ2Bt/90R2Ds4xijftXsG8ueqawR7Q3cz6BaW8EPY0qNfe9/IYlwWf3fiA65BmHO3xwSzA8lq99X+jz6EHnHRQgSqj/S5MIRSL871cJGEMXKQ9Wz4DX+wXDTzcOh6/+HAzL9HkUTv11WddODsDeAoSuCosRjuhCOZG9atIpyNltXw//6RoEh3MeVnAop9QaxgjpOgiRfWuWDtf+Lzhd86wHgyExKZd0FlMM3e5bpIRadod7FybkzBk5fKgHESNPt/sWKTkFh3JPASKGehAiIoUUIGIEOQh9JSIioABRRFjPgxARKaAAEUO3+xYRKaQAEamF0nIAABUfSURBVEPPpBYRKaQAESPoQegrEREBBYgiwpEIKbpQTkQEUIAoQjkIEZFCChAxdBaTiEghBYgYobByECIi+dQaxggpByEiUkABIioScSKOchAiIlEKEFHh6IOTlIMQEQkoQESFwkGAUA5CRCSg1jAqFIkA6kGIiORTgIgKR6JDTEpSi4gAChAFQhHlIEREYiU0QJhZHzObb2aLzOyBOPMHmFmWmc2Ivm6JmfdPM5sVff08kfWEwh6EchAiIoGEPZPazJKA54BzgExgipkNc/c5xRZ9z93vKLbuBcDJQCegCjDGzD539y2Jqm9eWDkIEZFYiTxc7gYscvcl7p4LvAtcXMJ1OwDfunvI3bcDM4E+CaonENuDUIAQEYHEBoimwMqY6cxoWXGXm9lMMxtqZs2jZT8Afcysmpk1AM4Amhdf0cwGmlmGmWVkZWUdVGVDSlKLiBRR1gPuw4E0d+8IjAJeB3D3kcAIYALwDjARCBdf2d1fcvd0d09v2LDhQVWk4Cwm5SBERIDEBohVFD3qbxYtK+Du2e6+Kzr5CtAlZt4j7t7J3c8BDFiQwLrGXCinHoSICCQ2QEwB2ppZKzOrDFwNDItdwMyOjpnsB8yNlieZWf3o7x2BjsDIBNZVF8qJiBSTsLOY3D1kZncAXwJJwGB3n21mDwMZ7j4MuMvM+gEhYAMwILp6CvCdmQFsAa5z91Ci6gqFOYgk5SBERIAEBggAdx9BkEuILXsw5vdBwKA46+UQnMl0yOTnIFKUgxARAco+SX3YUA5CRKQoBYioghyEhphERAAFiAIhXSgnIlKEAkRUOKwchIhILLWGUepBiIgUpQARpedBiIgUpQARlZ+kVg9CRCRQogBhZpeaWe2Y6TpmdkniqnXo5Z/mqiupRUQCJe1B/NndN+dPuPsm4M+JqVLZKBxiUqdKRARKHiDiLZfQq7APNT1yVESkqJIGiAwze8rMWkdfTwFTE1mxQy2sHISISBElDRB3ArnAewRPhssBbk9UpcpCnnIQIiJFlGiYKPrYzwcSXJcypUeOiogUVdKzmEaZWZ2Y6bpm9mXiqnXo5ecgUpSkFhEBSj7E1CB65hIA7r4RaJSYKpUN5SBERIoqaYCImFmL/AkzSwM8ERUqKwW32jAFCBERKPmpqn8AxpnZWILnQ/cCBiasVmUgFHYqGVRSD0JEBCh5kvoLM0snCArTgY+BnYms2KEWirgukhMRiVGiAGFmtwB3A82AGcCpwETgzMRV7dAKRyI6xVVEJEZJD5nvBroCy939DKAzsGnvqxxZQhFXglpEJEZJA0SOu+cAmFkVd58HtE9ctQ69UNjVgxARiVHSJHVm9DqIj4FRZrYRWJ64ah16QQ9COQgRkXwlTVJfGv31L2Y2GqgNfJGwWpWBcCRCih4WJCJSYL/vyOruYxNRkbKmHISISFEaU4kKR5SDEBGJpQARFQqrByEiEksBIioUiZCsJLWISIGEtohm1sfM5pvZIjPb7XbhZjbAzLLMbEb0dUvMvMfMbLaZzTWzZ80Se5OkcMRJVpJaRKRAwh4bamZJwHPAOUAmMMXMhrn7nGKLvufudxRbtwfQE+gYLRoHnA6MSVR9Q8pBiIgUkcgeRDdgkbsvcfdcgifRXVzCdR1IBSoDVYAUYG1CahkV1llMIiJFJDJANAVWxkxnRsuKu9zMZprZUDNrDuDuE4HRwE/R15fuPrf4imY20MwyzCwjKyvroCqbF1YOQkQkVlm3iMOBNHfvCIwCXgcwszbAcQQ3B2wKnGlmvYqv7O4vuXu6u6c3bNjwoCqiHoSISFGJDBCrgOYx082iZQXcPdvdd0UnXwG6RH+/FJjk7tvcfRvwOdA9gXWN3u5bAUJEJF8iA8QUoK2ZtTKzysDVwLDYBczs6JjJfkD+MNIK4HQzSzazFIIE9W5DTKVJF8qJiBSVsLOY3D1kZncAXwJJwGB3n21mDwMZ7j4MuMvM+gEhYAMwILr6UIJnTfxIkLD+wt2HJ6qukH+hXFmPuImIHD4SFiAA3H0EMKJY2YMxvw8CBsVZLwzcmsi6FRfSA4NERIrQIXOUchAiIkUpQEQpByEiUpQCRJRyECIiRalFjFIOQkSkKAWIqHDESVIOQkSkgAJEVCjipKgHISJSQAEiKqwchIhIEWoRo3Saq4hIUQoQUaFIRDfrExGJoQARpQcGiYgUpQABRCKOO3oehIhIDLWIBL0HQDkIEZEYChAE10AAykGIiMRQgADyIhEA5SBERGIoQBBcAwEKECIisRQgKMxBJCXp6xARyacWkcIchHoQIiKFFCAILpIDJalFRGIpQBA8CwLUgxARiaUAQex1EPo6RETyqUVEOQgRkXgUIFAOQkQkHgUIlIMQEYlHAYKY6yAUIERECihAUJiDSFGSWkSkgFpElIMQEYlHAQKdxSQiEk9CA4SZ9TGz+Wa2yMweiDN/gJllmdmM6OuWaPkZMWUzzCzHzC5JVD3zk9TqQYiIFEpO1BubWRLwHHAOkAlMMbNh7j6n2KLvufsdsQXuPhroFH2fesAiYGSi6lpwoZyeKCciUiCRLWI3YJG7L3H3XOBd4OIDeJ8rgM/dfUep1i5GOP95EHqinIhIgUQGiKbAypjpzGhZcZeb2UwzG2pmzePMvxp4J94HmNlAM8sws4ysrKwDrmhIOQgRkd2U9ZjKcCDN3TsCo4DXY2ea2dHAicCX8VZ295fcPd3d0xs2bHjAldAjR0VEdpfIALEKiO0RNIuWFXD3bHffFZ18BehS7D2uAj5y97yE1RLICysHISJSXCJbxClAWzNrZWaVCYaKhsUuEO0h5OsHzC32Hv3Zw/BSaVIOQkRkdwk7i8ndQ2Z2B8HwUBIw2N1nm9nDQIa7DwPuMrN+QAjYAAzIX9/M0gh6IGMTVcd8ykGIiOwuYQECwN1HACOKlT0Y8/sgYNAe1l1G/KR2qVMOQkRkdxp0RzkIEZF41CJSmINIUg5CRKSAAgTKQYiIxKMAAYT1wCARkd0oQKAHBomIxKMAQfA8iKRKhpkChIhIPgUIgh6Eeg8iIkUpQBDkIFIUIEREilCAQD0IEZF4FCAIrqROTtJXISISS60ihUlqEREppABB8ExqXQMhIlJUQm/Wd6QIhpgUIEQqory8PDIzM8nJySnrqiRUamoqzZo1IyUlpcTrKEAQJKl1oz6RiikzM5OaNWuSlpZWbq+Fcneys7PJzMykVatWJV5PrSJBD0I5CJGKKScnh/r165fb4ABgZtSvX3+/e0kKEEBeOKIchEgFVp6DQ74D2UYFCJSDEBGJRwGC/Avl9FWIyKG3adMmnn/++f1e7/zzz2fTpk0JqFEhtYpEexAaYhKRMrCnABEKhfa63ogRI6hTp06iqgXoLCYgyEEoSS0iDw2fzZzVW0r1PTs0qcWfLzp+j/MfeOABFi9eTKdOnUhJSSE1NZW6desyb948FixYwCWXXMLKlSvJycnh7rvvZuDAgQCkpaWRkZHBtm3b6Nu3L6eddhoTJkygadOmfPLJJ1StWvWg664eBOpBiEjZefTRR2ndujUzZszg8ccfZ9q0aTzzzDMsWLAAgMGDBzN16lQyMjJ49tlnyc7O3u09Fi5cyO23387s2bOpU6cOH3zwQanUTT0IghxENd2LSaTC29uR/qHSrVu3ItcqPPvss3z00UcArFy5koULF1K/fv0i67Rq1YpOnToB0KVLF5YtW1YqdVGAQD0IETl8VK9eveD3MWPG8NVXXzFx4kSqVatG7969417LUKVKlYLfk5KS2LlzZ6nURYfN6HbfIlJ2atasydatW+PO27x5M3Xr1qVatWrMmzePSZMmHdK6qQcBhHShnIiUkfr169OzZ09OOOEEqlatylFHHVUwr0+fPrz44oscd9xxtG/fnlNPPfWQ1k0BAt1qQ0TK1ttvvx23vEqVKnz++edx5+XnGRo0aMCsWbMKyu+9995Sq5eGmAiGmFKUpBYRKSKhraKZ9TGz+Wa2yMweiDN/gJllmdmM6OuWmHktzGykmc01szlmlpaoeqoHISKyu4QNMZlZEvAccA6QCUwxs2HuPqfYou+5+x1x3uIN4BF3H2VmNYBIouoaiigHISJSXCJ7EN2ARe6+xN1zgXeBi0uyopl1AJLdfRSAu29z9x2JqmgorB6EiEhxiQwQTYGVMdOZ0bLiLjezmWY21MyaR8vaAZvM7EMzm25mj0d7JEWY2UAzyzCzjKysrAOuqHIQIiK7K+tWcTiQ5u4dgVHA69HyZKAXcC/QFTgGGFB8ZXd/yd3T3T29YcOGB1wJ5SBERHaXyACxCmgeM90sWlbA3bPdfVd08hWgS/T3TGBGdHgqBHwMnJyoiioHISJl5UBv9w3w9NNPs2NHwkbfExogpgBtzayVmVUGrgaGxS5gZkfHTPYD5sasW8fM8rsFZwLFk9ulRj0IESkrh3OASNhZTO4eMrM7gC+BJGCwu882s4eBDHcfBtxlZv2AELCB6DCSu4fN7F7gawuekzcVeDlB9SQvrHsxiQjw+QOw5sfSfc/GJ0LfR/c4O/Z23+eccw6NGjXi/fffZ9euXVx66aU89NBDbN++nauuuorMzEzC4TB/+tOfWLt2LatXr+aMM86gQYMGjB49unTrTYKvpHb3EcCIYmUPxvw+CBi0h3VHAR0TWT+AiAc/k5WkFpEy8OijjzJr1ixmzJjByJEjGTp0KJMnT8bd6devH99++y1ZWVk0adKEzz77DAju0VS7dm2eeuopRo8eTYMGDRJStwp/q41QJLi8QkNMIrK3I/1DYeTIkYwcOZLOnTsDsG3bNhYuXEivXr245557+P3vf8+FF15Ir169Dkl9KnyACEe7EBpiEpGy5u4MGjSIW2+9dbd506ZNY8SIEfzxj3/krLPO4sEHH4zzDqWrwo+r5IWDAKEehIiUhdjbfZ933nkMHjyYbdu2AbBq1SrWrVvH6tWrqVatGtdddx333Xcf06ZN223dRFAPQj0IESlDsbf77tu3L9dccw3du3cHoEaNGrz55pssWrSI++67j0qVKpGSksILL7wAwMCBA+nTpw9NmjRJSJLa3L3U37QspKene0ZGxn6vt3lnHv/34Y9c1bU5p7c78IvtROTINHfuXI477riyrsYhEW9bzWyqu6fHW77C9yBqV03huWsTdg2eiMgRq8LnIEREJD4FCBGp8MrLUPveHMg2KkCISIWWmppKdnZ2uQ4S7k52djapqan7tV6Fz0GISMXWrFkzMjMzOZhHBhwJUlNTadas2X6towAhIhVaSkoKrVq1KutqHJY0xCQiInEpQIiISFwKECIiEle5uZLazLKA5QfxFg2A9aVUnSNFRdxmqJjbXRG3GSrmdu/vNrd097i3kSg3AeJgmVnGni43L68q4jZDxdzuirjNUDG3uzS3WUNMIiISlwKEiIjEpQBR6KWyrkAZqIjbDBVzuyviNkPF3O5S22blIEREJC71IEREJC4FCBERiavCBwgz62Nm881skZk9UNb1SRQza25mo81sjpnNNrO7o+X1zGyUmS2M/qxb1nUtbWaWZGbTzezT6HQrM/s+us/fM7PKZV3H0mZmdcxsqJnNM7O5Zta9vO9rM/tt9G97lpm9Y2ap5XFfm9lgM1tnZrNiyuLuWws8G93+mWa2X09Hq9ABwsySgOeAvkAHoL+ZdSjbWiVMCLjH3TsApwK3R7f1AeBrd28LfB2dLm/uBubGTP8T+Je7twE2AjeXSa0S6xngC3c/FjiJYPvL7b42s6bAXUC6u58AJAFXUz739WtAn2Jle9q3fYG20ddA4IX9+aAKHSCAbsAid1/i7rnAu8DFZVynhHD3n9x9WvT3rQQNRlOC7X09utjrwCVlU8PEMLNmwAXAK9FpA84EhkYXKY/bXBv4GfAqgLvnuvsmyvm+Jrg7dVUzSwaqAT9RDve1u38LbChWvKd9ezHwhgcmAXXM7OiSflZFDxBNgZUx05nRsnLNzNKAzsD3wFHu/lN01hrgqDKqVqI8DdwPRKLT9YFN7h6KTpfHfd4KyAKGRIfWXjGz6pTjfe3uq4AngBUEgWEzMJXyv6/z7WnfHlQbV9EDRIVjZjWAD4DfuPuW2HkenPNcbs57NrMLgXXuPrWs63KIJQMnAy+4e2dgO8WGk8rhvq5LcLTcCmgCVGf3YZgKoTT3bUUPEKuA5jHTzaJl5ZKZpRAEh7fc/cNo8dr8Lmf057qyql8C9AT6mdkyguHDMwnG5utEhyGgfO7zTCDT3b+PTg8lCBjleV+fDSx19yx3zwM+JNj/5X1f59vTvj2oNq6iB4gpQNvomQ6VCZJaw8q4TgkRHXt/FZjr7k/FzBoG3Bj9/Ubgk0Ndt0Rx90Hu3szd0wj27Tfufi0wGrgiuli52mYAd18DrDSz9tGis4A5lON9TTC0dKqZVYv+redvc7ne1zH2tG+HATdEz2Y6FdgcMxS1TxX+SmozO59gnDoJGOzuj5RxlRLCzE4DvgN+pHA8/v8I8hDvAy0Ibpd+lbsXT4Ad8cysN3Cvu19oZscQ9CjqAdOB69x9V1nWr7SZWSeCxHxlYAlwE8EBYbnd12b2EPBzgjP2pgO3EIy3l6t9bWbvAL0Jbuu9Fvgz8DFx9m00WP6HYLhtB3CTu2eU+LMqeoAQEZH4KvoQk4iI7IEChIiIxKUAISIicSlAiIhIXAoQIiISlwKEyH4ws7CZzYh5ldoN78wsLfYOnSJlLXnfi4hIjJ3u3qmsKyFyKKgHIVIKzGyZmT1mZj+a2WQzaxMtTzOzb6L34v/azFpEy48ys4/M7Ifoq0f0rZLM7OXocw1GmlnVMtsoqfAUIET2T9ViQ0w/j5m32d1PJLhy9elo2b+B1929I/AW8Gy0/FlgrLufRHCfpNnR8rbAc+5+PLAJuDzB2yOyR7qSWmQ/mNk2d68Rp3wZcKa7L4neFHGNu9c3s/XA0e6eFy3/yd0bmFkW0Cz2tg/R27CPij70BTP7PZDi7n9L/JaJ7E49CJHS43v4fX/E3icojPKEUoYUIERKz89jfk6M/j6B4E6yANcS3DARgsdC/hoKnpld+1BVUqSkdHQisn+qmtmMmOkv3D3/VNe6ZjaToBfQP1p2J8GT3e4jeMrbTdHyu4GXzOxmgp7CrwmehCZy2FAOQqQURHMQ6e6+vqzrIlJaNMQkIiJxqQchIiJxqQchIiJxKUCIiEhcChAiIhKXAoSIiMSlACEiInH9f6VcY8wg5TwvAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W2F8HWE_da0M"
      },
      "source": [
        "Evaluate the model on test set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UfViCGoHIl2Z",
        "outputId": "82841e28-0ea1-46b3-af48-dd31aa5050f5"
      },
      "source": [
        "y_pred = (clf_nn.predict(X_test)>0.5).astype(\"int32\")\n",
        "y_scores = clf_nn.predict(X_test)\n",
        "print('nn Eval')\n",
        "eval_nn = clf_eval(y_test,y_pred,y_scores)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "nn Eval\n",
            "acc: 0.6960352422907489\n",
            "f1 score(micro): 0.7779005524861878\n",
            "precision score(micro): 0.88\n",
            "recall score(micro): 0.697029702970297\n",
            "AUC score(micro): 0.8465115125869849\n",
            "hamming loss: 0.016099319183019625\n",
            "label ranking loss: 0.032936857562408224\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Op_mSwYXaECK"
      },
      "source": [
        "On training set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CAGJhegYZ4xC",
        "outputId": "0e47507c-a011-41bc-aa8b-2bc5578c6fa9"
      },
      "source": [
        "y_pred = (clf_nn.predict(X_train)>0.5).astype(\"int32\")\n",
        "y_scores = clf_nn.predict(X_train)\n",
        "print('nn Eval')\n",
        "eval_nn = clf_eval(y_train,y_pred,y_scores)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "nn Eval\n",
            "acc: 0.7514166981488477\n",
            "f1 score(micro): 0.8188268684957427\n",
            "precision score(micro): 0.9448689956331878\n",
            "recall score(micro): 0.7224540901502504\n",
            "AUC score(micro): 0.8603226430549955\n",
            "hamming loss: 0.013153827660816705\n",
            "label ranking loss: 0.035681178137384764\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HcpS3X9fB_NM"
      },
      "source": [
        "Comparison with random forest classifier."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 232
        },
        "id": "1IccTq1sCCwR",
        "outputId": "c154c64d-c980-4eab-c9b6-2d710ebfd70f"
      },
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "clf_RandomForest = RandomForestClassifier(n_estimators=100)\n",
        "clf_RandomForest.fit(X_train, y_train)\n",
        "y_pred, y_scores = clf_pre(clf_RandomForest,X_test)\n",
        "print('RandomForest Eval')\n",
        "eval_RF = clf_eval(y_test,y_pred,y_scores)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-49-840aec15f541>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mclf_RandomForest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRandomForestClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_estimators\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mclf_RandomForest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_scores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclf_pre\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclf_RandomForest\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'RandomForest Eval'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0meval_RF\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclf_eval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_scores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'clf_pre' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "li9dGLBEagvP"
      },
      "source": [
        "Random Forest on training set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kqgffwJWaj5n"
      },
      "source": [
        "y_pred, y_scores = clf_pre(clf_RandomForest,X_train)\n",
        "print('RandomForest Eval')\n",
        "eval_RF = clf_eval(y_train,y_pred,y_scores)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RTkPTnFt4hvF"
      },
      "source": [
        "For LDL Task"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t-IJg4GM4nbC"
      },
      "source": [
        "y_pred = clf_nn.predict(X_test)\n",
        "for i in range(len(y_pred)):\n",
        "    y_pred[i] = y_pred[i]/sum(y_pred[i])\n",
        "loss_test_mean = np.mean(scipy.stats.entropy(y_test,y_pred))\n",
        "print(\"Testing set\")\n",
        "print(\"kl_dist mean:\",loss_test_mean)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UcXvONf6N_ni"
      },
      "source": [
        "## AutoEncoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DmhqeKKbYsWQ"
      },
      "source": [
        "Fully connected network autoencoder\n",
        "\n",
        "creat & complie"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1vYCSz7nOiRQ"
      },
      "source": [
        "# use a fully connected network\n",
        "feature_dim = X_train.shape[1]\n",
        "net_input = Input(shape=(feature_dim,))\n",
        "## encoder\n",
        "encoder = Dropout(0.2)(net_input)\n",
        "encoder = Dense(500,activation=\"relu\")(encoder)\n",
        "encoder = Dropout(0.5)(encoder)\n",
        "encoder = Dense(32,activation=\"relu\")(encoder)\n",
        "## decoder\n",
        "decoder = Dense(500,activation=\"relu\")(encoder)\n",
        "decoder = Dropout(0.5)(decoder)\n",
        "decoder = Dense(feature_dim,activation=\"relu\")(decoder)\n",
        "## combined to autoencoder\n",
        "\n",
        "autoencoder = Model(inputs=net_input, outputs=decoder)\n",
        "autoencoder.summary()\n",
        "## compile\n",
        "autoencoder.compile(optimizer=\"adam\",loss=\"binary_crossentropy\")\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hhdk3DDhfUAN"
      },
      "source": [
        "Unfinished stack AE"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MZGALZlB_r12"
      },
      "source": [
        "# use a fully connected network\n",
        "input_dim = X_train.shape[1]\n",
        "input_features = Input(shape=(input_dim,))\n",
        "## encoder\n",
        "encoder1 = Dense(128,activation=\"relu\",name=\"enco1\")(input_features)\n",
        "decoder1 = Dense(input_dim,activation=\"relu\",name=\"deco1\")(encoder1)\n",
        "autoencoder1 = Model(input=input_features, output=decoder1)\n",
        "autoencoder1.compile(optimizer=\"adam\",loss=\"binary_crossentropy\")\n",
        "\n",
        "input2 = Input(shape=(128,))\n",
        "encoder2 = Dense(64,activation=\"relu\",name=\"enco2\")(input2)\n",
        "decoder2 = Dense(128,activation=\"relu\",name=\"deco2\")(encoder2)\n",
        "autoencoder2 = Model(input=input2, output=decoder2)\n",
        "autoencoder2.compile(optimizer=\"adam\",loss=\"binary_crossentropy\")\n",
        "\n",
        "input3 = Input(shape=(64,))\n",
        "encoder3 = Dense(32,activation=\"relu\",name=\"enco3\")(input3)\n",
        "decoder3 = Dense(64,activation=\"relu\",name=\"deco3\")(encoder3)\n",
        "autoencoder3 = Model(input=input3, output=decoder3)\n",
        "autoencoder3.compile(optimizer=\"adam\",loss=\"binary_crossentropy\")\n",
        "\n",
        "input4 = Input(shape=(32,))\n",
        "encoder4 = Dense(16,activation=\"relu\",name=\"enco4\")(input4)\n",
        "decoder4 = Dense(32,activation=\"relu\",name=\"deco4\")(encoder4)\n",
        "autoencoder4 = Model(input=input4, output=encoder4)\n",
        "autoencoder4.compile(optimizer=\"adam\",loss=\"binary_crossentropy\")\n",
        "\n",
        "autoencoder1_fit = autoencoder1.fit(X_train,X_train,batch_size=256,epochs=200,\n",
        "                        validation_data=(X_test,X_test),verbose=0)\n",
        "\n",
        "autoencoder2_fit = autoencoder2.fit(X_train,X_train,batch_size=256,epochs=200,\n",
        "                        validation_data=(X_test,X_test),verbose=0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ByOJ0qK1Y-wl"
      },
      "source": [
        "Fit the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LQj60XGLPBr9"
      },
      "source": [
        "autoencoder_fit = autoencoder.fit(X_train,X_train,batch_size=256,epochs=100,\n",
        "                        validation_data=(X_test,X_test),verbose=0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q_8RCc_uZOrj"
      },
      "source": [
        "Evaluate the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ra0MBfuPmmcD"
      },
      "source": [
        "# plot history\n",
        "plt.figure(figsize=(9,6))\n",
        "plt.plot(autoencoder_fit.history['loss'], label='train')\n",
        "plt.plot(autoencoder_fit.history['val_loss'], label='test')\n",
        "plt.ylabel('binary_crossentropy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.grid()\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lo3-AhWfZqXI"
      },
      "source": [
        "Get the encoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0eNqvJmoQ9_u"
      },
      "source": [
        "Encoder = Model(net_input, encoder)\n",
        "X_train_encoded = Encoder.predict(X_train)\n",
        "X_test_encoded = Encoder.predict(X_test)\n",
        "encoded_feature_train = pd.DataFrame(X_train_encoded)\n",
        "encoded_feature_train.describe()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pxDVGiQ9uqT3"
      },
      "source": [
        "encoded_feature_test = pd.DataFrame(X_test_encoded)\n",
        "encoded_feature_test.describe()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iGA6eB6kXtl4"
      },
      "source": [
        "### Classification task"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HXdAeQgXZ_6Y"
      },
      "source": [
        "Use encoded features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DOqDpYctUAqV"
      },
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "clf_RandomForest = RandomForestClassifier(n_estimators=100)\n",
        "clf_RandomForest.fit(X_train_encoded, y_train)\n",
        "y_pred, y_scores = clf_pre(clf_RandomForest,X_test_encoded)\n",
        "print('RandomForest Eval')\n",
        "eval_RF = clf_eval(y_test,y_pred,y_scores)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z2EFL3IKYe9K"
      },
      "source": [
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "clf_KNN = KNeighborsClassifier()\n",
        "clf_KNN.fit(X_train_encoded, y_train)\n",
        "y_pred, y_scores = clf_pre(clf_KNN,X_test_encoded)\n",
        "print('KNN Eval')\n",
        "eval_RF = clf_eval(y_test,y_pred,y_scores)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "flKhbYAaZ3Kc"
      },
      "source": [
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "def linear_model(feature_dim,label_dim):\n",
        "    model = Sequential()\n",
        "    model.add(Dense(label_dim,input_dim=feature_dim,activation='sigmoid'))\n",
        "    model.compile(optimizer='adam',loss='binary_crossentropy', metrics=['accuracy'])\n",
        "    return model\n",
        "    \n",
        "def train_linear(X_train,y_train,X_test,y_test):\n",
        "    feature_dim = X_train.shape[1]\n",
        "    label_dim = y_train.shape[1]\n",
        "    model = linear_model(feature_dim,label_dim)\n",
        "    model.fit(X_train,y_train,batch_size=128, epochs=100, validation_data=(X_test,y_test),verbose=0)\n",
        "    return model\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ta0yfcFliq-2"
      },
      "source": [
        "clf_linear = train_linear(X_train_encoded,y_train,X_test_encoded,y_test)\n",
        "y_pred = (clf_linear.predict(X_test_encoded)>0.5).astype(\"int32\")\n",
        "y_scores = clf_linear.predict(X_test_encoded)\n",
        "print('linear Eval')\n",
        "eval_linear = clf_eval(y_test,y_pred,y_scores)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uoSKau8BaJJC"
      },
      "source": [
        "Use raw features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mImgehODVD1L"
      },
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "clf_RandomForest = RandomForestClassifier(n_estimators=100)\n",
        "clf_RandomForest.fit(X_train, y_train)\n",
        "y_pred, y_scores = clf_pre(clf_RandomForest,X_test)\n",
        "print('RandomForest Eval')\n",
        "eval_RF = clf_eval(y_test,y_pred,y_scores)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jpPK9IqCYbye"
      },
      "source": [
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "clf_KNN = KNeighborsClassifier()\n",
        "clf_KNN.fit(X_train, y_train)\n",
        "y_pred, y_scores = clf_pre(clf_KNN,X_test)\n",
        "print('KNN Eval')\n",
        "eval_RF = clf_eval(y_test,y_pred,y_scores)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wgPZEVu8haKk"
      },
      "source": [
        "clf_linear = train_linear(X_train,y_train,X_test,y_test)\n",
        "y_pred = (clf_linear.predict(X_test)>0.5).astype(\"int32\")\n",
        "y_scores = clf_linear.predict(X_test)\n",
        "print('linear Eval')\n",
        "eval_linear = clf_eval(y_test,y_pred,y_scores)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sMQy1zWwXwYb"
      },
      "source": [
        "### Regression Task"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yPuwyvaHaM3G"
      },
      "source": [
        "Use encoded features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bHvwrbnDajTB"
      },
      "source": [
        "from sklearn.ensemble import RandomForestRegressor\n",
        "\n",
        "regressor = RandomForestRegressor(n_estimators=100)\n",
        "regressor.fit(X_train_encoded, y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v5hw0ZnoanAh"
      },
      "source": [
        "y_pred_train = regressor.predict(X_train_encoded)\n",
        "for i in range(len(y_pred_train)):\n",
        "  y_pred_train[i] = y_pred_train[i]/sum(y_pred_train[i])\n",
        "loss_train_mean = np.mean(scipy.stats.entropy(y_train,y_pred_train))\n",
        "print(\"train set\")\n",
        "print(\"kl_dist mean:\",loss_train_mean)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hwQuSoGmarg4"
      },
      "source": [
        "y_pred = regressor.predict(X_test_encoded)\n",
        "for i in range(len(y_pred)):\n",
        "  y_pred[i] = y_pred[i]/sum(y_pred[i])\n",
        "loss_test_mean = np.mean(scipy.stats.entropy(y_test,y_pred))\n",
        "print(\"test set\")\n",
        "print(\"kl_dist mean:\",loss_test_mean)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RKD_zd1fYQL3"
      },
      "source": [
        "Use raw features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r2nxS5KNXyMU"
      },
      "source": [
        "from sklearn.ensemble import RandomForestRegressor\n",
        "\n",
        "regressor = RandomForestRegressor(n_estimators=100)\n",
        "regressor.fit(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TSgATcuZYDrN"
      },
      "source": [
        "y_pred_train = regressor.predict(X_train)\n",
        "for i in range(len(y_pred_train)):\n",
        "  y_pred_train[i] = y_pred_train[i]/sum(y_pred_train[i])\n",
        "loss_train_mean = np.mean(scipy.stats.entropy(y_train,y_pred_train))\n",
        "print(\"train set\")\n",
        "print(\"kl_dist mean:\",loss_train_mean)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FSFA4M4AYBUg"
      },
      "source": [
        "y_pred = regressor.predict(X_test)\n",
        "for i in range(len(y_pred)):\n",
        "  y_pred[i] = y_pred[i]/sum(y_pred[i])\n",
        "loss_test_mean = np.mean(scipy.stats.entropy(y_test,y_pred))\n",
        "print(\"test set\")\n",
        "print(\"kl_dist mean:\",loss_test_mean)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uSD_adunU1IY"
      },
      "source": [
        "# Auto-Sklearn"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XVDqqOTpBQbY"
      },
      "source": [
        "## Classification model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WT9vB_vgIF6m"
      },
      "source": [
        "Install required libraries."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0L3NNJqOU1i0"
      },
      "source": [
        "#compiler\n",
        "!apt-get install swig\n",
        "\n",
        "#dependencies\n",
        "!curl https://raw.githubusercontent.com/automl/auto-sklearn/master/requirements.txt | xargs -n 1 -L 1 pip3 install\n",
        "\n",
        "#auto-sklearn\n",
        "!pip3 install auto-sklearn"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_dDxM7pNVoB8"
      },
      "source": [
        "import autosklearn.classification\n",
        "import sklearn.model_selection\n",
        "import sklearn.metrics\n",
        "import sklearn.datasets"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4fxMQuB36I7p"
      },
      "source": [
        "Load a multi-label dataset, if not done before."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "4lO1pKKx6Mtc",
        "outputId": "91f99917-abed-43a8-dd1e-40a3bbb0e97d"
      },
      "source": [
        "data = spio.loadmat(data_path['yeast'])\n",
        "X = data['features']\n",
        "y = data['labels']\n",
        "X_train,X_test,y_train,y_test = model_selection.train_test_split(X,y,test_size=0.3)\n",
        "print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1691, 103) (726, 103) (1691, 14) (726, 14)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BQX2W5mA6Swu"
      },
      "source": [
        "Set the upper limit of automl running time. The example is for demonstration only, so set for a short time."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 322
        },
        "id": "e8tUS-nEVL52",
        "outputId": "babddb48-e0b3-4b6c-fa98-dfc79d84919b"
      },
      "source": [
        "automl = autosklearn.classification.AutoSklearnClassifier(time_left_for_this_task=500,per_run_time_limit=50)\n",
        "automl.fit(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1515: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true nor predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  average, \"true nor predicted\", 'F-score is', len(true_sum)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "AutoSklearnClassifier(delete_output_folder_after_terminate=True,\n",
              "                      delete_tmp_folder_after_terminate=True,\n",
              "                      disable_evaluator_output=False,\n",
              "                      ensemble_memory_limit=1024, ensemble_nbest=50,\n",
              "                      ensemble_size=50, exclude_estimators=None,\n",
              "                      exclude_preprocessors=None, get_smac_object_callback=None,\n",
              "                      include_estimators=None, include_preprocessors=None,\n",
              "                      initial_configurations_via_metalearning=25,\n",
              "                      logging_config=None, max_models_on_disc=50,\n",
              "                      metadata_directory=None, ml_memory_limit=3072,\n",
              "                      n_jobs=None, output_folder=None, per_run_time_limit=50,\n",
              "                      resampling_strategy='holdout',\n",
              "                      resampling_strategy_arguments=None, seed=1,\n",
              "                      shared_mode=False, smac_scenario_args=None,\n",
              "                      time_left_for_this_task=500, tmp_folder=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vy4I7G4S8aY6"
      },
      "source": [
        "The return value format of predict_proba in auto-sklearn is not the same as the sklearn classifier. It is a score, and no need to further process."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 171
        },
        "id": "IHZEielLhSVD",
        "outputId": "50a6a717-68a1-48ef-dd6c-7f049698b6ce"
      },
      "source": [
        "y_pred = automl.predict(X_test)\n",
        "y_scores = automl.predict_proba(X_test)\n",
        "print('automl')\n",
        "eval_automl = clf_eval(y_test,y_pred,y_scores)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "automl\n",
            "f1 score(micro): 0.0\n",
            "precision score(micro): 0.0\n",
            "recall score(micro): 0.0\n",
            "hamming loss: 0.061615384615384614\n",
            "label ranking loss: 0.9589745324520381\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tqbDVpAi9jJL"
      },
      "source": [
        "Comparison with random forest classifier."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        },
        "id": "qO6-96DE9nJL",
        "outputId": "e8b1600d-5bb3-4f22-8083-f3ab3356eac4"
      },
      "source": [
        "clf_RandomForest.fit(X_train, y_train)\n",
        "y_pred, y_scores = clf_pre(clf_RandomForest,X_test)\n",
        "print('RandomForest Eval')\n",
        "eval_RF = clf_eval(y_test,y_pred,y_scores)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "RandomForest Eval\n",
            "f1 score(micro): 0.28365878725590954\n",
            "precision score(micro): 0.8023255813953488\n",
            "recall score(micro): 0.17228464419475656\n",
            "hamming loss: 0.05361538461538461\n",
            "label ranking loss: 0.12859110802084928\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p_5R9bbd4eQy"
      },
      "source": [
        "Note that auto-sklearn can customize proxy indicators:\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "def accuracy(solution, prediction):\n",
        "    # custom function defining accuracy\n",
        "    return np.mean(solution == prediction)\n",
        "\n",
        "accuracy_scorer = autosklearn.metrics.make_scorer(\n",
        "    name=\"accu\",\n",
        "    score_func=accuracy,\n",
        "    optimum=1,\n",
        "    greater_is_better=True,\n",
        "    needs_proba=False,\n",
        "    needs_threshold=False,\n",
        ")\n",
        "```\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ATa9pqdBcp53"
      },
      "source": [
        "# Auto-Keras"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cx8YP_eFcou6"
      },
      "source": [
        "!pip3 install autokeras"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "GV57tfIG4Ljy",
        "outputId": "861b8874-bc59-4020-b597-8fa20a1635c6"
      },
      "source": [
        "from autokeras import StructuredDataClassifier\n",
        "from keras.utils.np_utils import to_categorical\n",
        "\n",
        "data = spio.loadmat(data_path['image'])\n",
        "X = data['features']\n",
        "y = data['labels']\n",
        "X_train,X_test,y_train,y_test = model_selection.train_test_split(X,y,test_size=0.3)\n",
        "print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1400, 294) (600, 294) (1400, 5) (600, 5)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OMGUqtrOAvKi"
      },
      "source": [
        "y_train = to_categorical(y_train,y_train.shape[1])\n",
        "y_test = to_categorical(y_test,y_test.shape[1])\n",
        "print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)\n",
        "\n",
        "clf = StructuredDataClassifier(num_classes=5,multi_label=True)\n",
        "clf.fit(X_train, y_train, epochs=5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XlMpnIvH47Kn"
      },
      "source": [
        "clf.final_fit(X_train, y_train, X_test, y_test, retrain=true)\n",
        "y = clf.evaluate(X_test, y_test, batch_size=128)\n",
        "\n",
        "print(y)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YYFDULfKGN-K"
      },
      "source": [
        "# More LDL algorithm"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NCTSHXx3GbfO"
      },
      "source": [
        "## BC-LDL\n",
        "\n",
        "Wang K, Geng X. Binary Coding based Label Distribution Learning[C]//IJCAI. 2018: 2783-2789."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uNFr703xTpA7"
      },
      "source": [
        "!pip install pynndescent"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MhMCEkF-KjXk"
      },
      "source": [
        "from pynndescent import NNDescent\n",
        "from scipy import sparse\n",
        "train_num = X_train.shape[0]\n",
        "feature_num = X_train.shape[1]\n",
        "label_num = y_train.shape[1]\n",
        "bias = np.ones(train_num)\n",
        "alpha = 2\n",
        "X_train_b = np.column_stack((X_train, bias))\n",
        "y_train_mean = np.repeat(np.mean(y_train, axis=0).reshape(1, label_num), train_num, axis=0)\n",
        "temp1 = np.repeat(np.sqrt(np.sum((y_train-y_train_mean)**2, axis=1).reshape(-1, 1)), label_num, axis=1)\n",
        "d = (y_train-y_train_mean)/temp1\n",
        "g0 = np.dot(np.dot(X_train_b.T, d), np.dot(X_train_b.T, d).T)\n",
        "g1 = alpha*g0\n",
        "gt = g1\n",
        "w_b = np.zeros((feature_num+1, alpha), dtype='float64')\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ybw7ny7oUtPU"
      },
      "source": [
        "for t in range(alpha):\n",
        "    eigenvalue, eigenvector = np.linalg.eig(gt)\n",
        "    eigenvalue = np.real(eigenvalue)\n",
        "    eigenvector = np.real(eigenvector)\n",
        "    eigenvalue_index = np.argmax(eigenvalue)\n",
        "    w_b[:, t] = eigenvector[:, eigenvalue_index]\n",
        "    temp2 = np.dot(X_train_b.T, np.sign(np.dot(X_train_b, w_b[:, t].reshape(-1, 1))))\n",
        "    gt = gt - np.dot(temp2, temp2.T)\n",
        "w = w_b[0:feature_num, :]\n",
        "e = w_b[feature_num, :]\n",
        "binary_code_pre = np.sign(np.dot(X_test, w)+e)\n",
        "binary_code_matrix = np.sign(np.dot(X_train_b, w_b))\n",
        "\n",
        "ann = NNDescent(binary_code_matrix,compressed=True,n_jobs=-1)\n",
        "neighbors_index, neighbors_B = ann.query(binary_code_pre, k=5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yWFhqjvcRVnw"
      },
      "source": [
        "Evaluate the kl loss on the testing set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "id": "L2_qV6zWRQAO",
        "outputId": "ad307aa5-64b7-4b7e-ecc5-71e243860d13"
      },
      "source": [
        "y_pred = np.zeros((len(X_test), label_num))\n",
        "for i in range(len(X_test)):\n",
        "    y_pred[i] = np.average(y_train[neighbors_index[i]], axis=0)\n",
        "loss_test_mean = np.mean(scipy.stats.entropy(y_test,y_pred))\n",
        "print(\"Testing set\")\n",
        "print(\"kl_dist mean:\",loss_test_mean)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Testing set\n",
            "kl_dist mean: 0.1728530329588176\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HxcBO_uSbpEn"
      },
      "source": [
        "## MSLP-LDL"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P-v4xpAzcY8s"
      },
      "source": [
        "Peng C L, Tao A, Geng X. Label Embedding Based on Multi-Scale Locality Preservation[C]//IJCAI. 2018: 2623-2629."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c-bjOTFDe3m5"
      },
      "source": [
        "def predict_func(v, x_train, y_train, x_test, k):\n",
        "    test_num = len(x_test)\n",
        "    label_num = len(y_train[0])\n",
        "    y_train_l = np.dot(x_train, v)\n",
        "    y_test_l = np.dot(x_test, v)\n",
        "    neigh = neighbors.NearestNeighbors(n_neighbors=k)\n",
        "    neigh.fit(y_train_l)\n",
        "    neiy_index = neigh.kneighbors(y_test_l, return_distance=False)\n",
        "    y_pre = np.zeros((test_num, label_num))\n",
        "    for i in range(test_num):\n",
        "        y_pre[i] = np.average(y_train[neiy_index[i]], axis=0)\n",
        "    return y_pre\n",
        "\n",
        "    \n",
        "def euclidean(A, B):\n",
        "    if np.shape(A)[1] != np.shape(B)[1]:\n",
        "        raise ValueError('A and B must be of same dimensionality.')\n",
        "\n",
        "    if np.shape(A)[1] == 1:\n",
        "        A = A.reshape(-1, 1)\n",
        "        B = B.reshape(-1, 1)\n",
        "    an = np.shape(A)[0]\n",
        "    bn = np.shape(B)[0]\n",
        "\n",
        "    a = np.sum(A * A, 1)\n",
        "    b = np.sum(B * B, 1)\n",
        "    ab = A.dot(B.T)\n",
        "\n",
        "    D = np.sqrt(np.abs(np.repeat(a.reshape(-1, 1), bn, 1) + np.repeat(np.array([b]), an, 0) - 2 * ab))\n",
        "    return D"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AzeiNqVfcZt1"
      },
      "source": [
        "from sklearn import neighbors\n",
        "alpha = 10\n",
        "k = 1\n",
        "l = 1\n",
        "k1 = 2\n",
        "k2 = 2\n",
        "beta = 0.1\n",
        "lambda1 = 0.01\n",
        "\n",
        "train_num = X_train.shape[0]\n",
        "feature_num = X_train.shape[1]\n",
        "label_num = y_train.shape[1]\n",
        "w1 = np.zeros((train_num, train_num))\n",
        "w2 = np.zeros((train_num, train_num))\n",
        "neighx = neighbors.NearestNeighbors(n_neighbors=alpha * k1 + 1)\n",
        "neighx.fit(X_train)\n",
        "nx_index = neighx.kneighbors(X_train, return_distance=False)[:, 1:alpha * k1 + 1]\n",
        "ny_d = np.zeros((train_num, k1))\n",
        "ny_index = np.zeros((train_num, k1), dtype=int)\n",
        "neighy = neighbors.NearestNeighbors(n_neighbors=k1)\n",
        "for i in range(train_num):\n",
        "    neighy.fit(y_train[nx_index[i]])\n",
        "    ny_d[i], ny_index[i] = neighy.kneighbors(y_train[i].reshape(1, -1))\n",
        "\n",
        "neighx_k2 = neighbors.NearestNeighbors(n_neighbors=k2 + 1)\n",
        "neighx_k2.fit(X_train)\n",
        "nx_k2_index = neighx_k2.kneighbors(X_train, return_distance=False)[:, 1:k2 + 1]\n",
        "distence_y = euclidean(y_train, y_train)\n",
        "ny_far_index = np.argsort(-distence_y, axis=1)\n",
        "ny_far_index = np.argwhere(ny_far_index < k2)\n",
        "sigma = np.sum(ny_d ** 2) / (train_num * k1)\n",
        "for i in range(train_num):\n",
        "    for j in range(k1):\n",
        "        w1[i][ny_index[i, j]] = np.exp(-ny_d[i, j] / sigma)\n",
        "        w1[ny_index[i, j]][i] = w1[i][ny_index[i, j]]\n",
        "        w2[i][ny_far_index[i * k2 + j, 1]] = 1\n",
        "        w2[ny_far_index[i * k2 + j, 1]][i] = 1\n",
        "\n",
        "D1 = np.zeros((train_num, train_num))\n",
        "D2 = np.zeros((train_num, train_num))\n",
        "for i in range(train_num):\n",
        "    D1[i, i] = np.sum(w1[i])\n",
        "    D2[i, i] = np.sum(w2[i])\n",
        "M1 = D1 - w1\n",
        "M2 = D2 - w2\n",
        "\n",
        "M = beta * M1 - (1 - beta) * M2\n",
        "term1 = np.linalg.pinv(np.dot(np.dot(X_train.T, D1), X_train))\n",
        "term2 = np.dot(np.dot(X_train.T, M), X_train) + lambda1 * np.identity(len(x_train[0]))\n",
        "eigenvalue, eigenvector = np.linalg.eig(term1 * term2)\n",
        "eigenvalue_sort_index = np.argsort(eigenvalue)\n",
        "eigenvalue_index = np.argwhere(eigenvalue_sort_index < l)\n",
        "v = np.zeros((len(X_train[0]), l))\n",
        "for i in range(l):\n",
        "    v[:, i:i + 1] = eigenvector[:, eigenvalue_index[i]]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q4uwcmT1ft4o"
      },
      "source": [
        "from sklearn import neighbors\n",
        "train_num = X_train.shape[0]\n",
        "feature_num = X_train.shape[1]\n",
        "label_num = y_train.shape[1]\n",
        "w1 = np.zeros((train_num,train_num))\n",
        "w2 = np.zeros((train_num, train_num))\n",
        "neighbor_x = neighbors.NearestNeighbors(n_neighbors=20)\n",
        "neighbor_x.fit(X_train)\n",
        "neighbor_x_index = neighx.kneighbors(X_train, return_distance=False)[:, 1:20]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4aGu6Lv_e9ET"
      },
      "source": [
        "y_pred = predict_func(v, x_train, y_train, x_test, k)\n",
        "loss_test_mean = np.mean(scipy.stats.entropy(y_test,y_pred))\n",
        "print(\"Testing set\")\n",
        "print(\"kl_dist mean:\",loss_test_mean)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}