4 - Packed Padded Sequences, Masking and Inference.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import torchtext\n",
    "from torchtext.datasets import TranslationDataset, Multi30k\n",
    "from torchtext.data import Field, BucketIterator\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "\n",
    "import spacy\n",
    "\n",
    "import random\n",
    "import math\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED = 1\n",
    "\n",
    "random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "spacy_de = spacy.load('de')\n",
    "spacy_en = spacy.load('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_de(text):\n",
    "    \"\"\"\n",
    "    Tokenizes German text from a string into a list of strings\n",
    "    \"\"\"\n",
    "    return [tok.text for tok in spacy_de.tokenizer(text)]\n",
    "\n",
    "def tokenize_en(text):\n",
    "    \"\"\"\n",
    "    Tokenizes English text from a string into a list of strings\n",
    "    \"\"\"\n",
    "    return [tok.text for tok in spacy_en.tokenizer(text)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- include lengths in source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "SRC = Field(tokenize=tokenize_de, init_token='<sos>', eos_token='<eos>', lower=True, include_lengths=True)\n",
    "TRG = Field(tokenize=tokenize_en, init_token='<sos>', eos_token='<eos>', lower=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, valid_data, test_data = Multi30k.splits(exts=('.de', '.en'), fields=(SRC, TRG))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "SRC.build_vocab(train_data, min_freq=2)\n",
    "TRG.build_vocab(train_data, min_freq=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- all this stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "train_iterator, valid_iterator, test_iterator = BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data), \n",
    "     batch_size=BATCH_SIZE,\n",
    "     sort_key = lambda x : len(x.src),\n",
    "     sort_within_batch=True,\n",
    "     device=device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- encoder needs to accept lengths\n",
    "- pack sequence before going into rnn\n",
    "- unpack output before returning it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, input_dim, emb_dim, enc_hid_dim, dec_hid_dim, dropout):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.input_dim = input_dim\n",
    "        self.emb_dim = emb_dim\n",
    "        self.enc_hid_dim = enc_hid_dim\n",
    "        self.dec_hid_dim = dec_hid_dim\n",
    "        self.dropout = dropout\n",
    "        \n",
    "        self.embedding = nn.Embedding(input_dim, emb_dim)\n",
    "        \n",
    "        self.rnn = nn.GRU(emb_dim, enc_hid_dim, bidirectional=True)\n",
    "        \n",
    "        self.fc = nn.Linear(enc_hid_dim * 2, dec_hid_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, src, src_len):\n",
    "        \n",
    "        #src = [src sent len, batch size]\n",
    "        #src_len = [src sent len]\n",
    "        \n",
    "        embedded = self.dropout(self.embedding(src))\n",
    "        \n",
    "        #embedded = [src sent len, batch size, emb dim]\n",
    "        \n",
    "        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, src_len)\n",
    "        \n",
    "        packed_outputs, hidden = self.rnn(packed_embedded)\n",
    "               \n",
    "        outputs, _ = nn.utils.rnn.pad_packed_sequence(packed_outputs) \n",
    "            \n",
    "        #outputs = [sent len, batch size, hid dim * num directions]\n",
    "        #hidden = [n layers * num directions, batch size, hid dim]\n",
    "        \n",
    "        #hidden is stacked [forward_1, backward_1, forward_2, backward_2, ...]\n",
    "        #outputs are always from the last layer\n",
    "        \n",
    "        #hidden [-2, :, : ] is the last of the forwards RNN \n",
    "        #hidden [-1, :, : ] is the last of the backwards RNN\n",
    "        \n",
    "        #initial decoder hidden is final hidden state of the forwards and backwards encoder RNNs fed through a linear layer\n",
    "        hidden = torch.tanh(self.fc(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1)))\n",
    "        \n",
    "        #outputs = [sent len, batch size, enc hid dim * 2]\n",
    "        #hidden = [batch size, dec hid dim]\n",
    "        \n",
    "        return outputs, hidden"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- unchanged as mask is applied in decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Attention(nn.Module):\n",
    "    def __init__(self, enc_hid_dim, dec_hid_dim):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.enc_hid_dim = enc_hid_dim\n",
    "        self.dec_hid_dim = dec_hid_dim\n",
    "        \n",
    "        self.attn = nn.Linear((enc_hid_dim * 2) + dec_hid_dim, dec_hid_dim)\n",
    "        self.v = nn.Parameter(torch.rand(dec_hid_dim))\n",
    "        \n",
    "    def forward(self, hidden, encoder_outputs, mask):\n",
    "        \n",
    "        #hidden = [batch size, dec hid dim]\n",
    "        #encoder_outputs = [src sent len, batch size, enc hid dim * 2]\n",
    "        #mask = [batch size, src sent len]\n",
    "        \n",
    "        batch_size = encoder_outputs.shape[1]\n",
    "        src_len = encoder_outputs.shape[0]\n",
    "        \n",
    "        #repeat encoder hidden state src_len times\n",
    "        hidden = hidden.unsqueeze(1).repeat(1, src_len, 1)\n",
    "        \n",
    "        encoder_outputs = encoder_outputs.permute(1, 0, 2)\n",
    "        \n",
    "        #hidden = [batch size, src sent len, dec hid dim]\n",
    "        #encoder_outputs = [batch size, src sent len, enc hid dim * 2]\n",
    "        \n",
    "        energy = torch.tanh(self.attn(torch.cat((hidden, encoder_outputs), dim=2))) \n",
    "        \n",
    "        #energy = [batch size, src sent len, dec hid dim]\n",
    "                \n",
    "        energy = energy.permute(0, 2, 1)\n",
    "        \n",
    "        #energy = [batch size, dec hid dim, src sent len]\n",
    "        \n",
    "        #v = [dec hid dim]\n",
    "        \n",
    "        v = self.v.repeat(batch_size, 1).unsqueeze(1)\n",
    "        \n",
    "        #v = [batch size, 1, dec hid dim]\n",
    "            \n",
    "        attention = torch.bmm(v, energy).squeeze(1)\n",
    "        \n",
    "        #attention = [batch size, src sent len]\n",
    "        \n",
    "        attention = attention.masked_fill(mask == 0, -1e10)\n",
    "        \n",
    "        return F.softmax(attention, dim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- applies mask to attention output by decoder\n",
    "- renormalizes attention so each row sums to 1 still\n",
    "- returns attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, output_dim, emb_dim, enc_hid_dim, dec_hid_dim, dropout, attention):\n",
    "        super().__init__()\n",
    "\n",
    "        self.emb_dim = emb_dim\n",
    "        self.enc_hid_dim = enc_hid_dim\n",
    "        self.dec_hid_dim = dec_hid_dim\n",
    "        self.output_dim = output_dim\n",
    "        self.dropout = dropout\n",
    "        self.attention = attention\n",
    "        \n",
    "        self.embedding = nn.Embedding(output_dim, emb_dim)\n",
    "        \n",
    "        self.rnn = nn.GRU((enc_hid_dim * 2) + emb_dim, dec_hid_dim)\n",
    "        \n",
    "        self.out = nn.Linear((enc_hid_dim * 2) + dec_hid_dim + emb_dim, output_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, input, hidden, encoder_outputs, mask):\n",
    "             \n",
    "        #input = [batch size]\n",
    "        #hidden = [batch size, dec hid dim]\n",
    "        #encoder_outputs = [src sent len, batch size, enc hid dim * 2]\n",
    "        #mask = [batch size, src sent len]\n",
    "        \n",
    "        input = input.unsqueeze(0)\n",
    "        \n",
    "        #input = [1, batch size]\n",
    "        \n",
    "        embedded = self.dropout(self.embedding(input))\n",
    "        \n",
    "        #embedded = [1, batch size, emb dim]\n",
    "        \n",
    "        a = self.attention(hidden, encoder_outputs, mask)\n",
    "                \n",
    "        #a = [batch size, src sent len]\n",
    "        \n",
    "        a = a.unsqueeze(1)\n",
    "        \n",
    "        #a = [batch size, 1, src sent len]\n",
    "        \n",
    "        encoder_outputs = encoder_outputs.permute(1, 0, 2)\n",
    "        \n",
    "        #encoder_outputs = [batch size, src sent len, enc hid dim * 2]\n",
    "        \n",
    "        weighted = torch.bmm(a, encoder_outputs)\n",
    "        \n",
    "        #weighted = [batch size, 1, enc hid dim * 2]\n",
    "        \n",
    "        weighted = weighted.permute(1, 0, 2)\n",
    "        \n",
    "        #weighted = [1, batch size, enc hid dim * 2]\n",
    "        \n",
    "        rnn_input = torch.cat((embedded, weighted), dim=2)\n",
    "        \n",
    "        #rnn_input = [1, batch size, (enc hid dim * 2) + emb dim]\n",
    "            \n",
    "        output, hidden = self.rnn(rnn_input, hidden.unsqueeze(0))\n",
    "        \n",
    "        #output = [sent len, batch size, dec hid dim * n directions]\n",
    "        #hidden = [n layers * n directions, batch size, dec hid dim]\n",
    "        \n",
    "        #sent len, n layers and n directions will always be 1 in this decoder, therefore:\n",
    "        #output = [1, batch size, dec hid dim]\n",
    "        #hidden = [1, batch size, dec hid dim]\n",
    "        #this also means that output == hidden\n",
    "        assert (output == hidden).all(), print(output.shape, hidden.shape, output[0,0,:25], hidden[0,0,:25])\n",
    "        \n",
    "        embedded = embedded.squeeze(0)\n",
    "        output = output.squeeze(0)\n",
    "        weighted = weighted.squeeze(0)\n",
    "        \n",
    "        output = self.out(torch.cat((output, weighted, embedded), dim=1))\n",
    "        \n",
    "        #output = [bsz, output dim]\n",
    "        \n",
    "        return output, hidden.squeeze(0), a.squeeze(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- seq2seq needs to take in src_len, creates mask\n",
    "- pass mask to decoder to be used in attn calculation\n",
    "- receive attention from decoder and store in tensor and return it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Seq2Seq(nn.Module):\n",
    "    def __init__(self, encoder, decoder, pad_idx, sos_idx, eos_idx, device):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.pad_idx = pad_idx\n",
    "        self.sos_idx = sos_idx\n",
    "        self.eos_idx = eos_idx\n",
    "        self.device = device\n",
    "        \n",
    "    def create_mask(self, src):\n",
    "        mask = (src != self.pad_idx).permute(1, 0)\n",
    "        return mask\n",
    "        \n",
    "    def forward(self, src, src_len, trg, teacher_forcing_ratio=0.5):\n",
    "        \n",
    "        #src = [src sent len, batch size]\n",
    "        #src_len = [batch size]\n",
    "        #trg = [trg sent len, batch size]\n",
    "        #teacher_forcing_ratio is probability to use teacher forcing\n",
    "        #e.g. if teacher_forcing_ratio is 0.75 we use teacher forcing 75% of the time\n",
    "        \n",
    "        if trg is None:\n",
    "            inference = True\n",
    "            assert teacher_forcing_ratio == 0, \"Must be zero during inference\"\n",
    "            trg = torch.zeros((100, src.shape[1]), dtype=torch.long).fill_(self.sos_idx).to(src.device)\n",
    "        else:\n",
    "            inference = False\n",
    "            \n",
    "        batch_size = src.shape[1]\n",
    "        max_len = trg.shape[0]\n",
    "        trg_vocab_size = self.decoder.output_dim\n",
    "        \n",
    "        #tensor to store decoder outputs\n",
    "        outputs = torch.zeros(max_len, batch_size, trg_vocab_size).to(self.device)\n",
    "        \n",
    "        #tensor to store attention\n",
    "        attentions = torch.zeros(max_len, batch_size, src.shape[0]).to(self.device)\n",
    "        \n",
    "        #encoder_outputs is all hidden states of the input sequence, back and forwards\n",
    "        #hidden is the final forward and backward hidden states, passed through a linear layer\n",
    "        encoder_outputs, hidden = self.encoder(src, src_len)\n",
    "                \n",
    "        #first input to the decoder is the <sos> tokens\n",
    "        output = trg[0,:]\n",
    "        \n",
    "        mask = self.create_mask(src)\n",
    "                \n",
    "        #mask = [batch size, src sent len]\n",
    "                \n",
    "        for t in range(1, max_len):\n",
    "            output, hidden, attention = self.decoder(output, hidden, encoder_outputs, mask)\n",
    "            outputs[t] = output\n",
    "            attentions[t] = attention\n",
    "            teacher_force = random.random() < teacher_forcing_ratio\n",
    "            top1 = output.max(1)[1]\n",
    "            output = (trg[t] if teacher_force else top1)\n",
    "            if inference and output.item() == self.eos_idx:\n",
    "                return outputs[:t], attentions[:t]\n",
    "            \n",
    "        return outputs, attentions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20518405"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "INPUT_DIM = len(SRC.vocab)\n",
    "OUTPUT_DIM = len(TRG.vocab)\n",
    "ENC_EMB_DIM = 256\n",
    "DEC_EMB_DIM = 256\n",
    "ENC_HID_DIM = 512\n",
    "DEC_HID_DIM = 512\n",
    "ENC_DROPOUT = 0.5\n",
    "DEC_DROPOUT = 0.5\n",
    "PAD_IDX = SRC.vocab.stoi['<pad>']\n",
    "SOS_IDX = TRG.vocab.stoi['<sos>']\n",
    "EOS_IDX = TRG.vocab.stoi['<eos>']\n",
    "\n",
    "attn = Attention(ENC_HID_DIM, DEC_HID_DIM)\n",
    "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, ENC_DROPOUT)\n",
    "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, DEC_DROPOUT, attn)\n",
    "\n",
    "model = Seq2Seq(enc, dec, PAD_IDX, SOS_IDX, EOS_IDX, device).to(device)\n",
    "\n",
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "count_parameters(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_idx = TRG.vocab.stoi['<pad>']\n",
    "\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=pad_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion, clip):\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    \n",
    "    for i, batch in enumerate(iterator):\n",
    "        \n",
    "        src, src_len = batch.src\n",
    "        trg = batch.trg\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        output, _ = model(src, src_len, trg)\n",
    "        \n",
    "        loss = criterion(output[1:].view(-1, output.shape[2]), trg[1:].view(-1))\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, criterion):\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for i, batch in enumerate(iterator):\n",
    "\n",
    "            src, src_len = batch.src\n",
    "            trg = batch.trg\n",
    "\n",
    "            output, _ = model(src, src_len, trg, 0) #turn off teacher forcing\n",
    "\n",
    "            loss = criterion(output[1:].view(-1, output.shape[2]), trg[1:].view(-1))\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Epoch: 001 | Train Loss: 4.243 | Train PPL:  69.583 | Val. Loss: 3.689 | Val. PPL:  39.990 |\n",
      "| Epoch: 002 | Train Loss: 3.092 | Train PPL:  22.016 | Val. Loss: 3.368 | Val. PPL:  29.019 |\n",
      "| Epoch: 003 | Train Loss: 2.651 | Train PPL:  14.173 | Val. Loss: 3.288 | Val. PPL:  26.797 |\n",
      "| Epoch: 004 | Train Loss: 2.368 | Train PPL:  10.680 | Val. Loss: 3.152 | Val. PPL:  23.394 |\n",
      "| Epoch: 005 | Train Loss: 2.153 | Train PPL:   8.609 | Val. Loss: 3.229 | Val. PPL:  25.259 |\n",
      "| Epoch: 006 | Train Loss: 2.004 | Train PPL:   7.419 | Val. Loss: 3.288 | Val. PPL:  26.777 |\n",
      "| Epoch: 007 | Train Loss: 1.867 | Train PPL:   6.469 | Val. Loss: 3.354 | Val. PPL:  28.619 |\n",
      "| Epoch: 008 | Train Loss: 1.778 | Train PPL:   5.916 | Val. Loss: 3.389 | Val. PPL:  29.641 |\n",
      "| Epoch: 009 | Train Loss: 1.697 | Train PPL:   5.457 | Val. Loss: 3.384 | Val. PPL:  29.493 |\n",
      "| Epoch: 010 | Train Loss: 1.613 | Train PPL:   5.017 | Val. Loss: 3.445 | Val. PPL:  31.343 |\n"
     ]
    }
   ],
   "source": [
    "N_EPOCHS = 10\n",
    "CLIP = 10\n",
    "SAVE_DIR = 'models'\n",
    "MODEL_SAVE_PATH = os.path.join(SAVE_DIR, 'tut4_model.pt')\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "if not os.path.isdir(f'{SAVE_DIR}'):\n",
    "    os.makedirs(f'{SAVE_DIR}')\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "    \n",
    "    train_loss = train(model, train_iterator, optimizer, criterion, CLIP)\n",
    "    valid_loss = evaluate(model, valid_iterator, criterion)\n",
    "    \n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), MODEL_SAVE_PATH)\n",
    "    \n",
    "    print(f'| Epoch: {epoch+1:03} | Train Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f} | Val. Loss: {valid_loss:.3f} | Val. PPL: {math.exp(valid_loss):7.3f} |')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Test Loss: 3.201 | Test PPL:  24.566 |\n"
     ]
    }
   ],
   "source": [
    "SAVE_DIR = 'models'\n",
    "model.load_state_dict(torch.load(os.path.join(SAVE_DIR, 'tut4_model.pt')))\n",
    "\n",
    "test_loss = evaluate(model, test_iterator, criterion)\n",
    "\n",
    "print(f'| Test Loss: {test_loss:.3f} | Test PPL: {math.exp(test_loss):7.3f} |')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate_sentence(sentence):\n",
    "    tokenized = tokenize_de(sentence) #tokenize sentence\n",
    "    tokenized = ['<sos>'] + [t.lower() for t in tokenized] + ['<eos>'] #add <sos> and <eos> tokens and lowercase\n",
    "    numericalized = [SRC.vocab.stoi[t] for t in tokenized] #convert tokens into indexes\n",
    "    sentence_length = torch.LongTensor([len(numericalized)]).to(device) #need sentence length for masking\n",
    "    tensor = torch.LongTensor(numericalized).unsqueeze(1).to(device) #convert to tensor and add batch dimension\n",
    "    translation_tensor_probs, attention = model(tensor, sentence_length, None, 0) #pass through model to get translation probabilities\n",
    "    translation_tensor = torch.argmax(translation_tensor_probs.squeeze(1), 1) #get translation from highest probabilities\n",
    "    translation = [TRG.vocab.itos[t] for t in translation_tensor][1:] #ignore the first token, just like we do in the training loop\n",
    "    return translation, attention[1:] #ignore first attention array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_attention(candidate, translation, attention):\n",
    "    # Set up figure with colorbar\n",
    "    fig = plt.figure()\n",
    "    ax = fig.add_subplot(111)\n",
    "    attention = attention[:len(translation)].squeeze(1).cpu().detach().numpy() #cut attention to same length as translation\n",
    "    cax = ax.matshow(attention, cmap='bone')\n",
    "    fig.colorbar(cax)\n",
    "\n",
    "    # Set up axes\n",
    "    ax.set_xticklabels([''] + ['<sos>'] + [t.lower() for t in tokenize_de(candidate)] + ['<eos>'], rotation=90)\n",
    "    ax.set_yticklabels([''] + translation)\n",
    "\n",
    "    # Show label at every tick\n",
    "    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "\n",
    "    plt.show()\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "zwei junge weiße männer sind im freien in der nähe vieler büsche .\n",
      "two young , white males are outside near many bushes .\n"
     ]
    }
   ],
   "source": [
    "candidate = ' '.join(vars(train_data.examples[0])['src'])\n",
    "candidate_translation = ' '.join(vars(train_data.examples[0])['trg'])\n",
    "\n",
    "print(candidate)\n",
    "print(candidate_translation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['two', 'young', 'white', 'white', 'males', 'are', 'outside', 'near', 'bushes', 'near', 'bushes', '.']\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEaCAYAAAAboUz3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xe8ZFWZ7vHf000OggrXBAgyCANIECSrmBCZEVRwEHEUYURHkVEHHcaIOI5iwIteDIgoIoqIEi6DggloWgnd0GS4ZGnEgARFm9B9nvvH2gXVhxNqV+1zqrrO8+VTn67ateutdeg+q1atvdb7yjYRETF8ZvW7ARERMTXSwUdEDKl08BERQyodfETEkEoHHxExpNLBR0QMqXTwERFDKh18RMSQSgc/g0maLek7/W5HREyNdPAzmO0lwNqSVuh3WyKiecv1uwHRd7cDcyWdBfy1ddD20X1rUUQ0Ih18/La6zQJW73NbYhkj6dnARrZ/JmllYDnbf+l3u6JQko0FgKRVbf918jMjCklvAw4GnmJ7Q0kbAV+1/bI+Ny0qmYOf4STtKOk64Prq8ZaSvtznZsWy4V3AzsCfAWzfBPyvvrYolpIOPv438ErgTwC2rwRe1NcWxbLiYduPtB5IWg7o65SApOUlLZD0gn62Y1DMmA5e0iqS7peUr4+j2L5z1KElfWlILGsukPRBYGVJrwB+APzfPrdpL2AF4G19bsdAmDEdPPBPwLXAv/S7IQPmTkk7AZa0gqTDqKZrIiZxOPBH4Grg7cA5wIf72iI4CDgQ2FXSKn1uS9/NmIuski6k/CP8EbCT7fv63KSBIGkt4Bjg5YCA84B/s/2nvjYsoiZJ6wJn2N5G0meA621/s9/t6qcZ0cFL2gQ43vYukj4K3Gf7S31qi4D9gefYPlLSesDTbV9aM85zga8AT7O9uaQtgD1t/1fzrV72VN9K1qdtKbDtb/etQUNI0s7AEcCzKf+fBdj2c/rUno8CD9g+RtLfA1+3vUs/2jIoZkoH/1ngBtvfqDrUM2w/v09t+QowArzU9t9LejJwnu1aF4UkXQC8H/ia7a2rY9fY3rxmnLUp85Xrs3RneGCdOINE0knAhsACHr+eYNuH9q9Vg0HSLOCquv9Oxol1A/BeYD5t12368e2vGjjdCOxg+97q2K+BA2zfON3tGRRDv9FJ0vLA3sDzAGz/RtKfJG1re14fmrS97edLuqJqz31dpgpYxfal5d/1YxZ3EedMYA7wM4bn4uq2wKaeCaOXmmyPSLpS0nq2f9NjuAds/7iRhvVudeA9rc698k7Kt4oZa+g7eGB54HWjNvH8C/3rzB6VNJtqOVk1gh7pIs49kjZsi7MPcHcXcVax/R9dvG6QXQM8ne7+f8wEzwCulXQpS6en2LOTF0tqffv9ZfXt+EfAw21xLm+wrR2x/WfKRV7gsW8qt1THZ6wZMUXTrpoSWdf2VX16//2BfYHnAycC+wAftv2DmnGeAxwH7ATcB9wGvMn27TXj/BfwK9vnTHryMkLSL4GtgEtZuuPpqAMbZJJ2oaQG+GY1OFjN9m01Y7x4rOO2L+jw9b+c4Gnbfmmd9jRF0neBd1AGb/OBNYCjbX+2H+0ZBDOig5d0PrAn5RvLAsrSrgtsv69P7dkEeBnl6+PPbXe9LFHSqsCsbvN/SPoLsCqlI3yUxy+UPanbNvVbrx3YoJL0Mcr008a2nyvpmcAPbO/cRaz2HDKrALOX9RwykhbY3qoaRG0D/Acw3/YWfW5a38yUdfBrVF/VXgd80/Y2lGWB007SU4A/AN8Dvgv8vrpOUCfGZpKeUT2cDZwq6Q5J+9Ztj+3Vbc+yvbLtJ1WPl9nOHR7ryG8Hlq/uXwZM+7TBFHgtZaDyVwDbv6WLBHFVDpnTgK9Vh54FnNFFnP+WtGbb4ydX3wj7Zfnqd+k1wJm2H6XPO2v7baZ08MtVHeI/AWfXeaGkl1Z/vm6sWxdtuZzyDeL/ATdV92+TdLmkbTpoz9HAp4HWipBvAj8Ftgc+2EV7kPQsSTtJelHr1k2cQdFUBzaAHqkuHLeuu6zaZZymcsi8yvb9rQfV3pI9umxTE75G+WBfFbiw+pYyo+fgZ8JFVoAjgXOBubYvq+avb+rwtS8GfgG8eoznTLnAVMdPgNNtnwsgaTdgd+BU4MuUjnoiL7G9taSTJN1FmT8/uopVe7Qi6SjKNYHraFtSCFxYN9YAeRewHXAJlA5M0jAkwTpV0teANasPsQOBr3cR52Hbj7RWYPWQQ2a2pBVtP1zFWRlYsYs4jbD9ReCLbYfukPSSfrVnEMyIOfhBImme7W3HOtaaQ5zk9VcCr7F9m6Qnt3bkVqsG3mb7axO9fox4NwJbtH5J66rm8Mf9R9SP6R5Jl9jeXtIV1YfhcsDlwzAXW+V82Y1yreRc2z/tIsZngPuBNwPvpiwnvM72h2rG+QBlyuiblH8DBwJn2f5M3TY1QdIawMd4PFneBcCRth/oR3sGwYzo4CWtA3yJ8rXUwEWU7fgLa8R4GvDfwDNtv0rSpsCOtr9Rsy3nAT8HTqkO7Qu8gjKKv2yyDViS9gCOBZ46+im6uDgq6cfA620/WOd1Y8Q5EvgdcFLVlv2B1fvxy95UBzasqsHAQbR9UFB2enfzDXB32tJctL6Z9oOkH1KWyJ5YHfpnYEvb3UylDoWZ0sH/lHJB86Tq0JuA/W2/okaMH1NGKh+yvWU1KrzC9vNqtmUtyihjF8ovxUXAx4EHgPVs31wnXq+qX4otKR867UsKa+36bI2aJzs2HZrswAZJdc3nKMp8uejziqfqGsCiavPUxsDGwI+ri5v9aM8TvgF38q14mM2UDr7nv3hJl9l+QetrfzcxmiBpE9s3tG02WUrdTSaS3jJOnBPHOj5BnF9RvlmcQvmWtB/wLts71YkzbKpNbZ+2/f4GYt0MvLrbZbWSrmbi6bRaU1iS5gMvBJ4MXAzMA/5me/9u2tcrldQE77d9UfV4Z+BztnfsR3sGwUy5yHqPpDdRliZC6Xzq5sv4q6Sn8vgKhh0oo+5aVJKEHcYTc790ujnkfZQyaZ8f4zkDtTaZ1O3IJ/BGSlbKY6p2zK2OTRtJp9r+p/E6sn7MwdteImkbSWrgG8Tve9kzAfxjj+8/mmz/TdJBwJdsf0ZVCo4++VfgxGouHsoGwAP615z+mykj+PWA/wO0PsnnUubg76gRYxvKFfrNKfN8awP71N0RW10k/SpPTNA0v06cpqjU0fwUsCmwUlt7+pIRsBeSnmH77mp53BPU+ftukqTPAxtRCmK0pwboaAVW23LcF1NSMJzB0tNpdVdyNVIsu+rM3wl8ATjI9rWSrq47bdk0SU+Cx9IXzGgzooNvSjXvvjFl7vPGbuYaJc2vNlr12pZVKKP59WwfXHXUG9vuaJ2/pItc0idfRLkm8AXKUtC3Uv5dfKxmewYqK2UTHViDbRkrJ7k7/X8zzutrx2mL10ixbJUdw/9OWX58VLX8+D11r980pamFEMNkRnTw1aqK/wIWUdahb0n5h/idGjHmUNaGz6H8g+42NcARlJ2sp7P0KOze8V4zTpzvU74FvNklH/zKwK87vSYg6Um2/9z6wGkfeUmaY/uFNdvzK8r/m9HfTH5YJ04TmurAhpWkBVT7BNquJ/V95N2rphZCDJOZMge/m+0PSHotsBB4PfBLoOMOHngLZeXL3sBnJT0MzLH93pptaV3UbL/oZqDulMiGtveVtB+A7UWS6qRG/S5lTvbh6nU3SToEuIvudjU2kpVS0gaUpY3rs/Q3gTqJwgZqo5MaKs4iaSXK6qDNWHo6re63pEY2OqkkHRvrWkdfko0Ba9k+VdJ/Vu1YLGlYUmB3ZaZ08K1cL3sA37N9b72+EGzfKmkR8Eh1ewnw93UbYnuDuq8ZxyPVqL110XdD2r4RdNCO1gW39wKrUVIffILyc725i/acLWkP956V8gzgG5Tizd2kUYbmdmo25etUxVkAbF+lkvmwow5e0nG2D6Ys870BeCVld/b+dFc/9wItXSz7ndQolt26mE1ZLNCyEmXw001NgqY0shBimMyUKZpPUxIQLaKM7NYEzq6zRlvSLcA9lJHvHGCB7dodkKQxO0/XLCdX/WJ+mHJx9DzKJq4DbJ9fM862wIcoZddaH4TuYslcI1kpm1g7rwHb6NTrEtu2pbGtnblX2d5CJbHWuXVHzL3uE2hdzB7nuQtsj5nNc6pVS4e/RI8LIYbJjOjgAVTywP+5Wra2KmWX5e9qvP7fKFM061JGURcAF9q+pWY72mvBrkRJG3y57X1qxjmJUs1+EXArZT71njoxqjg3UkaXV9M2Yu7jipM3UlacnEeXRSR67cDGiNdTfddqbvgQSmrf56sUZznI9qs6fH1r+eeltrdTKSD/TsrO4Uune8WTpLNt/6NKZtSWWZRUxsfY3ng629OuiYUQw2ToO/hqtclGtq9sO7YesMT2XV3EW42y0uQwYB3bs3ts3xrASTXnmFHJcrkLZaPJcyh57i+0fUzNOBe5h8LEan7j1acoW8xv4fEPHHc6SlXZWHSi7TfVed8J4vVc31VjF2fZv9MP0bbln2+nJKV7HvAtytTaR9xh/iE1tE9A0hq2H5B0W1ucxZRMjke62mg0nZr+PR8WM6GDX54y4t7CVdk+lXwwH3SNmqySPkfpTFcDfk2Zpplj+9YG2neV7drz+VVn9gLKvPk7KNvGN6kZ42WUjV+jUxV0ukb7OJdlmu1Vfh77R9XF9EHr7+qROq8bFeNcyo7PrmO0xbqeLuu7ShpdUGZlyki3lc/96JrxVqTMc6/P0tNpR3b4+tYHxfso1a7ubH++7re26hrQOykDDVN+J75i+6E6cZrQ1O/5sBn6i6y2H5V0OiWp1wnVp/raXfylvw/4Z9sntw5IOo6yHK9jkv4vj3eAsylz6KfWbAuSfk6Z82592LzA9h/qxqF8G9mE0mE8NmKmwzTI1cU/KKtEflItvfwIpSThJ7poz5WUayTd/CwttwNzJZ3F0huLanWolV7qu7aKcWxM+SA+kzJ18M90l475TMpFw/nUuKDe0jZvvjrlgu+9lNQSp9n+fRftOZGSb72Vonc/yoXg13cRqycN/p4PlaEfwUOZRgC+bvuFkj5MmYv/4mSvGxXjVsqI5+etEZOkyz1J9scx4ryYpb/W3tHlVNEXKGXJHqbszL2Qsg5+Uc04jax/brvwtwtls8nnKaOnWhdMVcorbkGpwlSrnqqkk2z/s6T7KRu3lmL743XaUsXsub5rNZLc29XeCUmrU+bjd6/Zlmtsb17nNZPE24LSIe4NLLRdq8qZpCttbznZsenSxO/5sBn6ETxANUfcWo+8H+UrZV33Uy6IfrEahdea422b6z6b0sG31mlapVDHvcBnbX+5k3iu1t+3XRP4JmWkWbfgwsWSNrV9Xc3Xjdaan/4HyqaiM1U2ddVVawftKNuo7GD9DWU1RROOaCDGepSltS2PUKZZ6vqVpOfZvrqBNkH5lvQ7Sl6mbvYJXCFpB9sXA0janjLY6IuGfs+HyowYwQNIOoBSkOAu2/t18fr2JW4HULZoP9n2Og2176mU6kwdrUBQ2ZT0Qsoo/g6qXba2f1Hzfa+nXES8jTJCbS1vrLtM8mzKJqmXV21aRFnhMW2jOUmHUhJObQD8tv0pys/Ul/w6kj5EKRd5OuXD/bXA921/qmac64C/o/e/q3+ljNzXppQ2/H6dD/i2i7TLU6afflM9fjZlOWrP3zIkPb3OKre21x1AD7/nw2YmdfCrUOZR97b9sy5e//b21Qoqycfe5QZzrUy0vniMc99P6dTn2+56c4kaSsxV/f/dHbjaZefoM4Dn2T6vw9e3cuOMrhBVez29pK/Y/tc67Z/K9lTxnk/5QIay2ql21sUG/64+DZxie0HdNkzUjm7bM857/I/tf+jidT39ng+bGdPBR0TMNLP63YCIiJga6eAjIobUjOvgJdVatz6T4gxSWxJneuIMUlsGMc6ybsbNwUuaZ3vbxBnstiTO9MQZpLYMYpzJ7L777r7nns5SQM2fP//cunsfejUj1sFHREyFe+65h3nzOtssK2mtKW7OEwxVB19tGOr5vK223nrSGOuuuy5bP//5E8a5csGVEz3dag2zZs2eME6nWYk7/fmnOkbiLFtxBqkt0xznHttr9/o+gzwLMlQdfFMumNvMZrynPaWZIkIPPfRgI3EiYik9r9c3sGSk27o0Uy8dfERE14z7WixsYungIyK6ZRgZ3P49HXxERC8GeQ5+2tbBS1pT0jun6/0iIqaagRG7o1s/TOdGpzUp1V8iIoaG7Y5u/TCdHfyngQ0lLZD0TUl7Akg6XdIJ1f2DJP1Xdf99kq6pbu+ZxnZGRHTENktGRjq69cN0dvCHA7fY3opS5b6VOvVZlLJ1UBL0z6lS8b4V2B7YAXibpMkXp0dETLOM4J9oDvBCSZsC1wG/r/KH7wj8itLRn277r7YfpNQHfeFYgSQdLGmepBldezEi+sMd/tcPfVlFY/suSU+mFIi4EHgKpeLNg7b/IkkTBlg61nHAcdDcDriIiE6Ui6z9bsX4pnME/xcerzIP8GvgPVSl5oDDqj+pjr1G0iqSVqWUOJtDRMSAGeQpmmkbwdv+k6S5kq4BfkzpsHezfbOkOyij+DnVuZdL+halkj3A8d2UOIuImFLVRdZBNa1TNLbfOOrQN6rjjwKrjjr3aODoaWpaRERtZrA3OmUna0RED/q1iakT6eAjInqQEXxExFBKNsllzhqrrNJInKY+2WusGo2IaeRkk4yIGF4jWUUTETF8WtkkB1U6+IiIHuQia0TEMOpjrvdOpIOPiOhBRvAREUPIwJJ08BERw2mQR/BdZZOU9AlJ/9b2+JOS/k3SZ6sKTFdL2rd6bldJZ7ed+38kHVDdv13SxyVdXr1mk+r42pJ+Wh3/mqQ7JK3V008aETEFBjmbZLfpgr8BvAVA0izgDcBCYCtgS+DlwGerIh6Tucf284GvUFIGA3wM+EV1/HRgvS7bGRExZdxhwe1+XYjtaorG9u2S/lSV0XsacAWlCtP3bC+hVGi6AHgB8OdJwv2o+nM+8Lrq/i6UHPDY/omk+8Z7saSDgYO7+TkiIno1yFM0vczBHw8cADwdOAHYbZzzFrP0N4WVRj3/cPXnkrb2pKJTRCwTBrmD76Wi0+mUknsvoBTRvhDYV9JsSWsDL6IU7LgD2FTSipLWAF7WQeyLKCX8kLQb8OQe2hkRMSXKKpqRjm790PUI3vYjkn4J3G97iaTTKUWzr6T83B+w/TsASacCVwE3UaZzJvNx4HvVhdoLgLspJf8iIgbKUCYbqy6u7gC8HsDle8r7q9tSbH8A+MAYx9dvuz8P2LV6+ADwStuLJe0IvMT2w6NfHxHRV31cIdOJrjp4SZsCZwOn276p2SYBZdXMqdWHyCPA26bgPSIiejKUJftsXwc8p+G2tMe/Cdh6quJHRDQluWgiIobU0I3gozNNVWJq4h9QqkJFNM82S1LwIyJiOKUma0TEkBrkZZK9bHSKiJjRWqtomkg2Jml3STdKulnS4WM8v56kX0q6QtJVkvaYLGY6+IiIHjTRwUuaDRwLvArYFNivWo7e7sPAqba3piR4/PJkbcsUTUREt5q7yLodcLPtWwEknQLsBVzX/m7Ak6r7awC/nSzotIzgJT04zvF3SHpzdf8ASc+cjvZERDShwSmaZwF3tj1eWB1rdwTwJkkLgXOAd08WtK9TNLa/avvb1cMDgHTwEbFMqZEPfi1J89pu7WnOx1rHPPpTYT/gW7bXAfYATqp2+4+rkSkaSR8AHrL9RUlfALa0/VJJLwPeWp3zSeAfgUXAXrZ/L+kI4EHgdmBb4GRJiyhJyzYFjgZWA+4BDrB9dxPtjYhoSo1lkvfY3nac5xYC67Y9XocnTsEcRMngi+1fS1oJWAv4w3hv2NQI/kLghdX9bYHVJC1PKdwxB1gVuNj2ltW5S+WWsX0aMA/Y3/ZWlBzyXwL2sb0NJd/8Jxtqa0REY+zObpO4DNhI0gaSVqBcRD1r1Dm/oUq3LunvKbU1/jhR0KYuss4HtpG0OqWAx+WUjv6FwKGUhGFnt537iknibQxsDvy02oE5m5Iy+AlS0Ski+sU0k4umypx7CKW2xmzgBNvXSjoSmGf7LODfga9Lem/11gd4ksn9Rjp4249Kup0yHfMrSu73lwAbAtcDj7Y1pL1y03gEXGt7xw7eOxWdIqI/GkxVYPscysXT9mMfbbt/HbBznZhNXmS9kFI0+0LKtMw7gAWTfcK0+QuwenX/RmDtKhc8kpaXtFmDbY2I6FmTG52mQpMd/BzgGcCvbf8eeKg61qlvAV+VtIDyFWUf4ChJVwILgJ0abGtERCMGuYNvbKOT7Z8Dy7c9fm7b/dXa7p8GnFbdP6Lt+A+BH7aFXECp6xoRMbCSDz4iYig52SQjIoZRh0sg+yYdfERED1LwI3ry8KOP9hxjueVWaKAlsHjxI43EiRgGTa2Dnyrp4CMiepCarBERw6iPSyA7kQ4+IqIX6eAjIobTyJLB7eBT8CMioktlmeQM2MnaDdtfbXt4AHANHZShiogYFEM/B5+CHxExMw32RdYU/IiI6IFH3NGtH1LwIyKiS605+EGVgh8RET3wAKcqSMGPiIgeNFSTdUqk4EdERLfc2fz7sj4Hn4IfETEjDf0cfETETNSqyTqo0sFHRPQgHXxExDCy8ZLBXUWTDj4iogcZwUdPnrdZ7wuIPnPiKQ20BI7/5OcbiXPddXMbiRPRbwPcv6eDj4joVi6yRkQMq5mQqiAiYmYyI7nIGhExnAZ5BD8tFZ3GImlXSWdPfmZExGBKRaeIiGE2rCN4SetLukHS8ZKukXSypJdLmivpJknbVbdfSbqi+nPjMeKsKukESZdV5+1VHd9M0qWSFki6StJGvbQ3IqJpHuns1g9NjOD/Dng9pejGZcAbKZWc9gQ+CLwZeJHtxZJeDvw3sPeoGB8CfmH7QElrApdK+hkl5fAxtk+WtAIly2RExMAY5Dn4Jjr422xfDSDpWuDnti3pamB9YA3gxGr0bdoyTrbZDdhT0mHV45WA9YBfAx+StA7wI9s3jX5hKjpFRN/YjAx5wY+H2+6PtD0eoXyAfAL4pe3NgVdTOu/RBOxte6vqtp7t621/l/JNYBFwrqSXjn6h7eNsb2t72wZ+loiIjrU2OjVxkVXS7pJulHSzpMPHOeefJF0n6VpJ350s5nSsolkDuKu6f8A455wLvFtVAVZJW1d/Pge41fYXgbOALaa2qRERNbiZotuSZgPHAq8CNgX2k7TpqHM2Av4T2Nn2ZsB7JmvedHTwnwE+JWku48+hf4IydXOVpGuqxwD7AtdUVZ42Ab491Y2NiKilmZp92wE3277V9iPAKcBeo855G3Cs7fvK2/oPkwXtaQ7e9u3A5m2PDxjnuee2vewj1fPnA+dX9xcBbx8j/qeAT/XSxoiIqdPYGvdnAXe2PV4IbD/qnOcCtA2Wj7D9k4mCZh18REQPRjqvt7qWpHltj4+zfVx1X2OcPzrwcsBGwK7AOsAcSZvbvn+8N0wHHxHRJVdz8B26Z4LFIAuBddserwP8doxzLrb9KHCbpBspHf5l471h31IVREQMg4ZW0VwGbCRpg2rPzxsoC0vanQG8BEDSWpQpm1snCpoRfERED5qYg682gh5CWVE4GzjB9rWSjgTm2T6rem43SdcBS4D32/7TRHHTwS8Dbr758p5j3LLglgZaAq94zehNyN1pqqKT1MyXUPdrL3ks45pLJGb7HOCcUcc+2nbfwPuqW0fSwUdEdCsFPyIihpMBL0kHHxExlDKCj4gYRn0s5tGJdPARET2osQ5+2g1kBy9ptu0l/W5HRMRkBnkE35eNTpLOkDS/Snl5cHXsQUlHSroE2FHSNpIuqM47V9Iz+tHWiIjxNJkueCr0awR/oO17Ja0MXCbph8CqwDW2PyppeeACYC/bf5S0L/BJ4MA+tTci4olsPMAFP/rVwR8q6bXV/XUp+RSWAD+sjm1MyUT50ypF/Gzg7rECpaJTRPTTIO+Rm/YOXtKuwMuBHW3/TdL5lCpPD7XNuwu41vaOk8WrsrEdV8Ue3MmwiBhKmYNf2hrAfVXnvgmwwxjn3AisLWlHAEnLS9psOhsZETEpD/YcfD86+J8Ay0m6ilK56eLRJ1QVTfYBjpJ0JbAA2GlaWxkRMYlcZB3F9sOUuoOjrTbqvAXAi6alURERXTEjSwZ3En4g18FHRCwTkmwsImKIpYOPiBhOA9y/p4NfFjRRjOJ73zi6gZbA4Z8/ppE4yy+/YiNxNtpovBKX9dx44yWNxFmyZHEjcVZZ5UmNxFm06MGeY6QYyvhaF1kHVTr4iIhu1Su6Pe3SwUdEdM2MJFVBRMRwyhRNRMSwSgcfETF8POBz8I2nKpB0gKRnTnLOOyS9eYzj60u6puk2RURMFbuzWz9MxQj+AOAa4LfjnWD7q1PwvhER02wIarJKeh+PF9s4HjgDONv25tXzh1FyyVwDbAucLGkRsCPwMWBPYDFwnu3DJB0BPGj7c5K2AU4A/gZc1Paes4FPA7sCKwLH2v5aTz9tRESTzLK9iqbqgN8KbE/J034JpdrSE9g+TdIhwGG250l6CvBaYBPblrTmGC/7JvBu2xdI+mzb8YOAB2y/QNKKwFxJ59m+rdZPGBExRcyyPwe/C3C67b/afhD4EfDCDuP/GXgIOF7S6yij9MdIWgNY03brA+Oktqd3A94saQHlQ+WplMpPS5F0sKR5kuZ12KaIiMYs6+mCNcaxNVn6w2GlsV5oe7Gk7YCXAW8ADgFeOir2eD+5KCP7cydqXCo6RUT/9PEKagc6GcFfCLxG0iqSVqVMufwY+F+SnlpNn/xj2/l/AVYHkLQasIbtc4D3AFu1B7Z9P/CApF2qQ/u3PX0u8K9VAW4kPbd6/4iIwTDgFZ0mHcHbvlzSt4BLq0PH275M0pGUqZPbgBvaXvIt4KvVRdZXAWdKWokyIn/vGG/xVuAESX+jdOotxwPrA5erVN7+I/Cazn+0iIipN7JkcEfwHa2isX00cPSoY18EvjjGuT8Efth2aLsxzjmi7f58YMu2p49w2k1RAAAMsUlEQVSojo8AH6xuEREDJ9kkIyKGVSo6RUQMqyHY6BQREWNLBx99d++9v2skzpHvfnsjcWbNmt1InEceeaiROKuuskYjcRY91HsFJYD9D/qPRuJc9LOze45x/fUXN9ASGH9F9LJtWd/oFBERY2hlk+zkNhlJu0u6UdLNkg6f4Lx9JFnSpPUq08FHRPSgiXXwVe6tYylLyzcF9pO06RjnrQ4cSlmiPql08BERXeusc+9gnn474Gbbt9p+BDgF2GuM8z4BfIaSAmZS6eAjIrrV3BTNs4A72x4vrI49RtLWwLq2O76wkousERE9qLGKZq1RSRGPq3Jpwdg5vx4LLGkW8AVKvY2OpYOPiOhSzZ2s99ge78LoQmDdtsfrsHTRpNWBzYHzS+YWng6cJWlP2+Nm0l2mOnhJy9le3O92REQUxs0U/LgM2EjSBsBdlOy7b3zsXewHgLVajyWdT1V3Y6KgUzYHX9VXvV7S1yVdK+k8SStL2lDSTyTNlzRH0ibV+a+WdImkKyT9TNLTquNHSDpO0nnAt6eqvRERtRk80tltwjBl4HoIJeHi9cCptq+VdKSkPbtt3lSP4DcC9rP9NkmnAntTske+w/ZNkrYHvkzJEX8RsENV+elfgA8A/17F2QbYxfaiKW5vREQtTe1krdKqnzPq2EfHOXfXTmJOdQd/m+0F1f35lPS/OwE/qOaRoNRbhTLn9H1JzwBWoKQhbjlrvM5d0sHAwQ23OyKiIzM5VcHDbfeXAE8D7re91Rjnfgk42vZZknalShtc+et4b5CKThHRL4OeLni618H/GbhN0usBVLRywa9BubgA8JZpbldERH02I0tGOrr1Qz82Ou0PHCTpSuBaHt+tdQRl6mYOcE8f2hURUZ/d2a0PpmyKxvbtlHWbrcefa3t69zHOPxM4c4zjR0xB8yIiGuEBzpK5TK2Dj4gYJE5Fp4iIYWU82SL3PkoHHxHRg4zgYwA084/wwQfvayTO7NnN/NP7zW+ubSTO4sWPNhKnqZ/rgT8+0Eic3V77+p5j3HLLFQ20pLnqW4NmpJlUBVMiHXxERJdKrvd08BERwylTNBERwynLJCMihlQuskZEDCUzMrKk340YV0epCqrc7tf08kaSdpXUcS3BiIhB19ro1EDR7SmREXxERA8GeYqmTrKx5SSdKOkqSadJWkXS7ZLWApC0bVVGCkkvlrSgul0hafUqxmrVa2+QdLKqpPCStpF0QVXl6dwqJzySDpV0XfWepzT4c0dENGJYRvAbAwfZnivpBOCdE5x7GPCu6tzVgNYOh62BzSjFZOcCO0u6hJILfi/bf5S0L/BJ4EDgcGAD2w9LWrPWTxYRMeX6lymyE3U6+Dttz63ufwc4dIJz5wJHSzoZ+JHthdVg/VLbCwEkLaBUeLqfknXyp9U5s4G7qzhXASdLOgM4Y6w3SkWniOgnMxwbnUZ/TBlYzOPTPCs99oT9aUn/A+wBXCzp5dVToys8LQcIuNb2jmO85z8ALwL2BD4iabOqOO3jjUhFp4joE3uwUxXUmYNfT1KrE96PUiT7dkpBbCgFtQGQtKHtq20fBcwDNpkg7o3A2q3YkpaXtJmkWcC6tn9JKcC9JrBajfZGREyxzubf+zUHX6eDvx54i6SrgKcAXwE+DhxTVWFqXwz6HknXVFWbFgE/Hi+o7UeAfYCjqvMXUApzzwa+I+lq4ArgC7bvr9HeiIgpZ490dOuHjqZoqupMm47x1BzguWOc/+4xzj2/urXOOaTt/gLKVMxou3TSvoiIfhnkZZJZBx8R0YN08BERw6iPBbU7kQ4+IqJLBkY8uLlo0sFHXzS1tExqJk5TCaOa+rq+w6t3aCTO/5xwes8xlixZPPlJM1b/Vsh0Ih18REQP0sFHRAypdPAREUOoXGMd3J2s6eAjIrpmPMCpCtLBR0T0IDVZIyKGVObgGyJpudHZJCMi+scDPQdfJ9lYLVUd1+slfV3StZLOk7SypA0l/aSq3jRH0ibV+a+WdElVAepnkp5WHT9C0nGSzgO+PVXtjYioq8marJJ2l3SjpJslHT7G8+9rq3D3c0nPnizmlHXwlY2AY21vRinssTcld/u7bW9Dqfz05erci4AdbG8NnEJJEdyyDaXi0xunuL0REbU00cFLmg0cC7yKkthxP0mjEzxeAWxrewvgNOAzk7VtqqdobqsyRQLMp1Rw2gn4QVW9CWDF6s91gO9X9VhXAG5ri3OW7UVjvUEqOkVEPzW0K3s74GbbtwJUNaj3Aq5rnVDVxmi5GHjTZEGnuoMfXcHpacD9trca49wvAUfbPkvSrsARbc/9dbw3SEWniOgfQzNz8M8C7mx7vBDYfoLzD2KCOhstUz1FM9qfgdskvR5AxZbVc2sAd1X33zLN7YqI6Io7/A9YS9K8tlv7zIPGDD0GSW8CtgU+O1nb+rGKZn/gK5I+DCxPmW+/kjJi/4GkuyhfPzboQ9siIjrWusjaoXtsbzvOcwuBddserwP8dvRJVX3rDwEvtv3w6OdHm7IOvqoCtXnb48+1Pb37GOefCZw5xvEjpqB5ERGNaGgd/GXARpI2oMxkvAFYalGJpK2BrwG72/5DJ0GXqXXwERGDpZl18LYXSzoEOJdSj/oE29dKOhKYZ/ssypTMajy+SOU3tvecKG46+IiIHjRV28D2OcA5o459tO3+y+vGTAcfEdGlmnPw0y4dfNQ01sX++pZbbvlG4qyyypMaifPgg/c1EmfWrNmNxPnO57/aSJxdXvmEy1213XjDJQ20BBbe9f8aiTNYUpM1ImJomcHNRZMOPiKiB5miiYgYSm7sIutUSAcfEdGlQS/Z11Gqgir17zW9vJGkXSWd3UuMiIhB01S64KmQEXxERA8GeQ6+TrKx5SSdWCWbP03SKpJul7QWgKRtJZ1f3X+xpAXV7QpJq1cxVqtee4Okk1Vtx5K0jaQLqiIg51Ypg5F0aFuC+1Ma/LkjIhrg1jzN5Lc+qDOC3xg4yPZcSScA75zg3MOAd1XnrgY8VB3fGtiMkkRnLrCzpEsoqYL3sv1HSfsCnwQOBA4HNrD9sKQ1a/1kERHTYFiKbt9pe251/zvAoROcOxc4WtLJwI9sL6wG65faXgggaQGlAMj9lKRkP63OmQ3cXcW5CjhZ0hnAGWO9UQp+RES/2DAysqTfzRhXnQ5+9MeUgcU8Ps2z0mNP2J+W9D/AHsDFVYpLeGIBkOUoWyOvtb3jGO/5D8CLgD2Bj0jabHTR7RT8iIj+6d8F1E7UmYNfT1KrE96PUkP1dkq9VCj1VgGQtKHtq20fBcwDNpkg7o3A2q3YkpaXtJmkWcC6VZmqDwBrUjKpRUQMjEFeRVOng78eeIukq4CnAF8BPg4cI2kOZUTe8h5J10i6EljEBKWlbD8C7AMcVZ2/gFK3dTbwHUlXU4rNfsH2/TXaGxEx5Qa5g+9oiqYq3jG6wjfAHOC5Y5z/7jHOPb+6tc45pO3+AspUzGi7dNK+iIh+GeSNTlkHHxHRrT4ugexEOviIiC4ZGMkIPiJiOGWKJiJiKA32Msl08FFTM/+Ym/qlWLz4kUbiNNWepuJstd3OjcS5+9a7Jz9pEn9pqNrVsEoHHxExhFKTNSJiaBkPSaqCiIgYZViSjUVExCiZoomIGFLp4CMihlDJM5N18BERQykj+IiIITUykhH8lElFp4joq4zgp04qOkVE/xiTEXxExNAZ9J2sdSo69ZWkcyQ9s9/tiIhot8xXdBoEtvfodxsiIkYb5BH8MtPBR0QMHjOSXDQREcMnc/AREcOsVZd1stskJO0u6UZJN0s6fIznV5T0/er5SyStP1nMdPAREV1zx/9NRNJs4FjgVcCmwH6SNh112kHAfbb/DvgCcNRkrdMgf72oS9IfgTsmOW0t4J4G3m4Y4wxSWxJneuIMUlumO86zba/dy5tI8qxZnY2TR0ZG5tvedpw4OwJH2H5l9fg/AWx/qu2cc6tzfi1pOeB3wNqeoBMfqjn4Tv6yJM0b739yHcMYZ5DakjjTE2eQ2jKIcTrRUKqCZwF3tj1eCGw/3jm2F0t6AHgqE3yQDVUHHxExzc6lfFvoxEqS5rU9Pq7aiQ+gMc4fPTLv5JylpIOPiOiS7d0bCrUQWLft8TrAb8c5Z2E1RbMGcO9EQWfiRdbjJj9lxsYZpLYkzvTEGaS2DGKc6XIZsJGkDSStALwBOGvUOWcBb6nu7wP8YqL5dxiyi6wREcsqSXsA/xuYDZxg+5OSjgTm2T5L0krAScDWlJH7G2zfOmHMdPAREcNpJk7RRETMCOngIyKGVDr4iIghlQ4+ImJIpYOPiBhS6eAjIoZUOviIiCGVDj4iYkj9fyl5hzaCZs2oAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "translation, attention = translate_sentence(candidate)\n",
    "\n",
    "print(translation)\n",
    "\n",
    "display_attention(candidate, translation, attention)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eine gruppe von männern lädt baumwolle auf einen lastwagen\n",
      "a group of men are loading cotton onto a truck\n"
     ]
    }
   ],
   "source": [
    "candidate = ' '.join(vars(valid_data.examples[0])['src'])\n",
    "candidate_translation = ' '.join(vars(valid_data.examples[0])['trg'])\n",
    "\n",
    "print(candidate)\n",
    "print(candidate_translation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['a', 'group', 'of', 'men', 'loading', 'loading', 'a', 'a', '.']\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAErCAYAAADKTqhMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XmcXHWd7vHPQwADgjhj4sbqKMIggiyCDCgwohMcAVlkEWYGVHLVUdy9LjMKjDgqLuPCOAZE1CvgBhi9CArKpgJJ2APkiiwaUSCKgAQDST/3j3OKVNpeqrrP6ao6/bx51aurTp36nh/p6m/96rfKNhER0Rxr9boAERFRrST2iIiGSWKPiGiYJPaIiIZJYo+IaJgk9oiIhklij4homCT2iIiGSWKPiGiYJPaILkl6mqQvSfpB+XgbSa/rdbkiWpLYI7p3BnAh8Mzy8f8D3taz0kQMk8Qe0b1Ztr8JDAHYXgms6m2RIlZbu9cFiBhAD0t6CmAASS8CHuhtkfqPpL8DtqAtz9j+as8KNI0ksUd07x3AfODZkn4KzAYO6W2R+oukrwHPBq5j9bcZA0nsU0BZtjeie5LWBrYCBCyx/ViPi9RXJN0CbOMkmJ5IjT2iQ5IOGuWp50rC9jlTWqD+dhPwdOC3vS7IdJTEHtG5/cZ4zkAS+2qzgJslXQ2saB20vX/vijR9pCkmIionac+Rjtu+dKrLMh0lsUdPSNoY2Jw1R0xc1rsSjU/SO8Z63vanpqosg0DS5sCWti+StD4ww/ZDvS7XdJCmmJhykj4GHAbczJojJvo6sQMb9roAg0LSscBc4K8pRsdsDPwP8NJelmu6SI09ppykJcB2tleMe3IMJEnXAbsAV9neoTx2o+3n97Zk00NmnkYv3A6s0+tCTJSkTSSdK+leSfdI+o6kTXpdrj6zwvajrQfl8NDUIqdImmKiF5YD10m6mDVHTBzXuyJ15cvAmcCry8dHlcde1rMS9Z9LJb0fWE/Sy4A3Ad/rcZmmjTTFxJST9C8jHbf9lakuy0RIus72C8Y7Np1JWgt4HfByiklcFwKnZcLS1EiNPaaUpBnAy2wf1euyTMIySUcBZ5WPjwB+38Py9B3bQ8Cp5S2mWBJ7TCnbqyTNlrRuexvsgHkt8Hng0xTtxj8rj0VJ0o38ZZv6A8BC4MO280FYozTFxJST9EVgR4qFtB5uHR+UceCSZtr+c6/L0c8kfZxiKOuZ5aHDy58PAnvYHmsWb0xSauzRC3eXt7UYzLHhN0m6B7icYuz9T21n2d417W5797bHN0r6qe3dy2asqFES+4CR9FzgC8DTbG8raTtgf9sf7nHROmb7BABJT7T98Hjn9xvbz5G0GfBi4JXAf0v6YzpP17CBpF1tXwUgaRdgg/K5lb0r1vSQceyD51TgfcBjALZvYPXX3IEgaTdJNwO3lI+3l/TfPS5Wx8ox67tTJPYdgMXAN3paqP7zeuA0SXdIuhM4DThW0hOB/+xpyaaBtLEPGEkLbL9Q0rVtM/oGaqidpKsoNqaY3/b/cJPtbXtbss5IGgIWAB+x/d1el6efSdqIIs/8sYbY61D8Ho61vaDq+INsWjbFlAsS3Q0cbPviXpenS8skPZvV27IdwgCueW3715LaDw3SnqE7AHsAr5H0XuAXwKW2v9TbYvUXSf8IPA+Y2fpd2z6xwkscAKwLHEuR4KM0LRM7cCjF1+fXA4OW2P8VmAdsLek3wB3Akb0tUtd+Xe6HaUnrAsdRNssMAtvXS/ol8EuK5pijgJcASewlSf8DrA/sTdEMcwhwdcWXeR3FMNOvSlrf9vKK4w+sadkUI+ky4H9RbIzwd7bv73GRula2Va41iMugSpoFfAbYh2JW4g+Btw7K2GZJC4EnUIxfvwK4zPZdvS1Vf5F0g+3t2n5uAJxj++UVxd8UOM/2TuXQyltsf7mK2E0w7WrskramSIi3SDqLorb1uR4Xq2OSngJ8iKIpwJKuAE4clKQIYHsZg/cto92+tu/rdSH63CPlz+WSnkkxM/dZFcY/htUbY3+ZYlBBEntp2iV2iq9vrTfAGcB5DFBiB86mGDt9cPn4SIoRGfv0rERdkjSbol10C9bcaGNQZm8+Juk4/rL8g7KI2VT4vqQnAycD11D0CVWyvICKBvujgBcBlJW0GZK2sr2kimsMumnVFFP2oi8Bnt8aPy3pR8D7bC/saeE6JGmR7Z2GHVtoe+delalbkn5GMblnEW2dpra/07NCdaEs/5XAjcBQ6/gALWJ2EPAx4KkUTWECbPtJNV3vCcDMqiZxSXoSxezV89uO7QA8YvvWKq4x6KZbYl8feK7t69qObQ6ssr20dyXrnKRPUKy38c3y0CHA82x/qHel6s6gDc8cTtI1tnfsdTkmStJtwH62a+uwltSalXs5xczc2vqCypUkN7D9YF3XGDTTKrEPJ+mvgE3LST4DQdJDwBNZXVNci9XrrdRW66qSpA8DP2uvcQ0SSW8H/gR8nzXXk/9DzwrVhdbU/pqv8TcU/UAvpmgyWQFcbvvtFcU/E3gDxTe+RcBGwKdsn1xF/EE37RK7pEuA/SnaRq8D7qMYgzzmRsVRnbYPpxUUM2hrbQqomqR/BU4C/sjqFQxt+296V6rOSfoM8HSK/qX2D6ZzKr7OM4A9KZL73sCvbM+pKPZ1tl8g6UhgJ+B/A4tsb1dF/EE3HTtPN7L9oKTXA1+2/SFJA1Njh8fbSPegSCqX2z6vx0Xqiu1BXPir3TuA55SjewbRkyh2sWofemiK4b+VKMf5L6NY3fFLwFvKNdqrsk7ZZ/Yq4PO2H5M0vWqpY5iOiX3tsiZxKPCBXhemW+WaKs9h9SYPb5D0Mtv/2sNidU3SxsDmrDmq5LLelagriykS40CyfcwUXOazFJWPIyhm6l4q6TLbv6wo/heBO4HrgcvKvrK0sZemY1PMq4F/p+jQeWPZFniy7YPHeWmn8WtdfVHSYmDb1hZjZcfRjbafV0X8qSDpY8BhwM2sHhVj2/v3rlSdk3QuxVT5nzCAe7ZO5Qqh5cSkY4B3AZvYnlH1NdqutbbtrBzJNEzsdZN0KfBu4It1LHAl6Rzg7a2ZjmVN5aO2j6gi/lSQtATYzvaKcU+eWPwnDI890rFJxB/0PVtrfY+W8T5JUWPfAPg5xeiYy23fXlH8jSgm6r2kPHQpxUS9rIvPNGyKKZdc/RzFsqummBL+1gqHO65v++phC1xVWYt4CnCLpNa6Gy8Efi5pPsCA1HpvB9ahrbZbsZ9T7NA03rEJGZQEPoa636NQjPP/uO17Ko7bcjpwE0WTKsA/UUw8PKim6w2UaZfYKX75ZwKvLh8fVR57WUXx61598YMVxhpRuUDXFqzZ/v3VUV/QveXAdZIupsKmDElPBzYG1isnrLQy15MoFqSqhKQ7+Mv9PBmUUTFMzQqhb7D9rfYDki62/dKK4j97WPPpCZKuG/XsaWY6JvbZwxYLOkPS2yqMP9Lqi5VtBWb70qpijUTS14BnUwwFfbz9m9XrclRhfnmr2j8ARwObAO37pz4EvL/C67TP8p1JUUn468kGlbS77Z9W2Ww0itreo5JmUnyIzirnibR/uD6zimuUHpG0h+0ryuvuzur1aaa9adfGLukiijViWqNKjgCOqbAm0bpOLasvlmPAW7+0dSmaNB6uagy4pFuAbTzAbwxJB0/18gSSrrC9xyRjLCpXK5ySma11vEclvRV4G0US/w2rE/uDwKm2P1/RdV4AfIViYhLA/cDRtq+vIv6gm46JfTPg88Bu5aGfUrSxV7LsarkuxsH8ZVNGlRsMtF/vVcAutiupkUr6FnCc7do275C0JcX2aNtQ1HiByTdlSBpzkpntT431fBfXaU+6a1HU4N9oe/tJxr2SYl36VzDCVntVjbqZiveopLfYrn1xvXLdGLKcwJqmXVOM7V9RzDyty3eBByimOdf5dRoA2+ep2MWnKrOAm8vO2fb270n/m7XVar9MMaLh0xQzEo9hdc1uMloTn7ai6FRuNffsR7FuSVU+2XZ/JcV46kNHPrUrr6RYpfPvKd4/dZmK9+jvJG1o+yFJ/0bRcf1h29dUEVzS04CPAM+0va+kbYDdnF2sgOlZY/848GGK9rgLgO2Bt9n+PxXFr3XvznLWaUurtrin7d1GeUm38fcc6XgVbfuSnlTO+m01Odxo+/nlc5fbfvFkr1HG+iHFtocPlY83BL5V1XT2uknavs4mhbrfo+U1Whts7EHx7ewTwPtt71pR/B9QVBA+YHt7SWsD17beT9PdtKuxAy+3/R5JBwJLKTq+fgJUktiBn0l6vu0bK4o33H5t91u1xQOqCm770rI29MLy0NW2760o/JkUtdIVKsba/ULSmynaYp9a0TUANgMebXv8KEWzQ2XUtp9n61iFTRlvG2l6vKtbr77u9yis7nj/R+ALtr8r6fgK48+y/U1J7wOwvVLSIO2bW6vpmNjXKX++AjjL9h+GjeedrD2Ao8shcStYvcDVpBcnkjQDuMH2pycba4xrHEqxOcIlFGX/nKR32/72ZGPbfmV59+0UE1eOA/6Dojnmnycbv83XgKvLGaIGDqTCUT2qfz/P77fdn0lR/rsrjF/be7TNbyR9kaJp6WNlu/5aFcZ/WMVuYq0hmy+iaF4KpmdTzEcpFg56BNgFeDLw/Qq/Im4+0vEKO2d/YnvvKmKNEv964GWtWrqK3Y4ummzH4LBr7EyxTs/mrP6grTSxSNqJIoFBsSfptRXGrnU/zxGutxbF7+DvK4pX63u0vMb6wByK5S5+oWJ9pufb/mFF8XekmGi4LcVEpdnAIR6gJbjrNO0SOzy+DvuDtleVQ742tP27ScZstR+POJ7ZFa3VLekkiiFe32D1OuxU2Cn1eLt3+Xgt4Poq2y7LJQXezV/uQFTphtCSnsqaTSW/qijuVbZ3LUexHESxn+dNtresIv4I19sK+L+2nzPJOFPyHh12zVp+B2XstSk6ygUssf1YVbEH3bRqiilrEVsO65h6Cm3bs01Cq/14EavHmbfaeAxUNSvx78qfJ7RdwxQjKarwA0kXsnqc/2FA1Rti3Ge78glKkna0fY2k/SlGrjwTuJeizf1WijbxKrT28/w4q0evnFZR7OFzFQzcA7yngtBT9R6lzt/BsL/jxeWxzSStsv2bycZvgmlVY1exfvOtFAtQtfY8/SFFb30le56WNdwjgWfZPrEcN/8M21dVFP+dFH+E7X+QDwIL3bbl3yTiv4Ni85EXlNe43Pa5k4077BovpZgYNnxJgUmtBy7pv2y/TcXU8pdSNF/sIGlv4AjbcycTv+066wFvpNhAwhQLXH3B9p+riF9e46+BLVld27UrWta47vdoeY3rKSoblf8OpuLveNBV2ZnR98qvaudS1EJbk5VmV/xmOIViK7DWaosPUUyIqspOFFuCPYOiNjSXYpeaUyVVUavbEHgvRf/DL4GfVRBzuGMoPjjmUIzy2Y+iJjlZrW8BK23/HlhL0lq2f1Jerypfoah5fpainfdvqbZz9vUUqxVeABzf9rMqdb9HAR6r63cwRX/HA21aNcWUTgNOpVgd7p8pxsJWaVfbO0q6FsD2/ZLWrTD+U4Adbf8JQNKHgG9TLF+6iKJ5YMJsn0CxoNJ2FH84l0paanufyRV7DdvXNN74JcCPgQfKvpPLgK9LupdqVy/calhn8k/KGmpV3kox3PRK23tL2prVTW9VqPs9CvDHslO5rt9B3X/HA21a1dgBbN8KtDYbOIJiaFyVHiuHJbaGYc2mrYOwAsPHaD8GbG77EaqdRXgv8DuKjsEqx5gDXFnOFKyU7ePLu6+i+Ld4O0Vt95esOf5/sq4th9cBIGlXiqUpqvLnVrOOigXBbqXoJKxK3e9RKOZWPEJNv4Mp+DseaNOxxg7FHoynUYwJv7/i2J+l+Jr41HIEyyHAv1UY/0yKxPjd8vF+wFllDfXmyQaX9EaKmvpsim8Cx9qedNxh9gD+pa5x1F5zUavK1k6XdCNFMlwH+GdJvyofb04F//Ztlpads+cBP5J0P9WOY6/7PUqr7btU1/r1df4dD7Rp1XnaUvaq/5Zi2vlFNcTfmqLzTsDFtm+pOH5rjLaAK6psWyzH+Z9dRUfsGNeoZRz1sNEkazxVhJ/cCpijlbul6uGa5TX3pBjeeoHtR8c7v4u4tbxH6/4dDLtWrX/Hg2xaJvaIiCabdm3sERFNN60Tu6RKxjX3Kv5UXCPxmx1/Kq4x6PEH0bRO7BRjwAc5/lRcI/GbHX8qrjHo8QfOdE/sERGN06jOU42whvUg2Wmnnbp+zX333cfs2bM7Pn/Rojo35okYCMtsd/5HM4I5c+Z42bJlHZ27aNGiCz3Fm7xM13Hsfemqq6tc0ntk66y9zvgnTYJd9TyXiMpNeljqsmXLWLiws1HGkmZN9nrdSmKPiJiAfm7tSGKPiOiSgVVD/fvtNIk9IqJrxiNOsO0PSewREd0yDPVvXk9ij4iYiLSxR0Q0iIGhJPaIiGZJjT0iokFsZ1RMRETT9HONve/XipF0nqRFkhZnFbeI6Bfu8L9eGIQa+2tt/0HSesACSd8pdz8HHl+yMwk/IqZM0Xna61KMbhAS+3GSDizvbwpsSbHBMgC25wHzYPAXAYuIwdHPTTF9ndgl7QXsA+xme7mkS4CZPS1UREQ6TydlI+D+MqlvDbyo1wWKiDCpsU/GBcAbJN0ALAGu7HF5IiKATFCaMNsrgH17XY6IiOFSY4+IaJSs7hgR0SjO6o4REc0zlFExERHNkdUdIyIaKJ2nERFNYqfGHp2ZsVb9a7LZ/dsuGDFIUmOPiGgQA6uS2CMimiU19oiIhklij4hoEKfzNCKieVJjj4homCT2iIgGKUbF9O/Q4ST2iIgJyCJgERFNYqcpRtLatldOxbUiIuo2LbbGk/TvwJHAr4FlwCLglcDPgN2B+ZK+DZwOzAbuA46x/StJZwDft/3tMtafbG9QbmR9IvB7YCvgMuBNzpz4iOgDjR7uKGln4GBghzLeNRSJHeDJtvcsz/se8FXbX5H0WuCzwKvGCb8LsA1wF8X+pwcB355smSMiJqufa+xVrDq1B/Bd24/Yfgj4Xttz32i7vxtwZnn/a+XrxnO17dttrwLOGuk1kuZKWihp4cSKHxHRHdusGhrq6NYLVTTFaIznHh7judbH3UrKDxhJAtYd4ZzRHmN7HjCvfH3/foRGRKP0856nVdTYrwD2kzRT0gbAP45y3s+Aw8v7R5avA7gT2Km8fwCwTttrdpH0LElrAYe1vSYioqeG3NltPJLmSFoi6TZJ7x3h+c0k/UTStZJukPSK8WJOOrHbXgDMB64HzgEWAg+McOpxwDGSbgD+CXhrefxUYE9JVwO7smYt/+fAR4GbgDuAcydb3oiIyWqNiunkNhZJM4BTgH0p+hOPkLTNsNP+Dfim7R0oKsf/PV75qhru+Anbx0tan2L0yidtn9p+gu07gb8f/kLb9wAvajv0vrb7y20fVlEZIyIqU1Hn6S7AbbZvB5B0NkXLxc3tlwKeVN7fCLh7vKBVJfZ55afMTOArtq+pKG5ERP8pO08rsDHFMPGWpRQtF+2OB34o6S3AE4F9xgtaSWK3/Zoq4gyLeQlwSdVxIyImq8sJSrOGjdqbVw76gJEHnwwPfARwhu1PStoN+Jqkbcea05MlBSIiJqCLCUrLbO88ynNLgU3bHm/CXza1vA6YA2D755JmArOAe0e7YP27J0dENJA7/G8cC4Aty9F/61J0js4fds6vgJcCSPpbiibv+8YKmhp7RMQEVNF3anulpDcDFwIzgNNtL5Z0IrDQ9nzgncCpkt5O0UxztMdpB0pij4jokqlurRjb5wPnDzv2wbb7N1OsudWxJPY+8uQnP7X2axx6+Htqjb/p1puOf9IkfPakd9Ua/7HHVtQaPxqiulExtUhij4jo0rRYtjciYrpJYo+IaJhGr8ceETH9dDSUsWeS2CMiumRXM9yxLknsERETkFExERENUuU49joksUdETEA/j4oZiLViJB0n6RZJX+91WSIi6HCTjV4l/0Gpsb8J2Nf2Hb0uSEQE0Ne9p32X2CW9A3ht+fA0YGvgb4D5kk63/emeFS4iojS0Kom9I5J2Ao6h2EFEwFXAURRrEe9te1kPixcRAbSGOyaxd2oP4FzbDwNIOgd48VgvkDQXmDsFZYuIeFwSe+dG2iZqTOUWU/MAJPXvv3RENEjvOkY70W+jYi4DXiVpfUlPBA4ELu9xmSIi/oKH3NGtF/qqxm77GklnAFeXh06zfa3UdUU+IqI2aWPvku1PAZ8admyL3pQmImJkzpICERHN0scV9iT2iIiuuXft551IYo+ImIC0sUdENEj2PI2IaKAk9oiIJrHxqoyKiYholNTYoyMPPHBf7df41jc+UWv8Pz2yvNb4n/nwO2uNH9GpPs7rSewREd1K52lERNNkSYGIiKYxQ+k8jYholtTYIyIaJKs7RkQ0URJ7RESzuH+b2JPYIyImIk0xERFNYjPUxxtt1LrnqaQtJN0q6TRJN0n6uqR9JP1U0i8k7SLpiZJOl7RA0rWSDihfe7SkcyRdUJ778TrLGhHRqdYEpU5u45E0R9ISSbdJeu8o5xwq6WZJiyWdOV7MqaixPwd4NTAXWAC8BtgD2B94P3Az8GPbr5X0ZOBqSReVr30BsAOwAlgi6XO2fz0FZY6IGJ2pZKMNSTOAU4CXAUuBBZLm27657ZwtgfcBu9u+X9JTx4s7FYn9Dts3AkhaDFxs25JuBLYANgH2l/Su8vyZwGbl/YttP1C+9mZgc2CNxC5pLsWHRkTE1KmmjX0X4DbbtwNIOhs4gKLC23IscIrt+4vL+t7xgk5FYl/Rdn+o7fFQef1VwMG2l7S/SNKuw167ihHKa3seMK98Tf/2ZkREg3TWzNKBjVmzsroU2HXYOc8FkPRTYAZwvO0Lxgpaaxt7hy4E3iJJAJJ26HF5IiLGNTTkjm7ALEkL227tLQwaIfTwT4y1gS2BvYAjgNPKZutR9cOomP8A/gu4oUzudwKv7GmJIiLG4O7a2JfZ3nmU55YCm7Y93gS4e4RzrrT9GHCHpCUUiX7BaBesNbHbvhPYtu3x0aM8979GeO0ZwBltj5PsI6JvVNQUswDYUtKzgN8Ah1MMMGl3HkVN/QxJsyiaZm4fK2g/1NgjIgZOFYnd9kpJb6Zokp4BnG57saQTgYW255fPvbwcQLIKeLft348VN4k9IqJrlXWeYvt84Pxhxz7Ydt/AO8pbR5LYIyK6ldUdIyKaxYBXJbFHRDRKauwREU3S4TowvZLEHhExAVWsFVOXJPZpxjXvDrDh+k+sNf5Dyx+uNf5GGzyp1vgrVz5aa/yYOqmxR0Q0SGvZ3n6VxB4R0S0b9/FGG0nsERETkD1PIyIaJk0xERFNkpmnERHNks7TiIjGMUOr+reRfdwdlCT9qYoLSTq+ta+ppBMl7VNF3IiIKVc2xXRy64We1Njbl6SMiBhIfdwU0/GepyqcLOkmSTdKOqw8voGkiyVdUx4/oO01H5C0RNJFwFZtx8+QdEh5/05JJ7S9fuvy+GxJPyqPf1HSXeXuIRERPWd3duuFbjazPgh4AbA9sA9wsqRnAH8GDrS9I7A38MnyQ2Anim2edihf+8IxYi8rX/8F4F3lsQ8BPy6Pnwts1kVZIyJq0+o87demmG4S+x7AWbZX2b4HuJQiWQv4iKQbgIuAjYGnAS8GzrW93PaDwPwxYp9T/lwEbNF2vbMBbF8A3D/SCyXNbe3+3cX/S0TExJWbWXdy64Vu2tg1yvEjgdnATrYfk3QnMLN8rtP/qxXlz1VtZRrtemuwPQ+YByCpfxu9IqJBzFAfLynQTY39MuAwSTMkzQZeAlwNbATcWyb1vYHN284/UNJ6kjYE9uuybFcAhwJIejnwV12+PiKiNv3cFNNNjf1cYDfgeoqa+Hts/07S14HvlU0h1wG3Ati+RtI3ymN3AZd3WbYTgLPKTtpLgd8CD3UZIyKiHn08KmbcxG57g/KngXeXt/bnl1Ek/JFeexJw0gjHj267v0Xb/YXAXuXDB4B/sL1S0m7A3rZXEBHRY3Y22piozYBvSloLeBQ4tsfliYh4XB9X2Ps3sdv+BcVQyYiIPpM9TyMimsX09aiYJPaIiC6ZtLFHRDROmmIiIhqlhwvBdCCJPSKiW9lBKaaToaFVtcZf/uijtcZfb70Nao3/0EN/qDV+TJ2hVUnsERGNka3xIiKaJk0xERFNkwlKERGNk8QeEdEw/TxBqZv12CMigtWrO1axg5KkOeXe0LdJeu8Y5x0iyZJ2Hi9mEntExARUsdGGpBnAKcC+wDbAEZK2GeG8DYHjgKs6Kdu4iV3SnzoJ1EGc4yW9q7x/oqR9qogbETH1OkvqHbTD7wLcZvt2249S7PN8wAjn/QfwceDPnZSuJzV22x+0fVEvrh0RMWnVNcVsDPy67fHS8tjjJO0AbGr7+50Wr+PErsLJkm6SdGO5ZR2SNpB0saRryuMHtL3mA2Xb0UXAVm3Hz5B0SHn/TkkntL1+6/L4bEk/Ko9/UdJdkmZ1Wt6IiDp1UWOfJWlh221uWxiNFPrxJ4uNhj4NvLObsnUzKuYg4AXA9sAsYIGky4D7gANtP1gm3islzQd2BA6n2CxjbeAaYNEosZfZ3lHSm4B3Aa8HPgT82PZ/SpoDzB3ltRERU6rLmafLbI/W4bkU2LTt8SbA3W2PNwS2BS6RBPB0YL6k/cutREfUTVPMHsBZtlfZvodig+kXUnzifETSDcBFFF8jnga8GDjX9nLbDwLzx4h9TvlzEbBF2/XOBrB9AXD/SC+UNLf1SdjF/0tExCQYDw11dBvHAmBLSc+StC5FZfjxXGn7AduzbG9R7g99JTBmUofuauwjfWUAOBKYDexk+zFJdwIzW+XqMHZrk+pVbWUa7XprsD0PmAcgqX8HlkZEcxhcwQZKtldKejNwITADON32YkknAgttj1UhHlU3NfbLgMMkzZA0G3gJcDWwEXBvmdT3BjZvO/9ASeuVQ3X267JsVwCHAkh6OfBXXb4+IqI2FY2Kwfb5tp9r+9m2TyqPfXCkpG57r/Fq69Bdjf1cYDfgeoqa+Hts/07S14E+cjlSAAAGxUlEQVTvlU0h1wG3lgW4RtI3ymN3AZd3cS2AE4Czyk7aS4HfAg91GSMiohYDvaSA7Q3KnwbeXd7an19GkfBHeu1JwEkjHD+67f4WbfcXAnuVDx8A/qH8qrIbsLftFURE9FiW7Z24zYBvlsN9HgWO7XF5IiIKNkOrKmhkr0nfJnbbv6AYKhkR0X9SY4+IaBZ3POhv6iWxR0R0ydlBKSKiaYyrGMhekyT2iIgJSI09IqJhhsZfLqBnkthjoBz8ytfXGv/fP/s/tcY/+T3vqjX+smVLa40P9HUTxFQpZpX2779DEntExESkKSYiolky3DEiomHSeRoR0ShmaGhVrwsxqiT2iIguZYJSREQDJbFHRDRMEntERKM4wx0jIprGZIJSRERj2P29pEA3m1n3hKTzJC2StFjS3F6XJyKiWN2xms2s6zAINfbX2v6DpPWABZK+Y/v3rSfLZJ+EHxFTKmvFTM5xkg4s728KbAk8nthtzwPmAUjq396MiGiUjIqZIEl7AfsAu9leLukSYGZPCxURQRL7ZGwE3F8m9a2BF/W6QBER5dTTXpdiVP2e2C8A3iDpBmAJcGWPyxMRgYEhZ62YCbG9Ati31+WIiFhT70a8dKKvE3tERL9KYo+IaJgk9oiIBin6TjOOPSKiQYz7eEmBJPaIiAnInqcREQ2TNvaIiEZx2tgjqnLTTZfXGn+7W3apNf7uux9Ua/yLLvpKrfEBli9/sNb4/bxJdEuVe55KmgN8BpgBnGb7o8OefwfwemAlcB/Fwoh3jRWz75ftjYjoR1Us2ytpBnAKxUTMbYAjJG0z7LRrgZ1tbwd8G/j4eGVLYo+ImIChoaGObuPYBbjN9u22HwXOBg5oP8H2T2wvLx9eCWwyXtAk9oiIrhk81NltbBsDv257vLQ8NprXAT8YL2ja2CMiJqCL4Y6zJC1sezyv3EcCQCOGHoGko4CdgT3Hu2ASe0REl7rsPF1me+dRnltKsYFQyybA3cNPkrQP8AFgz3JxxDElsUdETEBFo2IWAFtKehbwG+Bw4DXtJ0jaAfgiMMf2vZ0ETWKPiOhaNePYba+U9GbgQorhjqfbXizpRGCh7fnAycAGwLckAfzK9v5jxU1ij4iYgA5GvHTE9vnA+cOOfbDt/j7dxkxij4joUpUTlOrQ98MdJZ0naZGkxZLm9ro8ERHFcMcObz0wCDX219r+g6T1gAWSvmP7970uVERMbyZrxUzGcZIOLO9vCmwJPJ7Yy1p8avIRMaX6uSmmrxO7pL2AfYDdbC+XdAkws/2ccqD/vPL8/v2XjogGcWWdp3Xo68QObATcXyb1rYEX9bpAERHZGm9yLgDeIOkGYAnFAjgRET2XppgJKqfO7tvrckREDJfEHhHRKL0bytiJJPaIiAnIZtYREQ1i9/cWfknsERFdG3/bu15KYo+ImIAk9oiIhklij4homExQiqhI3dO477r5jlrj/+K2RbXG3/Z5e9QaH+C3v7u91vh33bW41viV6OHKjZ1IYo+I6JKBodTYIyKaJU0xERGNkuGOERGNk8QeEdEg/b7naRJ7RETXjLOkQEREs2QRsIiIhklTTEREwySxR0Q0iO2MY4+IaJrU2GskaS4wt9fliIjppe51iyZj4BO77XnAPABJ/fsRGhHNkhp7RESTGNO/Nfa1el2ATkk6X9Ize12OiIjWzNNObr0wMDV226/odRkiIlrSeRoR0TBJ7BERjWKGslZMRERz9PvqjgPTeRoR0Vda+56OdxuHpDmSlki6TdJ7R3j+CZK+UT5/laQtxouZxB4R0TV3/N9YJM0ATgH2BbYBjpC0zbDTXgfcb/s5wKeBj41XuiT2iIgJsIc6uo1jF+A227fbfhQ4Gzhg2DkHAF8p738beKkkjRU0bewRERNQ0ZICGwO/bnu8FNh1tHNsr5T0APAUYNloQZuW2JcBd3Vx/izG+MepQN3xp+IafRX/j3+8p9b43/v+KbXGn4C8h6qPv3kF17ywvG4nZkpa2PZ4XrkUCsBINe/h7TednLOGRiV227O7OV/SQts711WeuuNPxTUSv9nxp+Iagx5/JLbnVBRqKbBp2+NNgLtHOWeppLWBjYA/jBU0bewREb2zANhS0rMkrQscDswfds584F/K+4cAP/Y4Yy0bVWOPiBgkZZv5mymadmYAp9teLOlEYKHt+cCXgK9Juo2ipn74eHGne2KfN/4pfR1/Kq6R+M2OPxXXGPT4tbJ9PnD+sGMfbLv/Z+DV3cRUP8+eioiI7qWNPSKiYZLYIyIaJok9IqJhktgjIhomiT0iomGS2CMiGiaJPSKiYf4/m9mBNjFNDJsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "translation, attention = translate_sentence(candidate)\n",
    "\n",
    "print(translation)\n",
    "\n",
    "display_attention(candidate, translation, attention)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ein mann mit einem orangefarbenen hut , der etwas anstarrt .\n",
      "a man in an orange hat starring at something .\n"
     ]
    }
   ],
   "source": [
    "candidate = ' '.join(vars(test_data.examples[0])['src'])\n",
    "candidate_translation = ' '.join(vars(test_data.examples[0])['trg'])\n",
    "\n",
    "print(candidate)\n",
    "print(candidate_translation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['a', 'man', 'wearing', 'an', 'orange', 'hat', 'is', 'something', 'something', 'something', '.']\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAFFCAYAAAAAUD2RAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XmcJFWZ7vHfYwOyb4KKgDRiI7bsOwqKggx4FfGKAsqMLOJVZLgO44KOguA4Kjp6dcQFkEVcUBnE1kFAURZZpLvZQRBk0VZntJWlAWmaruf+EVGQXdSSlRVVEVnxfPsTn8qMjDz5ZlfVW5EnzjmvbBMREe3wjLoDiIiIqZOkHxHRIkn6EREtkqQfEdEiSfoRES2SpB8R0SJJ+hERLZKkHxHRIsvVHUDEeEl6KTCTjp9f21+vLaCIPpKkH31F0tnAJsANwNJyt4Ek/YguKMswRD+R9CtgtvODG9GT9OlHv7kFeG7dQUT0q3TvRL9ZB7hN0rXA4sGdtvetL6SI/pGkH/3mo3UHENHP0qcffUfSRsAs2z+VtDIww/aiuuOK6Afp04++IukI4Fzgq+Wu9YHz64soor8k6Ue/eTfwMuAhANt3As+uNaKIPpKkH/1mse3HB+9IWo5inH5EdCFJP/rNZZI+BKwk6dXA94Af1hxTRN/IhdzoK5KeARwO7AUIuAg4LZO1IrqTpB8R0SIZpx99RdLLKMbqb0Tx8yvAtl9QZ1wR/SJn+tFXJN0O/BMwn6cWXMP2X2oLKqKP5Ew/+s2Dtn9cdxAR/Spn+tFXJH0SmAGcx7Jr71xXW1ARfSRJP/qKpJ8Ps9u2XzXlwUT0oST9iIgWyeSs6CuSniPpa5J+XN6fLenwuuOK6BdJ+tFvzqSYkPW88v6vgffUFk1En0nSj36zju3vAgMAtp+gY+hmRIwuST/6zSOSnkW5yJqknYEH6w0pon9knH70m2OAOcAmkq4E1gX2rzekiP6R0TvRd8rllF9EsQTDHbaX1BxSRN9I0o++ImlF4EhgV4ouniuAr9h+rNbAIvpEkn70FUnfBRYB3yh3HQSsZftN9UUV0T+S9KOvSLrR9lZj7YuI4WX0TvSb68sROwBI2gm4ssZ4IvpKzvSjL0i6maIPf3mKi7i/Le9vBNxme/Maw4voG0n60RckbTTa47bvm6pYIvpZkn70jbI+7k05q4/oXfr0o2/YHgBulPT8umOJ6FeZkRv9Zj3gVknXAo8M7rS9b30hRfSPJP3oNyfUHUBEP0uffkREi6RPP/qKpJ0lzZX0sKTHJS2V9FDdcUX0iyT96DdfpFh64U5gJeDt5b6I6EL69KPv2L5L0gzbS4EzJF1Vd0wR/SJJP/rNo5JWAG6QdBLwR2CVmmOK6Bvp3ol+8/cUP7dHUQzZ3BB4Y60RtZSkT3WzL5olo3eiL0i6xPYekj5l+wN1xxMg6Trb2w7Zd5PtLeuKKcaW7p3oF+tJegWwr6RzKKpmPcn2dfWE1T6S3kVRyGYTSTd1PLQaWfG08XKmH31B0v7A4RQVs+YNedi2XzX1UbWTpDWAtYBPAMd2PLTI9l/riSq6laQffUXSR2x/rO442i6L3/WvXMiNfvNxSQdLOg5A0vMl7Vh3UG2Txe/6V870o69I+jIwALzK9oslrQVcbHuHmkNrHUk/A3YAsvhdH8mF3Og3O9neVtL1ALbvL8ftx9Rr/OJ3kpYH5gJH2J5bdzxNkKQ/DpJWBv4AvNH2JXXH01JLJM2gKJWIpHUpzvxjCpXfg4/Y3rPuWMbwemAF4AiK5N96Sfrj82bgVor1XpL0x0HSlsBMOn7mbJ/XQ1NfAL4PPFvSx4H9gQ9XEWN0z/ZSSY9KWsP2g3XHM4rDgcOAr0ta2fajdQdUtyT98TmMIuGfJ2kt2/fXHVA/kHQ6sCXFH8zBs3ID4076tr8paT6wB8VY/f1s/6qqWGNcHgNulvQTlu3TP7q+kJ4iaUPg2bavkXQ+cABwRs1h1S4XcrskaTPgNNu7liNH7rf9H3XH1Q8k3WZ7dkVtrT3M7kW2l1TR/nRWrlX0r8DfgAuBrYD32P5Gj+29bbj9ts/qOcgKlb+nD9r+vKQXA6fa3rXuuOqWpN8lSZ8Gbrf9tXKY2vlDp6DXpezXPoKnd58cVldMnSR9Dfh327dV0Na9FOvt3E9xpr8mxaJrf6K4WDd/oq8xXUm6wfbWkt4A7Af8E/Bz21vVHFrlJAm4A9h5cMKYpKuBQ2zfUWtwNUv3ThfKEQBvBLYAsP1bSX+RtL3tobND6/AD4Argp8DSmmMZzlnA1ZL+G1hMkazd4xotFwLft30RgKS9gL2B7wJfAnaqJuRpafny62uAb9v+a5EbeyNpFsWs3NnAioP7bb9gIkFWZDWKTzGdM4SPZMjyHW2UM/0ulKN2NrV9Q8e+jYClthfUF9mTsdxge+u64xiJpLuAY4Cb6RhpY/u+HtqaZ3v74fY1/f+hbpI+SXGG/zdgR4pPST+y3dMfSkm/AI4HPge8DjiUIqccX03E1SlnEK9qu/VV1jIjtwu2Hx2S8NcC1mhCwi/9SNJr6g5iFL+1Pcf2PbbvG9x6bOuvkj4gaaNyez9wfzmEMEM3R2H7WGAXYPvyGsgjFEMae7VSOXRZ5ff0o0Bj1kCS9C1Jq0taBbgNuEPS++qOq2450++SpEuBfSm6xG4A/gxcZvuYOuMCkLSIopDIYmAJT3WfrF5rYCVJX6I4q/whRYxAb0M2Ja1DcXa5K8X7/AXFJKEHgefbvquKmKcrSZvz9O6Yr/fY1pXAbsC5wM+A3wOftP2iCkKdsI5rGG8FtgM+AMxv+9LP6dPv3hq2H5L0duAM28cPWVa2NrZXqzuGMaxEkez36tjX65DNhcA/jvBwEv4oJB0P7E6R9C8A9qH4o9lT0gfeA6wMHA18DHgl8A8TDrQ6y5fX4/YDvmh7iaTWn+Um6XdvOUnrUUzQ+pe6gxlK0vrARiw7eufy+iJ6iu1Dq2pL0qbAe3n6SKXGdCs02P4UwzSvt32opOcAp02gvZnl0gYPU/TnI+lNwC8nHGk1vgrcC9wIXF5eh2t9n366d7pU/jB/BLjS9rskvQD4tO3aS/WVJeoOoOi3HBy946YsfFUm6i8Dz7G9eTk7d1/b/9pDWzcCXwHm0zFSKUM1xybpWts7lpPbXgksAm6x/ZIe2xuuctbT9jWJpOVsP1F3HHVK0p8GJN0BbGl78ZgH10DSZcD7gK/a3qbcd0sva7FLmm97u6pjbIPy2sqHgAOBf6Y4Q79hvJ/EJO1DMezzzcB3Oh5aHZhtuxFLXZfFXo4HXl7uugw4seHLRky6jN7pkqQNJH1f0p8k/Y+k/5S0Qd1xle7mqTHYTbSy7WuH7Ov1bOuHko6UtJ6ktQe3iQbYBraPtP2A7a8Arwbe1mPX2x8oqpc9RvGJa3CbA/xdVfFW4HSKTzNvLreHyDIM6dMfhzOAbwFvKu8fXO57dW0RPeVR4AZJl7Ds6JhGrIECLJS0CU+tjLk/xSzaXgxO/e8cemegCROCGk3S1ykm8V1h+/Ze27F9I0UBlW8NLn9RDmPesGHrUW0ypPv1BEk3jHh0S+RMv3vr2j7D9hPldiawbt1BleZQjJ64imXPvHoiaeNu9o3Duykuqm0m6fcUoz7e1UtDtjceZus54ZfXasbcVwdJM8rlP6pyJrAe8B+SflN+Wv2/E2jvJ+U4+LUpLpaeIemzVQRakb9JenKtHUkvo5iY1mrp0++SpJ9S/NJ8u9x1EHCo7T1qC2qSjHCBbsJ96eUkmWfYXjTBdqoca97oi5EqqlPt4Yp+UctJbDtQXMh9J/A325v12Nb1trcphzFvODiMuSnj4CVtTbEEyBrlrvsp1t65sb6o6pfune4dBnyRYso5wJXlvnGT9EyKtXxmsuywwxN7bK+SNVBUrCT6EmANSf+746HVO9vtIb5l3u/gei+9vN+qxpp3XIxcX9IXOh5and6vN0yG64EfSPoeyy5f3MvEtksoJvFdTdHNs4PtP00gtkYPYy5n0W8lafXyfuuHa0KSftds/5ZiRm4VfkAxg3Q+HX3wE3AGT62B8krKNVB6aOdFwGspZs++rmP/IopVPHtV5futaqz54MXIfVm2K2wRxeqTTbE28BeWXd6gp4ltwE0UM1M3p/h+PCDpatu9dnmcCFwE/ML23HIY8509tlW58mfj34Dn2d5H0mxgF9tfqzm0etnO1sUGnERxFrg8RdWshcDBPbZ1S8WxzS+/3tyx74oJtLdLxfFV9n6Bawffc/n9EHDrBNpbbip/jpqwAatSzGq+D1hcdzyT+D5/TPEp5MbB73Xn70hbt5zpd28v2+9XsRb5AopRPD8HeilAcZWkLWzfXFFsj5WrCN4p6SiKNVCePd5GJL3f9knAWyQdNPRx9z4aqMr3O0/SmsCpFIn/YWDocNDxuHO4qfluxvLAVU9s+0eKNYu2o0j4p1N08/QaW6PrOADr2P6upA8C2H5CUhOXHp9SSfrdq3It8l2BQyTdw8TXl4fq1kAZLDs4j3J4ZUUqeb8q/sM/YfsB4CuSLgRWtz2RNZA6l2lekeKPeZPG/Z9KObENwPZNkr5FUQFrvFYEPkvxybCK6xZNr+PwiKRn8dRQ4Z0purVaLUm/ez+UdDvFkK8jy7Ocx3psa5/qwgKKH+qzKdbeGfzjdCpFXdruG7F/WN68jWLm5kye+hkxvS/MVcn7tW0VtU63K+/fW0Gbfxmy6/+pWCf+uIm2XZGVbV875ASj14S9pe1lhoBKOtv2308gtg/0+NypcAzFcOZNVKwIui7FNaFWS9Lvku1jyzVuHrK9VNKjjHMtckmruxhBMKEhi8P4JsXZ4DJFSibgG1W0N0nv9xpJO7hY6GvCJHUOzXwGxZl/k1YtrXJi2zJr7EhajvIPaI9+JOk1ti+YQBuTxvZ1kl5BMUBBwB1OLeWM0++GispZs9wxvldFndyltn8/jnZ+ZPu1ZTfH4H/84Cmce+1HlvQLV1jwuar2JuP9SrqN4pf4XoohjBPqGpP0847Ynijb/YztX/fSXtXKETGnAC+lGGd+D/BWj6MITdmn/SGKJa4fHdwNPA6cYvuDPcbW2DoOVf3OTkdJ+l1QsSb37RQfjx8p910MfMg91MgtL7q+FdjY9onlD+N6tntaklbSHhSTxYYuw9DLsL7JaK+y96tiedy1KIp3AFwOPDCeJDikvRV5+pwJu8c5E1WRNLQ4z0oUn0QeAbA97pmvkk6i+PT2AtsnlN+H5/rp6yKNp821gVksOz/ksl7bq0rVv7PTSZZh6EL5kfD7FMsXD54xrDuBH56TgZ0pEisU3R9fnECIhwJbUxQIf125vbZB7VX5fvejuH6xDkUf7dlMbP7E+RTvbwnFSKCH6ZgEVaPVym17iiUr1qKYP/FOiolpvVid4vtwYHl/EcX3piflTNzLKIrVf7T82ohrIZPwOzt91D1mtF82YDPKse/Ah4GjJ9DWdeXX6zv23TiB9iodezwJ7VX2fikmGK3ScX8V4KYJxFbpnImqN+BiYLWO+6sBF9b9fRj8OaE4w7+hvL8Z8J26/8864qvsd3Y6bTnT75LLVQnLcdMHUZxh9mpJuQbK4MW5dZnYBdhrytmGVam6vSrfr1h2eOBSept9POgqSVtM4PmjUrEE9DMn0MTzKfreBz1O0RXVi6p/7h6z/VjZ1jPL35FG1MeFyn9np42M3hmfr1FM+b/JE1tC9gsUHz2fLenjFMPIPjyB9nYF3lbhuP+q26vy/Z4B/FLS98v7+1F8X8ZF0s0UyW854FBJd1PNex3qbIohg/9p+709Pv/a8v0aeAPFImK9qPrnbkE5Ue58ihU376dY3qJSkp5r+797fHpVv7PTRi7kjkM5IuCPwBtt/3SCbW0G7EGRZC6x/asxnjJaWxsNt9+9X9ystL2yzSrf77YUf5gEXG77+h7aGPY9DprIex3mtURRUerWHp+/LR0Xrnt5vx1tVfZ9GNLuKyhWs7zQ9uNjHT/Otv/L9v/q8bmV/c5OF0n6EREtkj79iIgWSdKPiGiRJP0eSHpHU9trcmxta6/JsTW9vSbH1u+S9HtT9Q9Qle01Oba2tdfk2JreXpNj62tJ+hERLZLROx2GK6bRJNttN/aCiH/+859Zd911xzxu/vz5Yx4T0UILbY/9CzSKvffe2wsXLuzq2Pnz519ke++JvN54ZXJWH5k3r7plQyZQACZiOpvw/IyFCxd2/bsqaZ2Jvt54JelHRFSsyT0oSfoRERUysHSgilpGkyNJPyKiUsaVlpiuVpJ+RESVDAPNzflJ+hERVUuffkRESxgYaHDSn/aTsySdL2m+pFszFTsipsI4qntNuTac6R9m+6+SVgLmlsUs/lJ3UBExPdnO6J2aHS3pDeXtDYFZwJNJvzz7zyeAiKhM+vRrIml3YE9gF9uPSrqUopDzk2yfApxSHt/c71RE9I0M2azPGsD9ZcLfDNi57oAiYnorLuTWHcXIpnvSvxB4p6SbgDuAa2qOJyJaIN07NbG9GNin7jgiokVyITcioj1MzvQjIlqlyZOzkvQjIiqWM/2IiNbIKpsREa3hrLIZEdEuAxm9ExHRDk1fZTNJPyKiYrmQGxHRFnbO9CMi2iRn+hERLWFgaZJ+RER75Ew/IqJFmpz0G10jV9JMSbdLOk3SLZK+KWlPSVdKulPSjuV2laTry68vKp97iKTzJF1YHntS3e8nIqY/lxdyu9nq0OikX3oh8HlgS2Az4C3ArsB7gQ8BtwMvt70NcBzwbx3P3Ro4ANgCOEDShlMYd0S0VAqjT8w9tm8GkHQrcIltS7oZmElRHessSbMorqEs3/HcS2w/WD73NmAj4HedjadGbkRUrcndO/2Q9Bd33B7ouD9AEf/HgJ/bfoOkmcClIzx3KcO839TIjYgqFaN3sgzDZFoD+H15+5Aa44iIAJq94Fo/9OmP5STgE5KuBGbUHUxEtFyX/fnp0x+G7XuBzTvuHzLCY5t2PO0j5eNnAmd2HP/ayYozImJQyiVGRLRM1t6JiGiRnOlHRLSEbZamiEpERHukRm5ERIs0echmkn4fkVRZW08sXVpZWwDLzcho2QiodvSOpL0plqGZAZxm+5NDHn8+cBawZnnMsbYvGK3N6TBOPyKiUaoYpy9pBnAysA8wGzhI0uwhh30Y+G659tiBwJfGii1n+hERVaruQu6OwF227waQdA7weuC2zlcDVi9vrwH8YaxGk/QjIipUYffO+iy7QOQCYKchx3wUuFjSPwKrAHuO1Wi6dyIiKjaO9fTXkTSvY+tc8Xe4i3hD/5ocBJxpewPgNcDZkkbN6znTj4io2DiGbC60vf0Ijy0AOmuAbMDTu28OB/YGsH21pBWBdYA/jfSCfXemL+mdkv6h7jgiIkZid7eNYS4wS9LGklaguFA7Z8gxvwX2AJD0YmBF4M+jNdpXZ/qSlrP9lbrjiIgYialm7R3bT0g6CriIYjjm6bZvlXQiMM/2HOCfgVMl/VP50od4jAsKlSd9Se8HHrP9BUmfA7ay/SpJewCHAl8HTgCeCfwGONT2w5KOA14HrARcBfyfskLWpeX9lwFzJK0GPGz7M+VjvwReSTFO9XDbV0hamWKFzc2AX1FU2Hq37XlVv9+IiGVUuAxDOeb+giH7juu4fRtFbuzaZHTvXA7sVt7eHlhV0vIUdW1vphhXuqftbYF5wDHlsV+0vYPtzSkSf+dSyGvafoXtfx/m9ZazvSPwHuD4ct+RwP22t6SorLVddW8vImJkg6N32rSe/nxgu/KMfDFwHUXy342iP2o2cGU5u3QF4Oryea8sPyWsDKwN3Ar8sHzsO6O83nkdrzuzvL0rxSw2bN8i6aaRnpwauRFRtVatsml7iaR7KbpyrgJuouh+2QS4B/iJ7YM6n1Necf4SsL3t30n6KMUFiUGPjPKSg3VwO2vgdr1eQWrkRkTVmrye/mSN3rkceG/59QrgncANwDXAyyS9EEDSypI25akEv1DSqsD+E3z9XwBvLl9jNrDFBNuLiOiSu/5Xh8lK+lcA6wFX2/4f4DHgCtt/pihe/u2yy+UaYDPbDwCnUvT5n08xVGkivgSsW77GByg+bTw4wTYjIsbU7XDNuj4MqMl9T70qFypa3vZjkjYBLgE2tf34GM+bfv8ZI8gqmxHDmj/KZKmubPLiF/sTZ57Z1bEH7LzzhF9vvPpqnP44rAz8vBw1JOBdYyX8iIgqVDVOf7JMy6RvexHFiKGIiCnX5B6UaZn0IyJqU+MY/G4k6UdEVC1JP5pm7bWeU2l737jiF5W297G3HzP2QeNw113XVdbW0qVPVNZWTE8DS5P0IyJaoRiOmaQfEdEaSfoREa2RC7kREa3igST9iIhWSJ9+RETLuKIiKpMhST8iomINPtFP0o+IqJTd6D79yVpaeUpJOl/SfEm3lpWwkPSwpI9LulHSNZKqnY0UETGCJpdLnBZJHzjM9nYUi6wdLelZwCrANba3oijmckSdAUZEO7SxRm4djpb0hvL2hsAs4HHgR+W++cCrh3tiauRGRNUyemcSSdod2BPYxfajki6lKL+4xE/9z3fWz11GauRGRKVsvDSjdybTGsD9ZcLfDNi57oAiot1ypj+5LgTeWdbDvYOi7m5ERG0anPP7P+nbXgzsM8xDq3Yccy5w7pQFFRGtNXght6n6PulHRDRKlmGIiGgTM5ALuRER7ZEz/YiIlsgqm9FIDz20sNL2Dt5tt0rbe6LiOrTLzciPekyhJP2IiPZwc7v0k/QjIqqW7p2IiLawGWhwEZXpsspmREQjVLnKpqS9Jd0h6S5Jx45wzJsl3VYuLf+tsdrMmX5ERJVcTWF0STOAkylWCF4AzJU0x/ZtHcfMAj4IvMz2/ZKePVa7OdOPiKhaMW5z7G10OwJ32b7b9uPAOcDrhxxzBHCy7fuLl/Wfxmq0EUlfhUbEEhExMd117XTRvbM+8LuO+wvKfZ02BTaVdGVZIXDvsRqdskQr6RhJt5TbeyTNlPQrSV8CrgM2lPRlSfPKvqkTOp57r6QTJF0n6eZyCWUkrSvpJ+X+r0q6T9I65WMHS7pW0g3lYzOm6r1GRLsNDLirDVinzHmDW2dBJw3T9NC/FMtRFI3aHTgIOE3SmqPFNiVJX9J2wKHAThTr3R8BrAW8CPi67W1s3wf8i+3tgS2BV0jasqOZhba3Bb4MvLfcdzzws3L/94Hnl6/3YuAAin6urSmKqLx1kt9mRETRczPgrjaKvLZ9x3ZKR1MLKCoBDtoA+MOQl1sA/MD2Etv3UCwvP2u0+KbqTH9X4Pu2H7H9MHAesBtwn+3O9e/fLOk64HrgJcDsjsfOK7/OB2Z2tHsOgO0LgfvL/XsA21Fc+LihvP+C4QKT9I7Bv7ITe4sREYWKunfmArMkbSxpBeBAYM6QY84HXglQ9nJsCtw9WqNTNXpnuI8pAI88eYC0McUZ/A7lVegzKcoeDlpcfu0sfThSuwLOsv3BsQJLucSIqFoVk7NsPyHpKOAiYAZwuu1bJZ0IzLM9p3xsL0m3UeTG99n+y2jtTtWZ/uXAfpJWlrQK8AbgiiHHrE7xR+BBSc9h+MIoQ/0CeDOApL0ouowALgH2Hxy+JGltSRtN/G1ERIylsgu52L7A9qa2N7H98XLfcWXCx4VjbM+2vYXtc8Zqc0rO9G1fV565X1vuOo2numIGj7lR0vXArRQfT67soukTgG9LOgC4DPgjsMj2QkkfBi4uRwUtAd4N3FfF+4mIGFFW2SzY/izw2SG7Nx9yzCEjPHdmx+15FFeqAR4E/q78GLQL8MqyfCK2vwN8p4rYIyK6ZcBLk/Qny/OB75Zn849TjAqKiKhVzvQnie07gW3qjiMi4kld9tfXpa+TfkREE1Wx9s5kSdKPiKhYzvQjIlpicGnlpkrSj4pU+0O+zrOeV2l7V93568raeu1OL6+sLYC//vWPlbYXNbNxg4uoJOlHRFQsNXIjIlok3TsREW2RGbkREe2RC7kREa1iBpY2t1O/L0sUllW3bhnH8ftJmj32kRERE+TK1tOfFH2Z9HuwH8sWZImImDzVFEafFP2c9GdIOrWsp3uxpJUkHSFprqQbJf1nuX7/S4F9gU+X9XI3qTvwiJjeGpzz+zrpzwJOtv0S4AHgjcB5tnewvRXwK+Bw21dRlBh7n+2tbf+mvpAjYrobvJDb1O6dfr6Qe4/tG8rbg3VzN5f0r8CawKoUpcRGVVaff8dYx0VEdMVZcG2yLO64vRRYCTgT2K+swnUITxVbGVFq5EZEtcxAg5dh6OfuneGsBvxR0vLAWzv2Lyofi4iYdE3u3pluSf8jwC+BnwC3d+w/B3ifpOtzITciJl2Dr+T2ZfeO7XvpqK9r+zMdD395mOOvJEM2I2IKOH36ERHt0uBVGJL0IyKqlRq5ERHtYRo9eidJPyKiQiZ9+hERrZLunYhxeuCB/6m0vd1fskVlbX3j8ksrawvgmDf+Q6XtPfq3RZW2lxq+41XjwjpdSNKPiKhSKmdFRLTLwNIk/YiIVki5xIiINkn3TkREmzR7ctZ0W3ANAElX1R1DRLRXk1fZnJZn+rZfWncMEdFeTZ6cNV3P9B8uv64n6fKyNu4tknarO7aImN4GV9nsZhuLpL0l3SHpLknHjnLc/pIsafux2pyWZ/od3gJcZPvjkmYAK9cdUERMf1V03ZQ562Tg1cACYK6kObZvG3LcasDRFLVExjQtz/Q7zAUOlfRRYAvbT5uqKOkdkuZJmjfl0UXENNRdf34Xfxh2BO6yfbftxymKQb1+mOM+BpwEPNZNdNM66du+HHg58HvgbElPm+9u+xTb29se82NRRMSYquveWR/4Xcf9BeW+J0naBtjQ9o+6DW9ad+9I2gj4ve1TJa0CbAt8veawImKaG0f3zjpDehlOsX1KeVvDNT14Q9IzgM8Bh4wntmmd9IHdKWrjLgEeBqpd2SoiYohxzshdOEovwwJgw477GwB/6Li/GkXZ2EslATwXmCNpX9sjdldPy6Rve9Xy61nAWTWHExGtYlxNEZW5wCxJG1N0UR9IMTileBX7QWCdwfuSLgXeO1rCh2nepx8RMeUMHuhuG7UZ+wngKOAi4FfAd23fKulqBsMZAAAIaElEQVRESfv2Gt60PNOPiKhTVbNtbV8AXDBk33EjHLt7N20m6UdEVKzJa+8k6UdEVChLK0dEtInNwNJKLuROiiT9aIXHH19cWVuHv/o1lbUF8KpXHVxpe9vssW2l7X306EMqba8VcqYfEdEeJkk/IqIVnMpZERFtYjzWIPwaJelHRFQsZ/oRES0yUM0yDJOiMcswSFpT0pEd93eXNOxyoZJOkzR76qKLiOhOsVb+QFdbHRqT9IE1gSPHPAqw/fah1WMiIhqjuJo79laDMZO+pFUk/ZekG8s6swdI2kPS9ZJulnS6pGeWx94r6d8kXV1Wo9pW0kWSfiPpnR1tvk/SXEk3STqh3P1JYJOynu2ny32rSjpX0u2Svqly/VBJlw7WgpT0sKSPl/FdI+k55f5NyvtzywWKHq7w/y0iYkTu8l8dujnT3xv4g+2tbG8OXAicCRxgewuK6wLv6jj+d7Z3Aa4oj9sf2Bk4EUDSXsAsilJgWwPbSXo5cCzwG9tb235f2dY2wHuA2cALgJcNE98qwDW2twIuB44o938e+LztHVh2DeqIiElVUbnESdFN0r8Z2FPSpyTtBswE7rH96/LxsyhKEg6a0/G8X9peZPvPwGOS1gT2KrfrgeuAzSj+CAznWtsLXHR+3VC+9lCPA4N9//M7jtkF+F55+1sjvbnUyI2IapmBgaVdbXUYc/SO7V9L2g54DfAJ4OIxnjI4332g4/bg/eUoSoB9wvZXO58kaeYobQEsHSHeJX7qT+ZIx4yoLE12ShlDc8dZRURfaPrkrG769J8HPGr7G8BngJcCMyW9sDzk74HLxvGaFwGHSVq1bH99Sc8GFlGU/6rKNcAby9sHVthuRMSomty9081Z8RbApyUNAEso+u/XAL4naTmKkl5f6fYFbV8s6cXA1eV12YeBg23/RtKVkm4Bfgz81/jeytO8B/iGpH8u23pwgu1FRHSlyWf63XTvXERxdj7UNsMcO7Pj9pkUF3KHe+zzFBdahz7/LUN2Xdrx2FEdt3fvuL1qx+1zgXPLu78HdrZtSQcC6bOPiClQ33DMbkznGbnbAV8sh3k+ABxWczwR0RKmuTNyp23St30FsFXdcUREu9jNXoZh2ib9iIh61HeRthtJ+hERFcvSyhERLZIz/YjaVfdL+Mgj1Y7+/dvfql0W6lnPe1al7cX4JelHRLRFjStodiNJPyKiQgYGXM+6Ot1I0o+IqFRG70REtEqTk35jKmelXGJETBdNXnCtMUmflEuMiGmguI7bxzVyUy4xImI8jAcGutrqkHKJEREV6/cauSmXGBExDk3u00+5xJRLjIhKudFr76RcYkREhQZr5FZxpi9pb0l3SLpL0rHDPH6MpNvK66OXSNporDZTLjEiomJVdN1ImgGcDLwaWADMlTRnyMjF64HtbT8q6V3AScABo7WbcokRERWrqIjKjsBdtu8GkHQO8HrgyaRv++cdx18DHDxWo9N5Rm7KJUZEDQzV9OmvD/yu4/4CYKdRjj+copdkVNM26adcYkTUZRzDMdcZMnLwlHJwCRSDXp7e9DAkHQxsD7xirBectkk/IqIOgxdyu7TQ9vYjPLYA2LDj/gYMM+dI0p7AvwCvsL146ONDJelHRFSsojH4c4FZkjamuEZ5ILDMdU9J2wBfBfa2/aduGk3Sj4ioVDXj9G0/IekoioE0M4DTbd8q6URgnu05wKeBVSlGUwL81va+o7WbpB9RsxVWeGal7T30l4cqbW/FFVcd+6AuPfZYO5bAqmj0DrYvAC4Ysu+4jtt7jrfNJP2IiAqNs09/yiXpR0RUKjVyIyJaxTR37Z0k/YiIiqV7JyKiNVzZhdzJ0JhyiamRGxHTQd+XS5xCqZEbEdNCk4uopEZuRETF+jrpkxq5ERHj4ME+nrG3GqRGbmrkRkTFmlwYPTVyUyM3Iipkw8DA0rrDGFFq5EZEVKq7/vy6+vRTIzciomJ9PTkrNXIjIsanr5N+H0uN3IioRV0Tr7oxbZN+auRGRC1qHI7ZjWmb9CMi6mBgIGf6ERHtke6diIjWqG84ZjeS9CPGacaMan9tnvGMattbuGBhpe2ttupalbW1ePGjlbUFzT2jTtKPiGiJ1MiNiGgV4wYvw5CkHxFRsboWU+tGkn5ERMXSvRMR0SJJ+hERLVGsoNnMUUWQpB8RUbmc6UdEtMjAQM70IyLaI2f6zSXpHcA76o4jIqYLY3Km31ipkRsRVcqM3IiIlmly0h+zMPp0IemCssh7RMSk6vfC6NOC7dfUHUNEtIEZyNo7ERHt0PQ+/dZ070RETJnBOrljbWOQtLekOyTdJenYYR5/pqTvlI//UtLMsdpM0o+IqJS7/jcaSTOAk4F9gNnAQZJmDznscOB+2y8EPgd8aqzokvQjIipmD3S1jWFH4C7bd9t+HDgHeP2QY14PnFXePhfYQ5JGazR9+hERFatoGYb1gd913F8A7DTSMbafkPQg8CxgxJqZSfrLWgjc18Vx6zDKf2oPqmyvybFNi/aWLFlcWVsAc+b8R6XtMafb5hr9vagrto0qeK2LytfrxoqS5nXcP6WcMAow3Bn70D6hbo5ZRpJ+B9vrdnOcpHm2t6/qdatsr8mxta29JsfW9PaaHNtYbO9dUVMLgA077m8A/GGEYxZIWg5YA/jraI2mTz8iopnmArMkbSxpBeBAnv45bg7wtvL2/sDPPMZ40ZzpR0Q0UNlHfxRFd9EM4HTbt0o6EZhnew7wNeBsSXdRnOEfOFa7Sfq9OWXsQ2prr8mxta29JsfW9PaaHNuUsX0BcMGQfcd13H4MeNN42lSTZ45FRES10qcfEdEiSfoRES2SpB8R0SJJ+hERLZKkHxHRIkn6EREtkqQfEdEi/x9bpaL+4JxkzgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "translation, attention = translate_sentence(candidate)\n",
    "\n",
    "print(translation)\n",
    "\n",
    "display_attention(candidate, translation, attention)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}