English_to_French_training.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L2pukCDIb7yT"
      },
      "source": [
        "#Machine Learning Translation\n",
        "In this project we aim to convert English phrases to French using RNN "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cjdf4H6zl-qk",
        "outputId": "e1402e42-0bc5-4048-e11d-766e56ced104"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: Keras in /usr/local/lib/python3.10/dist-packages (2.12.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install Keras"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yva_P1dHb6aH"
      },
      "source": [
        "importing modules"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "1wmao4O0cnLS"
      },
      "outputs": [],
      "source": [
        "#Now importing modules\n",
        "import helper\n",
        "import numpy as np\n",
        "\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from keras.models import Model, Sequential\n",
        "from keras.layers import GRU, Input, Dense, TimeDistributed, Activation, RepeatVector, Bidirectional, Dropout, LSTM\n",
        "from tensorflow.keras.layers import Embedding\n",
        "from keras.optimizers import Adam\n",
        "from keras.losses import sparse_categorical_crossentropy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "X8aiTJTxdrPB"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Aziw3H_Wcyll"
      },
      "source": [
        "#Loading Data\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "cboaEhaZcvm3"
      },
      "outputs": [],
      "source": [
        "english_path='https://raw.githubusercontent.com/projjal1/datasets/master/small_vocab_en.txt'\n",
        "french_path='https://raw.githubusercontent.com/projjal1/datasets/master/small_vocab_fr.txt'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "7ozqkItv1Pd6"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "def load_data(path):\n",
        "  input_file = os.path.join(path)\n",
        "  with open(input_file, \"r\") as f:\n",
        "    data = f.read()\n",
        "\n",
        "  return data.split('\\n')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "C-jmqlMhdC0H",
        "outputId": "17114f2d-b510-45a6-f721-c65eb690219a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading data from https://nlp.stanford.edu/projects/nmt/data/iwslt15.en-vi/train.en\n",
            "13603614/13603614 [==============================] - 1s 0us/step\n",
            "Downloading data from https://nlp.stanford.edu/projects/nmt/data/iwslt15.en-vi/train.vi\n",
            "18074646/18074646 [==============================] - 1s 0us/step\n"
          ]
        }
      ],
      "source": [
        "#Using helper to inport dataset\n",
        "english_data=tf.keras.utils.get_file('file1',english_path)\n",
        "french_data=tf.keras.utils.get_file('file2',french_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "crFQWCx2eLsv"
      },
      "outputs": [],
      "source": [
        "#Now loading data\n",
        "english_sentences=load_data(english_data)\n",
        "french_sentences=load_data(french_data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_3fQhJ5J2E4D"
      },
      "outputs": [],
      "source": [
        "import collections"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lsl03eu-jW09"
      },
      "source": [
        "#Tokenize \n",
        "For a neural network to predict on text data, it first has to be turned into data it can understand. \n",
        "\n",
        "We turn each character into a number or each word into a number. These are called character and word ids.\n",
        "\n",
        "we Turn each sentence into a sequence of words ids using Keras's Tokenizer function. Use this function to tokenize english_sentences and french_sentences in the cell below."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cYLtZ0WmjTjD"
      },
      "outputs": [],
      "source": [
        "def tokenize(x):\n",
        "  tokenizer = Tokenizer()\n",
        "  tokenizer.fit_on_texts(x)\n",
        "  return tokenizer.texts_to_sequences(x), tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lkhE2V5bjyg_",
        "outputId": "4ad8350f-a81c-4c6c-fcd4-b72e37cc77c0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'the': 1, 'quick': 2, 'a': 3, 'brown': 4, 'fox': 5, 'jumps': 6, 'over': 7, 'lazy': 8, 'dog': 9, 'by': 10, 'jove': 11, 'my': 12, 'study': 13, 'of': 14, 'lexicography': 15, 'won': 16, 'prize': 17, 'this': 18, 'is': 19, 'short': 20, 'sentence': 21}\n",
            "\n",
            "Sequence 1 in x\n",
            "  Input:  The quick brown fox jumps over the lazy dog .\n",
            "  Output: [1, 2, 4, 5, 6, 7, 1, 8, 9]\n",
            "Sequence 2 in x\n",
            "  Input:  By Jove , my quick study of lexicography won a prize .\n",
            "  Output: [10, 11, 12, 2, 13, 14, 15, 16, 3, 17]\n",
            "Sequence 3 in x\n",
            "  Input:  This is a short sentence .\n",
            "  Output: [18, 19, 3, 20, 21]\n"
          ]
        }
      ],
      "source": [
        "# Tokenize Sample output\n",
        "text_sentences = [\n",
        "    'The quick brown fox jumps over the lazy dog .',\n",
        "    'By Jove , my quick study of lexicography won a prize .',\n",
        "    'This is a short sentence .']\n",
        "\n",
        "text_tokenized, text_tokenizer = tokenize(text_sentences)\n",
        "print(text_tokenizer.word_index)\n",
        "print()\n",
        "\n",
        "for sample_i, (sent, token_sent) in enumerate(zip(text_sentences, text_tokenized)):\n",
        "  print('Sequence {} in x'.format(sample_i + 1))\n",
        "  print('  Input:  {}'.format(sent))\n",
        "  print('  Output: {}'.format(token_sent))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cfiSCxQ7kRK3"
      },
      "source": [
        "#Padding \n",
        "When batching the sequence of word ids together, each sequence needs to be the same length. Since sentences are dynamic in length, we can add padding to the end of the sequences to make them the same length.\n",
        "\n",
        "we use Keras's pad_sequences function."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6Daf_ngukNIS"
      },
      "outputs": [],
      "source": [
        "def pad(x, length=None):\n",
        "  return pad_sequences(x, maxlen=length, padding='post')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9LHAL30ik5x4"
      },
      "outputs": [],
      "source": [
        "def preprocess(x, y):\n",
        "    \"\"\"\n",
        "    Preprocess x and y\n",
        "    :param x: Feature List of sentences\n",
        "    :param y: Label List of sentences\n",
        "    :return: Tuple of (Preprocessed x, Preprocessed y, x tokenizer, y tokenizer)\n",
        "    \"\"\"\n",
        "    preprocess_x, x_tk = tokenize(x)\n",
        "    preprocess_y, y_tk = tokenize(y)\n",
        "\n",
        "    preprocess_x = pad(preprocess_x)\n",
        "    preprocess_y = pad(preprocess_y)\n",
        "\n",
        "    # Keras's sparse_categorical_crossentropy function requires the labels to be in 3 dimensions\n",
        "    #Expanding dimensions\n",
        "    preprocess_y = preprocess_y.reshape(*preprocess_y.shape, 1)\n",
        "\n",
        "    return preprocess_x, preprocess_y, x_tk, y_tk\n",
        "\n",
        "preproc_english_sentences, preproc_french_sentences, english_tokenizer, french_tokenizer =\\\n",
        "    preprocess(english_sentences, french_sentences)\n",
        "    \n",
        "max_english_sequence_length = preproc_english_sentences.shape[1]\n",
        "max_french_sequence_length = preproc_french_sentences.shape[1]\n",
        "english_vocab_size = len(english_tokenizer.word_index)\n",
        "french_vocab_size = len(french_tokenizer.word_index)\n",
        "\n",
        "print('Data Preprocessed')\n",
        "print(\"Max English sentence length:\", max_english_sequence_length)\n",
        "print(\"Max French sentence length:\", max_french_sequence_length)\n",
        "print(\"English vocabulary size:\", english_vocab_size)\n",
        "print(\"French vocabulary size:\", french_vocab_size)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TSLZv-O_3Bgq"
      },
      "source": [
        "## Ids Back to Text\n",
        "The neural network will be translating the input to words ids, which isn't the final form we want. We want the French translation. The function logits_to_text will bridge the gap between the logits from the neural network to the French translation. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7BDBJKw22yJa"
      },
      "outputs": [],
      "source": [
        "def logits_to_text(logits, tokenizer):\n",
        "  index_to_words = {id: word for word, id in tokenizer.word_index.items()}\n",
        "  index_to_words[0] = '<PAD>'\n",
        "\n",
        "  #So basically we are predicting output for a given word and then selecting best answer\n",
        "  #Then selecting that label we reverse-enumerate the word from id\n",
        "  return ' '.join([index_to_words[prediction] for prediction in np.argmax(logits, 1)])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HN-RjzvH3rpp"
      },
      "source": [
        "#Building Model\n",
        "Here we use RNN model for translation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ta5yGpZ23amq"
      },
      "outputs": [],
      "source": [
        "def embed_model(input_shape, output_sequence_length, english_vocab_size, french_vocab_size):\n",
        "    \"\"\"\n",
        "    Build and train a RNN model using word embedding on x and y\n",
        "    :param input_shape: Tuple of input shape\n",
        "    :param output_sequence_length: Length of output sequence\n",
        "    :param english_vocab_size: Number of unique English words in the dataset\n",
        "    :param french_vocab_size: Number of unique French words in the dataset\n",
        "    :return: Keras model built, but not trained\n",
        "    \"\"\"\n",
        "    # TODO: Implement\n",
        "\n",
        "    # Hyperparameters\n",
        "    learning_rate = 0.005\n",
        "    \n",
        "    # TODO: Build the layers\n",
        "    model = Sequential()\n",
        "    model.add(Embedding(english_vocab_size, 256, input_length=input_shape[1], input_shape=input_shape[1:]))\n",
        "    model.add(GRU(256, return_sequences=True))    \n",
        "    model.add(TimeDistributed(Dense(1024, activation='relu')))\n",
        "    model.add(Dropout(0.5))\n",
        "    model.add(TimeDistributed(Dense(french_vocab_size, activation='softmax'))) \n",
        "\n",
        "    # Compile model\n",
        "    model.compile(loss=sparse_categorical_crossentropy,\n",
        "                  optimizer=Adam(learning_rate),\n",
        "                  metrics=['accuracy'])\n",
        "    return model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ubkWXBpu32Jh"
      },
      "outputs": [],
      "source": [
        "# Reshaping the input to work with a basic RNN\n",
        "tmp_x = pad(preproc_english_sentences, preproc_french_sentences.shape[1])\n",
        "tmp_x = tmp_x.reshape((-1, preproc_french_sentences.shape[-2]))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2NXsCzht3-1y"
      },
      "source": [
        "Finally calling the model function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8B9QZgl738qJ"
      },
      "outputs": [],
      "source": [
        "simple_rnn_model = embed_model(\n",
        "    tmp_x.shape,\n",
        "    preproc_french_sentences.shape[1],\n",
        "    len(english_tokenizer.word_index)+1,\n",
        "    len(french_tokenizer.word_index)+1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R-2dQpBQ4EZa"
      },
      "outputs": [],
      "source": [
        "simple_rnn_model.summary()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tMDG7ttA4JGS"
      },
      "source": [
        "#Training the model\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fY7037yU4HCp"
      },
      "outputs": [],
      "source": [
        "history=simple_rnn_model.fit(tmp_x, preproc_french_sentences, batch_size=1024, epochs=20, validation_split=0.2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Um2UZ6EbSge9"
      },
      "outputs": [],
      "source": [
        "simple_rnn_model.save('model_0.9396.h5')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "the following code is used to determine the number of epochs"
      ],
      "metadata": {
        "id": "NUniRXwXNraY"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gZW48yLpwuE6"
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iOTxeTc_ugsE"
      },
      "outputs": [],
      "source": [
        "loss=history.history['loss']\n",
        "val_loss=history.history['val_loss']\n",
        "epochs= range(1,len(loss)+1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CzBS8Z5gwDBm"
      },
      "outputs": [],
      "source": [
        "plt.plot(epochs, loss, 'bo' , label = 'training loss')\n",
        "plt.plot(epochs, val_loss, 'b' , label = 'validation loss')\n",
        "plt.xlabel('epochs') \n",
        "plt.ylabel('loss')\n",
        "plt.legend();"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lg0KtRlXBHPE"
      },
      "source": [
        "Predictions\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def final_predictions(text):\n",
        "  y_id_to_word = {value: key for key, value in french_tokenizer.word_index.items()}\n",
        "  y_id_to_word[0] = '<PAD>'\n",
        "\n",
        "  sentence = [english_tokenizer.word_index[word] for word in text.split()]\n",
        "  sentence = pad_sequences([sentence], maxlen=preproc_french_sentences.shape[-2], padding='post')\n",
        "  \n",
        "  print(sentence.shape)\n",
        "  print(logits_to_text(simple_rnn_model.predict(sentence[:1])[0], french_tokenizer))"
      ],
      "metadata": {
        "id": "cKoQh6AWSw39"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7XlW4fK_HndB"
      },
      "outputs": [],
      "source": [
        "import re"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pJp2gkLYE0zZ"
      },
      "outputs": [],
      "source": [
        "txt=input().lower()\n",
        "final_predictions(re.sub(r'[^\\w]', ' ', txt))"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}