nlp-course-lecture06-huawei-noahs-ark-lab.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep learning tips and tricks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook we will cover a couple of tips and tricks for tweaking a neural text classifier. We will use an LSTM model for our experiments. We use torch version 1.4.\n",
    "The code is inspired by [this](https://github.com/lukysummer/Movie-Review-Sentiment-Analysis-LSTM-Pytorch/blob/master/sentiment_analysis_LSTM.py) repository."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. LOAD THE TRAINING TEXT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package movie_reviews to\n",
      "[nltk_data]     C:/Users/Gera/mreviews/...\n",
      "[nltk_data]   Unzipping corpora\\movie_reviews.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "download(\"movie_reviews\", download_dir=\"C:/Users/Gera/mreviews/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies = load_files(\"C:/Users/Gera/mreviews/corpora/movie_reviews\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews, encoded_labels = [review.decode() for review in movies.data], movies.target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. TEXT PRE-PROCESSING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from string import punctuation\n",
    "import re\n",
    "\n",
    "word_re = re.compile(r\"\\b[a-z]{2,}\\b\")\n",
    "\n",
    "def tokenize(text):\n",
    "    processed_text = \"\".join(ch for ch in text.lower() if ch not in punctuation)\n",
    "    processed_text = processed_text.replace(\"\\n\", \" \")\n",
    "    return word_re.findall(processed_text)\n",
    "\n",
    "def flatten(tokenized_texts):\n",
    "    return [word for text in tokenized_texts for word in text]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_reviews = list(map(tokenize, reviews))\n",
    "all_words = flatten(all_reviews)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. CREATE DICTIONARIES & ENCODE REVIEWS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "word_counts = Counter(all_words)\n",
    "word_list = sorted(word_counts, key=lambda k: word_counts[k], reverse = True)\n",
    "vocab_to_int = {word:idx+1 for idx, word in enumerate(word_list)}\n",
    "int_to_vocab = {idx:word for word, idx in vocab_to_int.items()}\n",
    "encoded_reviews = [[vocab_to_int[word] for word in review] for review in all_reviews]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. CHECK LABELS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(encoded_reviews) == len(encoded_labels), \"# of encoded reviews & encoded labels must be the same!\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. GET RID OF LENGTH-0 REVIEWS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "\n",
    "encoded_labels = np.array( [label for idx, label in enumerate(encoded_labels) if len(encoded_reviews[idx]) > 0] )\n",
    "encoded_reviews = [review for review in encoded_reviews if len(review) > 0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. MAKE ALL REVIEWS SAME LENGTH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_text(encoded_reviews, seq_length):\n",
    "    \n",
    "    reviews = []\n",
    "    \n",
    "    for review in encoded_reviews:\n",
    "        if len(review) >= seq_length:\n",
    "            reviews.append(review[:seq_length])\n",
    "        else:\n",
    "            reviews.append([0] * (seq_length - len(review)) + review)\n",
    "        \n",
    "    return np.array(reviews)\n",
    "\n",
    "padded_reviews = pad_text(encoded_reviews, seq_length=200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. SPLIT DATA & GET (REVIEW, LABEL) DATALOADER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ratio = 0.8\n",
    "valid_ratio = (1 - train_ratio)/2\n",
    "total = padded_reviews.shape[0]\n",
    "train_cutoff = int(total * train_ratio)\n",
    "valid_cutoff = int(total * (1 - valid_ratio))\n",
    "\n",
    "train_x, train_y = torch.from_numpy(padded_reviews[:train_cutoff]), torch.from_numpy(encoded_labels[:train_cutoff])\n",
    "valid_x, valid_y = torch.from_numpy(padded_reviews[train_cutoff:valid_cutoff]), torch.from_numpy(encoded_labels[train_cutoff:valid_cutoff])\n",
    "test_x, test_y = torch.from_numpy(padded_reviews[valid_cutoff:]), torch.from_numpy(encoded_labels[valid_cutoff:])\n",
    "\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "\n",
    "train_data = TensorDataset(train_x, train_y)\n",
    "valid_data = TensorDataset(valid_x, valid_y)\n",
    "test_data = TensorDataset(test_x, test_y)\n",
    "\n",
    "BATCH_SIZE = 50\n",
    "train_loader = DataLoader(train_data, batch_size = BATCH_SIZE, shuffle = True)\n",
    "valid_loader = DataLoader(valid_data, batch_size = BATCH_SIZE, shuffle = True)\n",
    "test_loader = DataLoader(test_data, batch_size = BATCH_SIZE, shuffle = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. DEFINE THE LSTM MODEL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "During the model definition step, we might re-implement model weight initialisation, apply another tricks such as adding various types of dropout to the needed layers etc. There is a noteworthy [discussion](https://stackoverflow.com/questions/49433936/how-to-initialize-weights-in-pytorch) on wheather one should initialize weights manually or not, and, if yes, how? The functions that implement various initialisation methods are located in the `torch.nn.init` module."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn\n",
    "\n",
    "class SentimentLSTM(nn.Module):\n",
    "    \n",
    "    def __init__(self, n_vocab, n_embed, n_hidden, n_output, n_layers, drop_p = 0.5):\n",
    "        super().__init__()\n",
    "        # params: \"n_\" means dimension\n",
    "        self.n_vocab = n_vocab     # number of unique words in vocabulary\n",
    "        self.n_layers = n_layers   # number of LSTM layers \n",
    "        self.n_hidden = n_hidden   # number of hidden nodes in LSTM\n",
    "        \n",
    "        self.embedding = nn.Embedding(n_vocab, n_embed)\n",
    "        self.lstm = nn.LSTM(n_embed, n_hidden, n_layers, batch_first = True, dropout = drop_p) \n",
    "        self.fc = nn.Linear(n_hidden, n_output)\n",
    "        self.dropout = nn.Dropout(drop_p) # this layer is designed to turn any neuron off with probability drop_p\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "        self.__init_linear()\n",
    "        \n",
    "        \n",
    "    def forward (self, input_words, hidden_state):\n",
    "                                             # INPUT   :  (batch_size, seq_length)\n",
    "        embedded_words = self.embedding(input_words)    # (batch_size, seq_length, n_embed)\n",
    "        lstm_out, h = self.lstm(embedded_words, hidden_state)         # (batch_size, seq_length, n_hidden)\n",
    "        lstm_out = self.dropout(lstm_out)\n",
    "        lstm_out = lstm_out.contiguous().view(-1, self.n_hidden) # (batch_size*seq_length, n_hidden)\n",
    "        fc_out = self.fc(lstm_out)                      # (batch_size*seq_length, n_output)\n",
    "        sigmoid_out = self.sigmoid(fc_out)              # (batch_size*seq_length, n_output)\n",
    "        sigmoid_out = sigmoid_out.view(input_words.shape[0], -1)  # (batch_size, seq_length*n_output)\n",
    "        \n",
    "        # extract the output of ONLY the LAST output of the LAST element of the sequence\n",
    "        sigmoid_last = sigmoid_out[:, -1]               # (batch_size, 1)\n",
    "        \n",
    "        return sigmoid_last, h\n",
    "    \n",
    "    def __init_linear(self):\n",
    "        self.fc.weight.data.normal_(0.0, 1/np.sqrt(self.n_hidden))\n",
    "        self.fc.bias.data.fill_(0)\n",
    "    \n",
    "    \n",
    "    def init_hidden (self, batch_size):  # initialize hidden weights (h,c) to 0\n",
    "        \n",
    "        device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "        weights = next(self.parameters()).data\n",
    "        h = (weights.new(self.n_layers, batch_size, self.n_hidden).zero_().to(device),\n",
    "             weights.new(self.n_layers, batch_size, self.n_hidden).zero_().to(device))\n",
    "        \n",
    "        return h"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9. INSTANTIATE THE MODEL W/ HYPERPARAMETERS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_vocab = len(vocab_to_int)\n",
    "n_embed = 100\n",
    "n_hidden = 256\n",
    "n_output = 1   # 1 (\"positive\") or 0 (\"negative\")\n",
    "n_layers = 2\n",
    "\n",
    "net = SentimentLSTM(n_vocab, n_embed, n_hidden, n_output, n_layers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 10. DEFINE LOSS & OPTIMIZER"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "L2-regularization is already included into SGD optimizer. The `weight_decay` parameter is responsible for controlling its intensity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import optim\n",
    "\n",
    "criterion = nn.BCELoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr = 0.001, weight_decay=1e-5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 11. TRAIN THE NETWORK!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To prevent the exploding gradient problem in LSTM/RNN we use the `clip_grad_norm_` function, that takes the `clip` parameter.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4/4 Step: 100 Training Loss: 0.6944 Validation Loss: 0.6935\n"
     ]
    }
   ],
   "source": [
    "print_every = 100\n",
    "step = 0\n",
    "n_epochs = 4  # validation loss increases from ~ epoch 3 or 4\n",
    "clip = 5  # gradient clip to prevent exploding gradient problem in LSTM/RNN\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "net.to(device)\n",
    "net.train()\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    h = net.init_hidden(BATCH_SIZE)\n",
    "    \n",
    "    for inputs, labels in train_loader:\n",
    "        step += 1\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "        # making requires_grad = False for the latest set of h\n",
    "        h = tuple([each.data for each in h])   \n",
    "        \n",
    "        net.zero_grad()\n",
    "        output, h = net(inputs.long(), h)\n",
    "        loss = criterion(output.squeeze(), labels.float())\n",
    "        loss.backward()\n",
    "        nn.utils.clip_grad_norm_(net.parameters(), clip)\n",
    "        optimizer.step()\n",
    "        \n",
    "        if (step % print_every) == 0:            \n",
    "            ######################\n",
    "            ##### VALIDATION #####\n",
    "            ######################\n",
    "            net.eval()\n",
    "            valid_losses = []\n",
    "            v_h = net.init_hidden(BATCH_SIZE)\n",
    "            \n",
    "            for v_inputs, v_labels in valid_loader:\n",
    "                v_inputs, v_labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "                v_h = tuple([each.data for each in v_h])\n",
    "                \n",
    "                v_output, v_h = net(v_inputs.long(), v_h)\n",
    "                v_loss = criterion(v_output.squeeze(), v_labels.float())\n",
    "                valid_losses.append(v_loss.item())\n",
    "\n",
    "            print(\"Epoch: {}/{}\".format((epoch+1), n_epochs),\n",
    "                  \"Step: {}\".format(step),\n",
    "                  \"Training Loss: {:.4f}\".format(loss.item()),\n",
    "                  \"Validation Loss: {:.4f}\".format(np.mean(valid_losses)))\n",
    "            net.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 12. TEST THE TRAINED MODEL ON THE TEST SET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss: 0.6939\n",
      "Test Accuracy: 0.46\n"
     ]
    }
   ],
   "source": [
    "net.eval()\n",
    "test_losses = []\n",
    "num_correct = 0\n",
    "test_h = net.init_hidden(BATCH_SIZE)\n",
    "net.to(device)\n",
    "for inputs, labels in test_loader:\n",
    "    test_h = tuple([each.data for each in test_h])\n",
    "    inputs, labels = inputs.to(device), labels.to(device)\n",
    "    test_output, test_h = net(inputs.long(), test_h)\n",
    "    loss = criterion(test_output, labels.float())\n",
    "    test_losses.append(loss.item())\n",
    "    \n",
    "    preds = torch.round(test_output.squeeze())\n",
    "    correct_tensor = preds.eq(labels.float().view_as(preds))\n",
    "    correct = np.squeeze(correct_tensor.cpu().numpy())\n",
    "    num_correct += np.sum(correct)\n",
    "    \n",
    "print(\"Test Loss: {:.4f}\".format(np.mean(test_losses)))\n",
    "print(\"Test Accuracy: {:.2f}\".format(num_correct / len(test_loader.dataset)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 13. TEST THE TRAINED MODEL ON A RANDOM SINGLE REVIEW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is a negative review.\n",
      "This is a negative review.\n",
      "This is a positive review.\n",
      "This is a negative review.\n",
      "This is a negative review.\n"
     ]
    }
   ],
   "source": [
    "def predict(net, review, seq_length = 200):\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "    \n",
    "    words = tokenize(review)\n",
    "    encoded_words = [vocab_to_int[word] for word in words]\n",
    "    padded_words = pad_text([encoded_words], seq_length)\n",
    "    padded_words = torch.from_numpy(padded_words.reshape(1, -1)).to(device)\n",
    "    \n",
    "    if(len(padded_words) == 0):\n",
    "        \"Your review must contain at least 1 word!\"\n",
    "        return None\n",
    "    \n",
    "    net.eval()\n",
    "    h = net.init_hidden(1)\n",
    "    output, h = net(padded_words.long(), h)\n",
    "    pred = torch.round(output.squeeze())\n",
    "    msg = \"This is a positive review.\" if pred == 0 else \"This is a negative review.\"\n",
    "    \n",
    "    print(msg)\n",
    "\n",
    "\n",
    "review1 = \"It made me cry.\"\n",
    "review2 = \"It was so good it made me cry.\"\n",
    "review3 = \"It's ok.\"\n",
    "review4 = \"This movie had the best acting and the dialogue was so good. I loved it.\"\n",
    "review5 = \"it's best movie ever\"\n",
    "                       ### OUTPUT ###\n",
    "predict(net, review1)  ## negative ##\n",
    "predict(net, review2)  ## positive ##\n",
    "predict(net, review3)  ## negative ##\n",
    "predict(net, review4)  ## positive ##\n",
    "predict(net, review5)  ## negative ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}