3_depression_detector.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Nm1Ba7FR5nKO"
   },
   "source": [
    "### **Project Showcase - Depression Detection using Twitter Data**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cFIRyYt97knb"
   },
   "source": [
    "This script contains pre-processing for the twitter data and model creation using PyTorch. \n",
    "\n",
    "We used TorchText, a PyTorch libray that made pre-pre-processing both simple and efficient, and applied custom techniques to work with our unique twitter data.\n",
    "\n",
    "A lot of the preprocessing and model section was inspired by code from this article, as we are new to NLP using PyTorch - https://medium.com/@sonicboom8/sentiment-analysis-torchtext-55fb57b1fab8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KlrX5NJiBE8G"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MzadFKTlP0wU"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import spacy\n",
    "from tqdm import tqdm, tqdm_notebook, tnrange\n",
    "tqdm.pandas(desc='Progress')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nCm2NnqrP4fL"
   },
   "outputs": [],
   "source": [
    "import torchtext\n",
    "from torchtext.data import Field, BucketIterator, TabularDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MSy7O88mQMhU"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6DkH-8NbQUiU"
   },
   "outputs": [],
   "source": [
    "import os, sys\n",
    "import re\n",
    "import string\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "p1fswumgQZGs"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jfAbtdwOQa6s"
   },
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 126
    },
    "colab_type": "code",
    "id": "ht17_lvKQpun",
    "outputId": "1587fb32-637b-442a-bbf4-965d8a633913"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python version: 3.6.8 (default, Jan 14 2019, 11:02:34) \n",
      "[GCC 8.0.1 20180414 (experimental) [trunk revision 259383]]\n",
      "Pandas version: 0.24.2\n",
      "Pytorch version: 1.1.0\n",
      "Torch Text version: 0.3.1\n",
      "Spacy version: 2.1.8\n"
     ]
    }
   ],
   "source": [
    "print('Python version:',sys.version)\n",
    "print('Pandas version:',pd.__version__)\n",
    "print('Pytorch version:', torch.__version__)\n",
    "print('Torch Text version:', torchtext.__version__)\n",
    "print('Spacy version:', spacy.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Y9Cy0aZfvPE0"
   },
   "source": [
    "## **1. Load Data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "cl-sPuUTAPDF",
    "outputId": "5bf14b17-94e6-40cb-dc2a-505e592246bf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2lvtAgdT7gPN"
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/content/gdrive/My Drive/Colab Notebooks/data/tweets_combined.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hPVE4THcCGId"
   },
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 198
    },
    "colab_type": "code",
    "id": "yXa79WdbBDPh",
    "outputId": "ee177313-ada5-482b-8755-d7a565788127"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>tweet</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Wanted to watch the TNF game but I forgot my parents cancelled the NFL Network package ????</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>For all the years I took the lynx box sets for granted and this year I didn't get one ???¢???‚</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Hon. Miss Dashwood, whose manners very pretty face, she offended you have done before.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>ebullient [ih BUL yunt] adj.boiling; bubbling with excitement; exuberant. A boiling liquid can be called ebullient. Excited or enthusiastic.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>All the proud parents on fb about their kids school report and am shitting myself for Graces arriving 😂😂😂  #troublemaker</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  ... target\n",
       "0  0           ...  0    \n",
       "1  1           ...  0    \n",
       "2  2           ...  0    \n",
       "3  3           ...  0    \n",
       "4  4           ...  0    \n",
       "\n",
       "[5 rows x 3 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 72
    },
    "colab_type": "code",
    "id": "wMpkcMGEBqO5",
    "outputId": "03c6a2a4-11bf-4037-c8bc-528ce5489924"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1936\n",
       "1    585 \n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.target.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 228
    },
    "colab_type": "code",
    "id": "3x_LwHt-Pqm9",
    "outputId": "ec4799bc-84d9-4e98-dfe6-0407e4a603dc"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVcAAADTCAYAAAA1Z1BiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADaJJREFUeJzt3X+s3XV9x/HnS5ARgUlZS1PbYolr\ntrEIFe4Af8zUudRCslWTxcDc6BhZ3QaL2zIm8w9xEKfZD+fAH0kdDZAojmVzbZZutTKEbAHXW8UC\nImsDdLQUWqxDGcYpvvfH+d54KPeW03o/99xz+3wkJ+d73t/P93veN7l55Xs+3+/3nFQVkqTp9bJh\nNyBJc5HhKkkNGK6S1IDhKkkNGK6S1IDhKkkNGK6S1IDhKkkNNAvXJEuT3Jnka0keTPKern5akq1J\ndnbP87p6ktyQZFeSHUnO7dvX2m78ziRrW/UsSdMlre7QSrIIWFRVX05yCrAdeDvwG8DBqvpwkmuA\neVX13iQXA78HXAxcAPxNVV2Q5DRgHBgDqtvPeVX1zanee/78+bVs2bImf5ekY9f27dufrqoFg4w9\nvlUTVbUP2NctfzvJQ8BiYA2wsht2C/BF4L1d/dbqpf29SU7tAnolsLWqDgIk2QqsBm6b6r2XLVvG\n+Ph4g79K0rEsye5Bx87InGuSZcDrgC8BC7vgBXgSWNgtLwYe79tsT1ebqn7oe6xLMp5k/MCBA9Pa\nvyQdqebhmuRk4B+A36+qb/Wv645Sp2VeoqrWV9VYVY0tWDDQUbskNdM0XJO8nF6wfrqq/rErP9V9\n3J+Yl93f1fcCS/s2X9LVpqpL0qzV8mqBADcBD1XVR/pWbQImzvivBTb21S/rrhq4EHimmz7YAqxK\nMq+7smBVV5OkWavZCS3gjcCvA/cnua+rvQ/4MHB7kiuA3cA7u3Wb6V0psAt4DrgcoKoOJrke2NaN\nu27i5JYkzVbNLsUaprGxsTqaqwXOu/rWBt1opmz/i8uG3YLmuCTbq2pskLHeoSVJDRiuktSA4SpJ\nDRiuktSA4SpJDRiuktSA4SpJDRiuktSA4SpJDRiuktSA4SpJDRiuktSA4SpJDRiuktSA4SpJDRiu\nktSA4SpJDRiuktSA4SpJDRiuktSA4SpJDRiuktSA4SpJDRiuktSA4SpJDRiuktSA4SpJDRiuktSA\n4SpJDRiuktSA4SpJDTQL1yQbkuxP8kBf7QNJ9ia5r3tc3LfuT5LsSvJwkrf11Vd3tV1JrmnVryRN\np5ZHrjcDqyep/3VVregemwGSnAVcAvxst80nkhyX5Djg48BFwFnApd1YSZrVjm+146q6O8myAYev\nAT5bVd8FHk2yCzi/W7erqh4BSPLZbuzXprldSZpWw5hzvSrJjm7aYF5XWww83jdmT1ebqv4iSdYl\nGU8yfuDAgRZ9S9LAZjpcPwm8BlgB7AP+arp2XFXrq2qsqsYWLFgwXbuVpKPSbFpgMlX11MRykk8B\n/9y93Ass7Ru6pKtxmLokzVozeuSaZFHfy3cAE1cSbAIuSfJjSc4ElgP/CWwDlic5M8kJ9E56bZrJ\nniXpaDQ7ck1yG7ASmJ9kD3AtsDLJCqCAx4B3A1TVg0lup3ei6vvAlVX1fLefq4AtwHHAhqp6sFXP\nkjRdWl4tcOkk5ZsOM/6DwAcnqW8GNk9ja5LUnHdoSVIDhqskNWC4SlIDhqskNWC4SlIDhqskNWC4\nSlIDhqskNWC4SlIDhqskNWC4SlIDhqskNWC4SlIDhqskNWC4SlIDhqskNWC4SlIDhqskNWC4SlID\nhqskNfCS4ZrkjYPUJEk/NMiR640D1iRJnSl/WjvJ64E3AAuS/GHfqh8HjmvdmCSNsinDFTgBOLkb\nc0pf/VvAr7RsSpJG3ZThWlV3AXclubmqdid5RVU9N4O9SdLIGmTO9VVJvgZ8HSDJOUk+0bYtSRpt\ng4TrR4G3Ad8AqKqvAm9u2ZQkjbqBrnOtqscPKT3foBdJmjMOd0JrwuNJ3gBUkpcD7wEeatuWJI22\nQY5cfxu4ElgM7AVWdK8lSVN4ySPXqnoaeNcM9CJJc8ZLhmuSGyYpPwOMV9XG6W9JkkbfINMCJ9Kb\nCtjZPc4GlgBXJPnoVBsl2ZBkf5IH+mqnJdmaZGf3PK+rJ8kNSXYl2ZHk3L5t1nbjdyZZe5R/pyTN\nqEHC9WzgLVV1Y1XdCPwi8NPAO4BVh9nuZmD1IbVrgDuqajlwR/ca4CJgefdYB3wSemEMXAtcAJwP\nXDsRyJI0mw0SrvPo3QY74STgtKp6HvjuVBtV1d3AwUPKa4BbuuVbgLf31W+tnnuBU5Msond97daq\nOlhV3wS28uLAlqRZZ5BLsf4cuC/JF4HQu4Hgz5KcBHzhCN9vYVXt65afBBZ2y4uB/mtp93S1qeov\nkmQdvaNezjjjjCNsS5Km12HDNUmAzwOb6X0sB3hfVT3RLV99tG9cVZWkjnb7Sfa3HlgPMDY2Nm37\nlaSjcdhpgaoqYHNV7auqjd3jicNt8xKe6j7u0z3v7+p7gaV945Z0tanqkjSrDTLn+uUkPzdN77cJ\nmDjjvxbY2Fe/rLtq4ELgmW76YAuwKsm87kTWqq4mSbPaIHOuFwDvSrIb+F96865VVWcfbqMktwEr\ngflJ9tA76/9h4PYkVwC7gXd2wzcDFwO7gOeAy+m9ycEk1wPbunHXVdWhJ8kkadYZJFzfdjQ7rqpL\np1j11knGFlPcUltVG4ANR9ODJA3LILe/7gZIcjq9GwokSS9hkF9//eUkO4FHgbuAx4B/adyXJI20\nQU5oXQ9cCPxXVZ1J72P9vU27kqQRN0i4fq+qvgG8LMnLqupOYKxxX5I00gY5ofU/SU4G7gY+nWQ/\n8GzbtiRptA0Srl+ld3nUH9D7XtdX8sLvGpAkHWKQcH1LVf0A+AHdl64k2dG0K0kacVOGa5LfAX4X\neM0hYXoK8B+tG5OkUXa4I9fP0Lvk6kP88HtXAb7tXVKSdHhThmtVPUPv51ymutNKkjSFQS7FkiQd\nIcNVkhowXCWpAcNVkhowXCWpAcNVkhowXCWpAcNVkhowXCWpAcNVkhowXCWpAcNVkhowXCWpAcNV\nkhowXCWpAcNVkhowXCWpgUF+oFDSJP77utcOuwX9CM54//1N9++RqyQ1YLhKUgOGqyQ1MJRwTfJY\nkvuT3JdkvKudlmRrkp3d87yuniQ3JNmVZEeSc4fRsyQdiWEeub6lqlZU1Vj3+hrgjqpaDtzRvQa4\nCFjePdYBn5zxTiXpCM2maYE1wC3d8i3A2/vqt1bPvcCpSRYNo0FJGtSwwrWAzyfZnmRdV1tYVfu6\n5SeBhd3yYuDxvm33dLUXSLIuyXiS8QMHDrTqW5IGMqzrXN9UVXuTnA5sTfL1/pVVVUnqSHZYVeuB\n9QBjY2NHtK0kTbehHLlW1d7ueT/wOeB84KmJj/vd8/5u+F5gad/mS7qaJM1aMx6uSU5KcsrEMrAK\neADYBKzthq0FNnbLm4DLuqsGLgSe6Zs+kKRZaRjTAguBzyWZeP/PVNW/JtkG3J7kCmA38M5u/Gbg\nYmAX8Bxw+cy3LElHZsbDtaoeAc6ZpP4N4K2T1Au4cgZak6RpM5suxZKkOcNwlaQGDFdJasBwlaQG\nDFdJasBwlaQGDFdJasBwlaQGDFdJasBwlaQGDFdJasBwlaQGDFdJasBwlaQGDFdJasBwlaQGDFdJ\nasBwlaQGDFdJasBwlaQGDFdJasBwlaQGDFdJasBwlaQGDFdJasBwlaQGDFdJasBwlaQGDFdJasBw\nlaQGDFdJamBkwjXJ6iQPJ9mV5Jph9yNJhzMS4ZrkOODjwEXAWcClSc4ableSNLWRCFfgfGBXVT1S\nVf8HfBZYM+SeJGlKxw+7gQEtBh7ve70HuKB/QJJ1wLru5bNJHp6h3kbJfODpYTfRSv5y7bBbmGvm\n9P8L1+Zotnr1oANHJVxfUlWtB9YPu4/ZLMl4VY0Nuw+NBv9ffjSjMi2wF1ja93pJV5OkWWlUwnUb\nsDzJmUlOAC4BNg25J0ma0khMC1TV95NcBWwBjgM2VNWDQ25rFDltoiPh/8uPIFU17B4kac4ZlWkB\nSRophqskNWC4HiO8fViDSrIhyf4kDwy7l1FmuB4DvH1YR+hmYPWwmxh1huuxwduHNbCquhs4OOw+\nRp3hemyY7PbhxUPqRTomGK6S1IDhemzw9mFphhmuxwZvH5ZmmOF6DKiq7wMTtw8/BNzu7cOaSpLb\ngHuAn0qyJ8kVw+5pFHn7qyQ14JGrJDVguEpSA4arJDVguEpSA4arJDVguGpOSPLsEYz9QJI/arV/\nCQxXSWrCcNWcleSXknwpyVeSfCHJwr7V5yS5J8nOJL/Vt83VSbYl2ZHkTyfZ56Ikdye5L8kDSX5+\nRv4YjRzDVXPZvwMXVtXr6H3N4h/3rTsb+AXg9cD7k7wqySpgOb2vaFwBnJfkzYfs81eBLVW1AjgH\nuK/x36ARNRK//iodpSXA3yVZBJwAPNq3bmNVfQf4TpI76QXqm4BVwFe6MSfTC9u7+7bbBmxI8nLg\nn6rKcNWkPHLVXHYj8LGqei3wbuDEvnWH3vddQIAPVdWK7vGTVXXTCwb1vkj6zfS+VezmJJe1a1+j\nzHDVXPZKfvjVimsPWbcmyYlJfgJYSe+IdAvwm0lOBkiyOMnp/RsleTXwVFV9Cvhb4NyG/WuEOS2g\nueIVSfb0vf4I8AHg75N8E/g34My+9TuAO4H5wPVV9QTwRJKfAe5JAvAs8GvA/r7tVgJXJ/let94j\nV03Kb8WSpAacFpCkBgxXSWrAcJWkBgxXSWrAcJWkBgxXSWrAcJWkBv4frDxfrP70iVoAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<Figure size 360x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(5,3))\n",
    "ax = sns.barplot(x=df.target.unique(),y=df.target.value_counts());\n",
    "ax.set(xlabel='Labels');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 417
    },
    "colab_type": "code",
    "id": "aQfsoDz286Tz",
    "outputId": "db8002df-548e-48a6-e91a-c79e2ea88ea8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0    Wanted to watch the TNF game but I forgot my parents cancelled the NFL Network package ????                                                 \n",
       " 1    For all the years I took the lynx box sets for granted and this year I didn't get one ???¢???‚                                              \n",
       " 2    Hon. Miss Dashwood, whose manners very pretty face, she offended you have done before.                                                      \n",
       " 3    ebullient [ih BUL yunt] adj.boiling; bubbling with excitement; exuberant. A boiling liquid can be called ebullient. Excited or enthusiastic.\n",
       " 4    All the proud parents on fb about their kids school report and am shitting myself for Graces arriving 😂😂😂  #troublemaker                    \n",
       " 5    \"I  a temporary, contract job but I am hoping for permanent\" -  that's heard all too often amongst GenY.                                    \n",
       " 6    He aint flinch tho, I'll give him that lol                                                                                                  \n",
       " 7    Scratched 2 inches ice off the car 2 go sledging.After being done we had enough snow and stayed at home   pic.twitter.com/RaRHNljLMt        \n",
       " 8    Your  perseveres. Good luck in Mercury Awards, darling @ANOHNItweets   pic.twitter.com/is67imvghq                                           \n",
       " 9    @Jesus_Luvs_Us Hello Jillian &amp; JC - a nice wisg from Betty for blessings to us all - I wish you the same - have a wonderful Tuesday! 😀  \n",
       " Name: tweet, dtype: object,\n",
       " 2511    I need to work on being a stronger person emotionally                                                                                \n",
       " 2512    Yay... a big shout out for my friend @GuptaJuhi90 . Welcoming her back on twitter. She a terrific writer. Stay tuned for tweets.     \n",
       " 2513    @SaidTheSky @DasEnergiFest @mitchell_thayne Bro, you should be #smiling you're bring all of us smiles. Share the experience #together\n",
       " 2514    Over the past 2 days I've drank 27 bottles of beer, 2 pints of cider and half a bottle of prosecco.                                  \n",
       " 2515    @thedramble Tried it once. Poured into the sink. Didn't bother to make tasting notes :-) Looks like cheap sake, tasted like it too...\n",
       " 2516    honestly just wanna watch OTH & have no Netflix to do that, so                                                                       \n",
       " 2517    Mayweather's trash talking is on par with Nate Diaz #terrible would love to see McGregor KO him #MayweatherVsMcGregor                \n",
       " 2518    Shakespeare Dictionary: The word 'knotty-pated' means 'block-headed, dull-witted'                                                    \n",
       " 2519    @help_dms___ So good lunch 😊                                                                                                         \n",
       " 2520    So excited for tonight, I actually feel a bit sick 😅                                                                                 \n",
       " Name: tweet, dtype: object)"
      ]
     },
     "execution_count": 15,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tweet.head(10), df.tweet.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 126
    },
    "colab_type": "code",
    "id": "Xopy9Qr2i6eD",
    "outputId": "2dab77be-e45b-4032-e49c-ec1c7dbf757f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Wanted to watch the TNF game but I forgot my parents cancelled the NFL Network package ????                                                 \n",
       "1    For all the years I took the lynx box sets for granted and this year I didn't get one ???¢???‚                                              \n",
       "2    Hon. Miss Dashwood, whose manners very pretty face, she offended you have done before.                                                      \n",
       "3    ebullient [ih BUL yunt] adj.boiling; bubbling with excitement; exuberant. A boiling liquid can be called ebullient. Excited or enthusiastic.\n",
       "4    All the proud parents on fb about their kids school report and am shitting myself for Graces arriving 😂😂😂  #troublemaker                    \n",
       "Name: tweet, dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check non-depressive tweets\n",
    "df[df[\"target\"]==0].tweet.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 126
    },
    "colab_type": "code",
    "id": "C-OqFJlvBs2B",
    "outputId": "67291707-6791-4bcb-93d5-bb44b224c9d7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16    It's weird that I get 300x more depressed over Christmas break every year.                                              \n",
       "20    It's so sad when you talk so highly of someone then they end up disappointing u and making u look like a pendeja        \n",
       "25    A whopping 9% of adults in America are depressed. Another 3.4% are deeply depressed. Let's Talk!                        \n",
       "27    Not only have I posted a blog post daily for 2 months I've also filmed and uploaded a daily vlog whilst being severely  \n",
       "33    HealthTap: I haven't eaten anything all day because I feel  and low What should I do?                                   \n",
       "Name: tweet, dtype: object"
      ]
     },
     "execution_count": 17,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check depressive tweets\n",
    "df[df[\"target\"]==1].tweet.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Ydm2HSMOwBSQ"
   },
   "source": [
    "## **2. Define How to Preprocess Data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "Ts4ioqHeRCC8",
    "outputId": "1963ad0a-0b6b-484f-f356-afb6d2b9edf3"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Progress: 100%|██████████| 2521/2521 [00:00<00:00, 177030.26it/s]\n"
     ]
    }
   ],
   "source": [
    "# torchtext have trouble handling \\n. Replace \\n character with space\n",
    "df['tweet'] = df.tweet.progress_apply(lambda x: re.sub('\\n', ' ', x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yo57ODfhRWO0"
   },
   "outputs": [],
   "source": [
    "contraction_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dy7D7zhvpsvh"
   },
   "outputs": [],
   "source": [
    "def _get_contractions(contraction_dict):\n",
    "    contraction_re = re.compile('(%s)' % '|'.join(contraction_dict.keys()))\n",
    "    return contraction_dict, contraction_re\n",
    "\n",
    "contractions, contractions_re = _get_contractions(contraction_dict)\n",
    "\n",
    "def replace_contractions(text):\n",
    "    def replace(match):\n",
    "        return contractions[match.group(0)]\n",
    "    return contractions_re.sub(replace, text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hc_P0uFHkYJh"
   },
   "outputs": [],
   "source": [
    "def tweet_clean(text):\n",
    "    text = re.sub(r'https?:/\\/\\S+', ' ', text) # remove urls\n",
    "    text = re.sub(r'<([^>]*)>', ' ', text) # remove emojis\n",
    "    text = re.sub(r'@\\w+', ' ', text) # remove at mentions\n",
    "    text = re.sub(r'#', '', text) # remove hashtag symbol\n",
    "    text = re.sub(r'[0-9]+', ' ', text) # remove numbers\n",
    "    text = replace_contractions(text)\n",
    "    pattern = re.compile(r\"[ \\n\\t]+\")\n",
    "    text = pattern.sub(\" \", text)      \n",
    "    text = \"\".join(\"\".join(s)[:2] for _, s in itertools.groupby(text))    \n",
    "    text = re.sub(r'[^A-Za-z0-9,?.!]+', ' ', text) # remove all symbols and punctuation except for . , ! and ?\n",
    "    return text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iCE-ztcy0RFq"
   },
   "outputs": [],
   "source": [
    "nlp = spacy.load('en',disable=['parser', 'tagger', 'ner'])\n",
    "def tokenizer(s): return [w.text.lower() for w in nlp(tweet_clean(s))]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "l5c4RdL90Wgu"
   },
   "source": [
    "**Define fields**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gFlQw-Pt05eH"
   },
   "outputs": [],
   "source": [
    "TEXT = Field(sequential=True, tokenize=tokenizer, include_lengths=True, use_vocab=True)\n",
    "TARGET = Field(sequential=False, use_vocab=False, pad_token=None, unk_token=None, is_target =False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LXS_2wDd0qQ9"
   },
   "outputs": [],
   "source": [
    "data_fields = [\n",
    "    (None, None),\n",
    "    (\"tweet\", TEXT), \n",
    "    (\"target\", TARGET)\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SJHSyAjrxsEc"
   },
   "source": [
    "## **3. Create Train, Valid and Test datasets**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MnN_75qpgMHN"
   },
   "outputs": [],
   "source": [
    "def split_train_test(df, test_size=0.2):\n",
    "    train, val = train_test_split(df, test_size=test_size,random_state=42)\n",
    "    return train.reset_index(drop=True), val.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HEFA4z2CE_Sq"
   },
   "outputs": [],
   "source": [
    "# create train and validation set \n",
    "train_val, test = split_train_test(df, test_size=0.2)\n",
    "train, val = split_train_test(train_val, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "19vn3wuLFeae"
   },
   "outputs": [],
   "source": [
    "train.to_csv(\"train.csv\", index=False)\n",
    "val.to_csv(\"val.csv\", index=False)\n",
    "test.to_csv(\"test.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "VFi9p4ibRvX0",
    "outputId": "6775feee-0dec-49c8-ac99-58ee4cd4daf3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1612, 3), (404, 3), (505, 3))"
      ]
     },
     "execution_count": 28,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape, val.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 314
    },
    "colab_type": "code",
    "id": "HC87TsYER-0Q",
    "outputId": "429f32a6-a9f8-4b3c-fab2-4e8cca19b0d1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Text(0, 0.5, 'counts'), Text(0.5, 0, 'Labels'), Text(0.5, 1.0, 'test')]"
      ]
     },
     "execution_count": 29,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm4AAAEWCAYAAADfMRsiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X2UZXV95/v3R0BQUXnqMC2Naa72\nNUGjLVNDyJjJEFB5GJPWuWogiXSQNa03mNHRUcF7Z8AoE3JHZSQTyWoEAUdFRDMyhsQQwGF5hwcb\n7CAP8dICSncauuVJiJGk8Xv/OL+CQ1HVnOquU6f2qfdrrbNq799+qG/Vqu/a3/rtvX+/VBWSJEla\n+J416gAkSZI0GAs3SZKkjrBwkyRJ6ggLN0mSpI6wcJMkSeoICzdJkqSOsHATAEn+JMl/GHUc0rAl\nOTzJxr71W5McPsi+O/C9zCtJc8rCbUwkuTvJa3f0+Kp6Z1V9ZC5jkrqgql5eVd/Y2fMk+Z0k35xy\nbvNKY2tnrzvtHE/LG22fhdsikGTXUccgSZJ2noXbGEjyWeDFwP9I8miSDySpJCcl+QFwVdvvS0nu\nTfJwkmuSvLzvHBck+WhbPjzJxiTvS7IlyeYkJ47kh5NmkOSDSS6d0vbJJGcnOTHJ7UkeSXJnknds\n5zxP9BokeU7LhQeT3Ab8syn7npLke+28tyV5U2v/eeBPgF9qOfhQa38ir9r6v0myIckDSS5L8qK+\nbZXknUnuSPJQkj9Okjn4VUlzbobrzmFJ/lf7+/3r/kcQWs/anS137kryWzPljbbPwm0MVNXbgB8A\nv1ZVewKXtE3/Evh54Ki2/ufACuBngJuAz23ntP8EeCFwAHAS8MdJ9p776KUddjFwbJLnAyTZBXgr\n8HlgC/AG4AXAicBZSQ4Z4JynAS9pn6OA1VO2fw/4F/Ry48PAf0uytKpuB94JXFtVe1bVXlNPnOQI\n4A9ajEuB77efod8b6BWLr2z7HYW0AE1z3fkc8GfAR4F9gH8PfDnJkiTPA84Gjqmq5wP/HFg/SN7o\n6SzcxtvpVfV3VfX3AFV1flU9UlWPAacDr0rywhmO/Ufg96vqH6vqcuBR4GXzErU0gKr6Pr1/QN7U\nmo4AflxV11XVn1XV96rnfwJ/Sa/geiZvBc6oqgeq6h56F5v+7/mlqvrbqvppVX0RuAM4dMCQfws4\nv6puajl4Kr2ehuV9+5xZVQ9V1Q+Aq4GVA55bGrXfBi6vqstbflwBrAOObdt/CrwiyXOqanNV3Tqy\nSDvOwm283TO5kGSXJGe22zw/Au5um/ab4dj7q2pb3/qPgT2HE6a0wz4PHN+Wf7Otk+SYJNe1W5IP\n0bt4zPS33u9F9OUNvV6xJyQ5Icn6divoIeAVA5538txPnK+qHgXup9erPenevmVzTl3ys8BbJnOj\n5ccvA0ur6u+A36DXu7Y5yZ8l+blRBttlFm7jo56h7TeBVcBr6d3mWd7afYZGXfYl4PAky+j1vH0+\nye7Al4GPAfu32y+XM9jf+mbgwL71F08uJPlZ4FzgXcC+7by39J13uhzs97f0Lm6T53sesC+waYC4\npIWo/2/+HuCzVbVX3+d5VXUmQFV9vapeR+8xgb+hl0tTz6EBWLiNj/uA/207258PPEbvP/znAv9p\nPoKShqmqtgLfAD4D3NWemXk2sDuwFdiW5Bjg9QOe8hLg1CR7t2Lw9/q2PY/eRWYrQHth5xV92+8D\nliV59gzn/gJwYpKVrbj8T8D1VXX3gLFJC03/dee/Ab+W5Kh2h2eP9qLbsiT7J1nV/ll5jN6jNz/t\nO8f28kZTWLiNjz8A/u/WPf3mabZfRO82zSbgNuC6eYxNGqbP0+tJ/jxAVT0C/Ft6RdiD9HqbLxvw\nXB+mlyd30Xsu7rOTG6rqNuDjwLX0Lja/APy/fcdeBdwK3Jvkh1NPXFV/BfwHer2Bm+m9AHHcgHFJ\nC1H/dec36N3V+RC9f27uAd5Pr854FvBeer3OD9B7ce7/bOfYbt7o6VJlL6UkSVIX2OMmSZLUERZu\nkiRJHWHhJkmS1BEWbpIkSR0xlpOP77fffrV8+fJRh6ExcOONN/6wqpaMOo75YN5orpg30uzMJmfG\nsnBbvnw569atG3UYGgNJvv/Me40H80ZzxbyRZmc2OeOtUknS2GqDwX47ydfa+kFJrk+yIckXJwd+\nTbJ7W9/Qti8fZdzSTCzcJEnj7N3A7X3rfwicVVUvpTdA80mt/STgwdZ+VttPWnAs3CRJY6lNW/av\ngE+39QBHAJe2XS4E3tiWV7V12vYj2/7SgmLhJkkaV/8F+ABPzou5L/BQVW1r6xuBA9ryAfSmaaJt\nf7jt/zRJ1iRZl2Td1q1bhxW7NC0LN0nS2EnyBmBLVd041+euqrVVNVFVE0uWLIqXZ7WAjOVbpZKk\nRe81wK8nORbYA3gB8ElgryS7tl61ZcCmtv8m4EBgY5JdgRcC989/2NL22eMmSRo7VXVqVS2rquXA\nccBVVfVbwNXAm9tuq4GvtuXL2jpt+1VVVfMYsjQQCzdJ0mLyQeC9STbQe4btvNZ+HrBva38vcMqI\n4pO2y1ulkqSxVlXfAL7Rlu8EDp1mn58Ab5nXwKQdsCgLt3/6/otGHcK8ufE/nzDqEDQGzBlp9swb\nDYO3SqUOSLJHkhuS/HWSW5N8uLU7CrwkLSIWblI3PAYcUVWvAlYCRyc5DEeBl6RFZWiFW5Lzk2xJ\ncktf239O8jdJbk7yp0n26tt2ausd+G6So/raj25tG5L4sKgWpep5tK3u1j6Fo8BL0qIyzB63C4Cj\np7RdAbyiql4J/H/AqQBJDqb3uvbL2zGfahMD7wL8MXAMcDBwfNtXWnRaTqwHttDLpe+xk6PAOwK8\nJHXL0Aq3qroGeGBK21/2XWSuozf4IfR6By6uqseq6i5gA723fg4FNlTVnVX1D8DFbV9p0amqx6tq\nJb28ORT4uTk4pyPAS1KHjPIZt7cDf96Wn+gdaCZ7DmZqlxatqnqI3iCiv0QbBb5tmm4UeBwFXpLG\nx0gKtyT/F7AN+NwcntNbPhpbSZZMPhOa5DnA64DbcRR4SVpU5n0ctyS/A7wBOLLvQvJE70DT33Mw\nU/tTVNVaYC3AxMSEFyiNm6XAhe25z2cBl1TV15LcBlyc5KPAt3nqKPCfbaPAP0DvGVJJUsfNa+GW\n5GjgA8C/rKof9226DPh8kk8ALwJWADcAAVYkOYhewXYc8JvzGbO0EFTVzcCrp2l3FHhJWkSGVrgl\n+QJwOLBfko3AafTeIt0duKKNTHBdVb2zqm5NcglwG71bqCdX1ePtPO8Cvg7sApxfVbcOK2ZJkqSF\nbGiFW1UdP03zedO0Te5/BnDGNO2XA5fPYWiSJEmd5MwJkiRJHWHhJkmS1BEWbpIkSR1h4SZJktQR\nFm6SJEkdYeEmSRo7SfZIckOSv05ya5IPt/YLktyVZH37rGztSXJ2kg1Jbk5yyGh/Aml68z5zgiRJ\n8+Ax4IiqejTJbsA3k0zOj/3+qrp0yv7H0Bv8fQXwi8A57au0oNjjJkkaO9XzaFvdrX22Nx3iKuCi\ndtx1wF5Jlg47Tmm2LNwkSWMpyS5J1gNbgCuq6vq26Yx2O/SsJLu3tgOAe/oO39japjvvmiTrkqzb\nunXr0OKXpmPhJkkaS1X1eFWtBJYBhyZ5Bb2pF38O+GfAPsAHd+C8a6tqoqomlixZMqcxS8/Ewk2S\nNNaq6iHgauDoqtrcboc+BnwGOLTttgk4sO+wZa1NWlAs3CRJYyfJkiR7teXnAK8D/mbyubUkAd4I\n3NIOuQw4ob1dehjwcFVtHkHo0nb5VqkkaRwtBS5Msgu9TopLquprSa5KsgQIsB54Z9v/cuBYYAPw\nY+DEEcQsPSMLN0nS2Kmqm4FXT9N+xAz7F3DysOOSdpa3SiVJkjrCwk2SJKkjLNwkSZI6wsJNkiSp\nIyzcJEmSOsLCTZIkqSMs3CRJkjrCwk2SJKkjLNykDkhyYJKrk9yW5NYk727tpyfZlGR9+xzbd8yp\nSTYk+W6So0YXvSRprjhzgtQN24D3VdVNSZ4P3JjkirbtrKr6WP/OSQ4GjgNeDrwI+Ksk/3tVPT6v\nUUuS5pQ9blIHVNXmqrqpLT8C3A4csJ1DVgEXV9VjVXUXvfkXDx1+pJKkYRpa4Zbk/CRbktzS17ZP\nkiuS3NG+7t3ak+Tsdlvn5iSH9B2zuu1/R5LVw4pX6ooky+nNwXh9a3pXy5vzJ3OKXlF3T99hG9l+\noSdJ6oBh9rhdABw9pe0U4MqqWgFc2dYBjgFWtM8a4BzoFXrAacAv0ustOK3vwiQtOkn2BL4MvKeq\nfkQvV14CrAQ2Ax+f5fnWJFmXZN3WrVvnPF5J0twaWuFWVdcAD0xpXgVc2JYvBN7Y135R9VwH7JVk\nKXAUcEVVPVBVDwJX8PRiUFoUkuxGr2j7XFV9BaCq7quqx6vqp8C5PHk7dBNwYN/hy1rbU1TV2qqa\nqKqJJUuWDPcHkCTttPl+xm3/qtrclu8F9m/LM93WGfh2jz0HGmdJApwH3F5Vn+hrX9q325uAyUcT\nLgOOS7J7koPo9WbfMF/xSpKGY2RvlVZVJak5PN9aYC3AxMTEnJ1XWiBeA7wN+E6S9a3tQ8DxSVYC\nBdwNvAOgqm5NcglwG703Uk/2jVJJ6r75LtzuS7K0qja3noItrX2m2zqbgMOntH9jHuKUFpSq+iaQ\naTZdvp1jzgDOGFpQkqR5N9+3Si8DJt8MXQ18ta/9hPZ26WHAw+2W6teB1yfZu72U8PrWJkmStOgM\ncziQLwDXAi9LsjHJScCZwOuS3AG8tq1Dr9fgTnpjTZ0L/C5AVT0AfAT4Vvv8fmuTJGm7kuyR5IYk\nf91mHPlwaz8oyfVtCKovJnl2a9+9rW9o25ePMn5pOkO7VVpVx8+w6chp9i3g5BnOcz5w/hyGJkla\nHB4DjqiqR9tb2d9M8ufAe+nNOHJxkj8BTqI3tM5JwINV9dIkxwF/CPzGqIKXpuPMCZKksdSGmHq0\nre7WPgUcAVza2qcOTTU5ZNWlwJHtjW5pwbBwkySNrSS7tDext9AbC/R7wENVta3t0j/M1BNDULXt\nDwP7TnNOh5/SyFi4SZLGVhugeiW9UQkOBX5uDs7pwNUaGQs3SdLYq6qHgKuBX6I3O8/kM979s4o8\nMTRV2/5C4P55DlXaLgs3SdJYSrIkyV5t+TnA64Db6RVwb267TR2aanLIqjcDV7WX56QFY2QzJ0iS\nNGRLgQuT7EKvo+KSqvpaktuAi5N8FPg2venkaF8/m2QDvbm2jxtF0NL2WLhJksZSVd0MvHqa9jvp\nPe82tf0nwFvmITRph3mrVJIkqSMs3CRJkjrCwk2SJKkjLNwkSZI6wsJNkiSpIyzcJEmSOsLCTZIk\nqSMs3CRJkjrCwk2SJKkjLNwkSZI6wsJNkiSpIyzcJEmSOsLCTZIkqSMs3CRJkjrCwk2SJKkjLNyk\nDkhyYJKrk9yW5NYk727t+yS5Iskd7everT1Jzk6yIcnNSQ4Z7U8gSZoLFm5SN2wD3ldVBwOHAScn\nORg4BbiyqlYAV7Z1gGOAFe2zBjhn/kOWJM21kRRuSf5d6zW4JckXkuyR5KAk17cegi8meXbbd/e2\nvqFtXz6KmKVRqqrNVXVTW34EuB04AFgFXNh2uxB4Y1teBVxUPdcBeyVZOs9hS5Lm2LwXbkkOAP4t\nMFFVrwB2AY4D/hA4q6peCjwInNQOOQl4sLWf1faTFq32z8urgeuB/atqc9t0L7B/Wz4AuKfvsI2t\nbeq51iRZl2Td1q1bhxazNN+283jB6Uk2JVnfPsf2HXNq6yT4bpKjRhe9NLNR3SrdFXhOkl2B5wKb\ngSOAS9v2qT0Hkz0KlwJHJsk8xiotGEn2BL4MvKeqftS/raoKqNmcr6rWVtVEVU0sWbJkDiOVRm6m\nxwug10mwsn0uB2jbjgNeDhwNfCrJLqMIXNqeeS/cqmoT8DHgB/QKtoeBG4GHqmpb262/d+CJnoO2\n/WFg36nntedA4y7JbvSKts9V1Vda832Tt0Db1y2tfRNwYN/hy1qbtChs5/GCmawCLq6qx6rqLmAD\ncOjwI5VmZxS3SvemlyAHAS8Cnkfvv5udYs+BxlnrZT4PuL2qPtG36TJgdVteDXy1r/2E9nbpYcDD\nfbdUpUVlyuMFAO9qb1ufP/kmNgM+XtDOZ0eBRmYUt0pfC9xVVVur6h+BrwCvoffw9K5tn/7egSd6\nDtr2FwL3z2/I0si9BngbcMSUZ3POBF6X5A56uXVm2/9y4E56vQbnAr87gpilkZvm8YJzgJcAK+nd\n9fn4bM9pR4FGaddn3mXO/QA4LMlzgb8HjgTWAVcDbwYu5uk9B6uBa9v2q9qzPNKiUVXfBGZ6tvPI\nafYv4OShBiUtcNM9XlBV9/VtPxf4Wlv18QJ1wiiecbue3ksGNwHfaTGsBT4IvDfJBnrPsJ3XDjkP\n2Le1v5cnx6mSJGlaMz1eMGVYnDcBt7Tly4Dj2hBUB9EbA/GG+YpXGtQoetyoqtOA06Y038k0D4JW\n1U+At8xHXJKksTH5eMF3kqxvbR8Cjk+ykt4b2HcD7wCoqluTXALcRu+N1JOr6vF5j1p6BiMp3CRJ\nGqbtPF5w+XaOOQM4Y2hBSXPAKa8kSZI6wsJNkiSpIyzcJEmSOsLCTZIkqSMs3CRJkjpioMItybuT\nvKBNn3NekpuSvH7YwUnjyHySZseckZ40aI/b29tUIa8H9qY3Ns6Z2z9E0gzMJ2l2zBmpGbRwmxwL\n51jgs1V1KzNPvyNp+8wnaXbMGakZtHC7Mclf0kuaryd5PvDT4YUljTXzSZodc0ZqBp054SRgJXBn\nVf04yb7AicMLSxpr5pM0O+aM1Aza43ZFVd1UVQ8BVNX9wFnDC0saa+aTNDvmjNRst8ctyR7Ac4H9\nkuzNk88UvAA4YMixSWPFfJJmx5yRnu6ZbpW+A3gP8CLgRp5Mmh8B/3WIcUnjyHySZseckabYbuFW\nVZ8EPpnk96rqj+YpJmksmU/S7Jgz0tMN9HJCVf1Rkn8OLO8/pqouGlJc0tgyn6TZMWekJw1UuCX5\nLPASYD3weGsuwKSRZsl8kmbHnJGeNOhwIBPAwVVVwwxGWiTMJ2l2zBmpGXQ4kFuAfzLMQKRFxHyS\nZmfWOZPkwCRXJ7ktya1J3t3a90lyRZI72te9W3uSnJ1kQ5KbkxwyhJ9D2mmD9rjtB9yW5AbgscnG\nqvr1oUQljTfzSZqdHcmZbcD7quqmNtPCjUmuAH4HuLKqzkxyCnAK8EHgGGBF+/wicE77Ki0ogxZu\npw8zCGmROX3UAUgdc/psD6iqzcDmtvxIktvpjf22Cji87XYh8A16hdsq4KJ2O/a6JHslWdrOIy0Y\ng75V+j+HHYi0WJhP0uzsbM4kWQ68Grge2L+vGLsX2L8tHwDc03fYxtb2tMItyRpgDcCLX/zinQlN\nmrWBnnFL8kiSH7XPT5I8nuRHww5OGkfmkzQ7O5MzSfYEvgy8p6qeckzrXZv1Cw9VtbaqJqpqYsmS\nJbM9XNopAxVuVfX8qnpBVb0AeA7wfwCfGmpk0pjakXxKcn6SLUlu6Ws7PcmmJOvb59i+bae2h6y/\nm+Soof0w0jzY0WtQkt3oFW2fq6qvtOb7kixt25cCW1r7JuDAvsOXtTZpQRn0rdInVM9/B7wYSDtp\nFvl0AXD0NO1nVdXK9rkcIMnBwHHAy9sxn0qyyxyGLY3MoDmTJMB5wO1V9Ym+TZcBq9vyauCrfe0n\ntLdLDwMe9vk2LUSDDsD7r/tWn0VvTJ2f7Og3TbIX8GngFfS6qd8OfBf4Ir2Rse8G3lpVD7bk+yRw\nLPBj4Heq6qYd/d7SqO1IPlXVNe05nUGsAi6uqseAu5JsAA4Frp19tNLo7eA16DXA24DvJFnf2j4E\nnAlckuQk4PvAW9u2y+ldZzbQu9acODfRS3Nr0LdKf61veRu9wmrVTnzfTwJ/UVVvTvJs4Ln0EspX\ntLUYzGU+vSvJCcA6ekMfPEjvgerr+vaZfMj6aXzIWh0x65ypqm/y5KT0Ux05zf4FnLyD8UnzZtC3\nSufsP48kLwR+hd5YOlTVPwD/kMRXtLUozGE+nQN8hF6v9UeAj9PrvZ5NLGuBtQATExOOSq8FaS6v\nQVLXDfpW6bIkf9oejt6S5MtJlu3g9zwI2Ap8Jsm3k3w6yfOY/SvaU2Nck2RdknVbt27dwdCk4Zur\nfKqq+6rq8ar6KXAuvduh4EPWGjNzfA2SOm3QlxM+Q+/BzRe1z/9obTtiV+AQ4JyqejXwd/Ruiz5h\nR17R9vVsdcic5NPkm3HNm+hNC0Q793FJdk9yEL3HDG7YqYil0ZrLa5DUaYMWbkuq6jNVta19LgB2\ntDraCGysquvb+qX0Cjlf0dZiMet8SvIFei8XvCzJxvZg9f+T5DtJbgZ+Ffh3AFV1K3AJcBvwF8DJ\nVfX4EH8eadjm8hokddqghdv9SX47yS7t89vA/TvyDavqXuCeJC9rTUfSu8D4irYWi1nnU1UdX1VL\nq2q3qlpWVedV1duq6heq6pVV9ev9eVFVZ1TVS6rqZVX150P/iaThmrNrkNR1g75V+nbgj4Cz6N3C\n/F+0lwt20O8Bn2tvlN5J77XrZ+Er2loc5jqfpHFnzkjNoIXb7wOr21ADJNkH+BizfINtUlWtpzcO\nz1S+oq3FYE7zSVoEzBmpGfRW6SsnEwagqh6gN2GvpNkzn6TZMWekZtDC7VlJ9p5caf/tDNpbJ+mp\nzCdpdswZqRn0D//jwLVJvtTW3wKcMZyQpLFnPkmzY85IzaAzJ1yUZB1wRGv611V12/DCksaX+STN\njjkjPWngruaWJCaKNAfMJ2l2zBmpZ9Bn3CRJkjRiFm6SJEkdYeEmSZLUERZukiRJHWHhJkmS1BEW\nbpIkSR1h4SZJGktJzk+yJcktfW2nJ9mUZH37HNu37dQkG5J8N8lRo4la2j4LN0nSuLoAOHqa9rOq\namX7XA6Q5GDgOODl7ZhPJdll3iKVBmThJkkaS1V1DfDAgLuvAi6uqseq6i5gA3Do0IKTdpCFmyRp\nsXlXkpvbrdTJyesPAO7p22dja3uaJGuSrEuybuvWrcOOVXoKCzdJ0mJyDvASYCWwmd4E9rNSVWur\naqKqJpYsWTLX8UnbZeEmSVo0quq+qnq8qn4KnMuTt0M3AQf27bqstUkLioWbJGnRSLK0b/VNwOQb\np5cBxyXZPclBwArghvmOT3omu446AEmShiHJF4DDgf2SbAROAw5PshIo4G7gHQBVdWuSS4DbgG3A\nyVX1+CjilrbHwk2SNJaq6vhpms/bzv5nAGcMLyJp53mrVJIkqSMs3CRJkjrCwk2SJKkjLNykDphh\nzsV9klyR5I72de/WniRntzkXb05yyOgilyTNpZEVbkl2SfLtJF9r6wclub5dbL6Y5Nmtffe2vqFt\nXz6qmKURuoCnz7l4CnBlVa0ArmzrAMfQG8pgBbCG3oCjkqQxMMoet3cDt/et/yG9iX9fCjwInNTa\nTwIebO1ntf2kRWWGORdXARe25QuBN/a1X1Q91wF7TRm7SpLUUSMp3JIsA/4V8Om2HuAI4NK2y9SL\n0OTF6VLgyLa/tNjtX1Wb2/K9wP5t2TkXJWlMjWoct/8CfAB4flvfF3ioqra19f4LzRMXoaraluTh\ntv8P+0+YZA2920K8+MUvHmrw0kJTVZWkduC4tcBagImJiVkfr6f6we//wqhDmDcv/o/fGXUI0qI0\n7z1uSd4AbKmqG+fyvE76q0XovslboO3rltbunIuSNKZGcav0NcCvJ7kbuJjeLdJP0nsOZ7IHsP9C\n88RFqG1/IXD/fAYsLVCXAavb8mrgq33tJ7S3Sw8DHu67pSpJ6rB5L9yq6tSqWlZVy4HjgKuq6reA\nq4E3t92mXoQmL05vbvt7S0eLSptz8VrgZUk2JjkJOBN4XZI7gNe2dYDLgTuBDcC5wO+OIGRJ0hAs\npLlKPwhcnOSjwLd5cj6584DPJtlA762640YUnzQyM8y5CHDkNPsWcPJwI5IkjcJIC7eq+gbwjbZ8\nJ3DoNPv8BHjLvAYmH7KWJA2d15rZc+YESZKkjrBwkyRJ6ggLN0mSpI6wcJMkSeoICzdJ0lhKcn6S\nLUlu6WvbJ8kVSe5oX/du7UlydpINSW5OcsjoIpdmZuEmSRpXFwBHT2k7BbiyqlYAV7Z1gGOAFe2z\nBjhnnmKUZsXCTZI0lqrqGnrjf/ZbBVzYli8E3tjXflH1XEdvNp+l8xOpNDgLN0nSYrJ/3xRw9wL7\nt+UDgHv69tvY2qQFxcJNkrQotVlGZj2FYpI1SdYlWbd169YhRCbNzMJNkrSY3Dd5C7R93dLaNwEH\n9u23rLU9TVWtraqJqppYsmTJUIOVprJwkyQtJpcBq9vyauCrfe0ntLdLDwMe7rulKi0YC2mSeUmS\n5kySLwCHA/sl2QicBpwJXJLkJOD7wFvb7pcDxwIbgB8DJ857wNIALNwkSWOpqo6fYdOR0+xbwMnD\njUjaed4qlSRJ6ggLN0mSpI6wcJMkSeoICzdJkqSOsHCTJEnqCAs3SZKkjrBwkyRJ6ggLN0mSpI6w\ncJMkSeoICzdJkqSOsHCTJEnqiHkv3JIcmOTqJLcluTXJu1v7PkmuSHJH+7p3a0+Ss5NsSHJzkkPm\nO2ZpIUtyd5LvJFmfZF1rmzafJEndNooet23A+6rqYOAw4OQkBwOnAFdW1QrgyrYOcAywon3WAOfM\nf8jSgverVbWyqiba+kz5JEnqsHkv3Kpqc1Xd1JYfAW4HDgBWARe23S4E3tiWVwEXVc91wF5Jls5z\n2FLXzJRPkqQOG+kzbkmWA68Grgf2r6rNbdO9wP5t+QDgnr7DNra2qedak2RdknVbt24dWszSAlTA\nXya5Mcma1jZTPj2FeSNJ3TKywi3JnsCXgfdU1Y/6t1VV0bsYDayq1lbVRFVNLFmyZA4jlRa8X66q\nQ+g9VnBykl/p37i9fDJvJKlbRlK4JdmNXtH2uar6Smu+b/IWaPu6pbVvAg7sO3xZa5MEVNWm9nUL\n8KfAocycT5KkDhvFW6UBzgNur6pP9G26DFjdllcDX+1rP6G9XXoY8HDfLSBpUUvyvCTPn1wGXg/c\nwsz5JEnqsF1H8D1fA7wN+E7xxDOMAAAFeUlEQVSS9a3tQ8CZwCVJTgK+D7y1bbscOBbYAPwYOHF+\nw5UWtP2BP+39P8SuwOer6i+SfIvp80kSvWF0gEeAx4FtVTWRZB/gi8By4G7grVX14KhilKYz74Vb\nVX0TyAybj5xm/wJOHmpQUkdV1Z3Aq6Zpv59p8knSU/xqVf2wb31yGJ0zk5zS1j84mtCk6TlzgiRJ\nPQ6jowXPwk2StBg5jI46aRTPuEmSNGq/XFWbkvwMcEWSv+nfWFWVZMZhdIC1ABMTE7MaukraWfa4\nSZIWHYfRUVdZuEmSFhWH0VGXeatUkrTYOIyOOsvCTZK0qDiMjrrMW6WSJEkdYeEmSZLUERZukiRJ\nHWHhJkmS1BEWbpIkSR1h4SZJktQRFm6SJEkdYeEmSZLUERZukiRJHWHhJkmS1BEWbpIkSR1h4SZJ\nktQRFm6SJEkdYeEmSZLUERZukiRJHWHhJkmS1BEWbpIkSR1h4SZJktQRnSnckhyd5LtJNiQ5ZdTx\nSAudOSPNnnmjha4ThVuSXYA/Bo4BDgaOT3LwaKOSFi5zRpo980Zd0InCDTgU2FBVd1bVPwAXA6tG\nHJO0kJkz0uyZN1rwdh11AAM6ALinb30j8Iv9OyRZA6xpq48m+e48xTao/YAfzvc3zcdWz/e33Fnz\n/3s6Ldvb+rPzFcYce8acAfNmOubMgMybhZo3XmsG09lrTVcKt2dUVWuBtaOOYyZJ1lXVxKjjWOj8\nPc0v86b7/B3Nv4WcN/49DKbLv6eu3CrdBBzYt76stUmanjkjzZ55owWvK4Xbt4AVSQ5K8mzgOOCy\nEcckLWTmjDR75o0WvE7cKq2qbUneBXwd2AU4v6puHXFYs7Ugu9UXIH9Pc2BMcgb8exiEv6M5MiZ5\n49/DYDr7e0pVjToGSZIkDaArt0olSZIWPQs3SZKkjrBwmwdOofLMkpyfZEuSW0Ydi0bPnHlm5oym\nMm+e2TjkjYXbkDmFysAuAI4edRAaPXNmYBdgzqgxbwZ2AR3PGwu34XMKlQFU1TXAA6OOQwuCOTMA\nc0ZTmDcDGIe8sXAbvummUDlgRLFIXWDOSLNn3iwSFm6SJEkdYeE2fE6hIs2OOSPNnnmzSFi4DZ9T\nqEizY85Is2feLBIWbkNWVduAySlUbgcu6eAUKkOX5AvAtcDLkmxMctKoY9JomDODMWfUz7wZzDjk\njVNeSZIkdYQ9bpIkSR1h4SZJktQRFm6SJEkdYeEmSZLUERZukiRJHWHh1nFJHp3Fvqcn+ffDOr/U\nBeaMNHvmzcJh4SZJktQRFm5jKMmvJbk+ybeT/FWS/fs2vyrJtUnuSPJv+o55f5JvJbk5yYenOefS\nJNckWZ/kliT/Yl5+GGkemDPS7Jk3o2HhNp6+CRxWVa8GLgY+0LftlcARwC8B/zHJi5K8HlgBHAqs\nBP5pkl+Zcs7fBL5eVSuBVwHrh/wzSPPJnJFmz7wZgV1HHYCGYhnwxSRLgWcDd/Vt+2pV/T3w90mu\nppdAvwy8Hvh222dPesl1Td9x3wLOT7Ib8N+rymTSODFnpNkzb0bAHrfx9EfAf62qXwDeAezRt23q\nHGcFBPiDqlrZPi+tqvOeslPVNcCvAJuAC5KcMLzwpXlnzkizZ96MgIXbeHohvT96gNVTtq1KskeS\nfYHD6f1383Xg7Un2BEhyQJKf6T8oyc8C91XVucCngUOGGL8038wZafbMmxHwVmn3PTfJxr71TwCn\nA19K8iBwFXBQ3/abgauB/YCPVNXfAn+b5OeBa5MAPAr8NrCl77jDgfcn+ce23f+C1FXmjDR75s0C\nkaqpvZmSJElaiLxVKkmS1BEWbpIkSR1h4SZJktQRFm6SJEkdYeEmSZLUERZukiRJHWHhJkmS1BH/\nP71ova2JZQVRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x288 with 3 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(10,4))\n",
    "fig.subplots_adjust(hspace=0.4, wspace=0.4)\n",
    "\n",
    "ax = fig.add_subplot(1,3,1)\n",
    "ax = sns.barplot(x=train.target.unique(),y=train.target.value_counts())\n",
    "ax.set(xlabel='Labels', ylabel=\"counts\", title=\"train\")\n",
    "\n",
    "ax1 = fig.add_subplot(1,3,2)\n",
    "ax1 = sns.barplot(x=val.target.unique(),y=val.target.value_counts())\n",
    "ax1.set(xlabel='Labels', ylabel=\"counts\", title=\"validation\")\n",
    "\n",
    "ax2 = fig.add_subplot(1,3,3)\n",
    "ax2 = sns.barplot(x=test.target.unique(),y=test.target.value_counts())\n",
    "ax2.set(xlabel='Labels', ylabel=\"counts\", title=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "uswTGPRH2v6J",
    "outputId": "c0867882-56cf-4cc6-f566-7b06a9e0fe66"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.75 s, sys: 52 ms, total: 1.8 s\n",
      "Wall time: 1.76 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_data, val_data, test_data = TabularDataset.splits(path='./', format='csv', train='train.csv', validation='val.csv', test='test.csv', fields=data_fields, skip_header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "eSgxCIC23keA",
    "outputId": "e9018a7f-76c2-4d40-b148-5d05c4e3865f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1612, 404, 505)"
      ]
     },
     "execution_count": 31,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_data), len(val_data), len(test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "z54cb2_B4b79"
   },
   "source": [
    "## **4. Load pretrained embeddings and build vocab**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "n-tsm29k_WTq",
    "outputId": "58cded06-7fee-4c04-cb03-92531d227e4e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "glove.6B.100d.txt  glove.twitter.27B.100d.txt\t  uncased_L-12_H-768_A-12.zip\n",
      "glove.6B.200d.txt  glove.twitter.27B.100d.txt.pt\n"
     ]
    }
   ],
   "source": [
    "!ls '/content/gdrive/My Drive/embedding'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "9GfaJCDJEBK7",
    "outputId": "9a37baf2-ae28-448b-a672-ac30d9241f6c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 532 ms, sys: 644 ms, total: 1.18 s\n",
      "Wall time: 1.45 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "vec = torchtext.vocab.Vectors('glove.twitter.27B.100d.txt', '/content/gdrive/My Drive/embedding')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "w8HuaHBEod81",
    "outputId": "c3fbff7a-33e2-4562-af45-3711f353e421"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1612"
      ]
     },
     "execution_count": 34,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "jGAbh_g2QxAt",
    "outputId": "b14ed9d5-7bb8-41a3-b803-e3aa2084116d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 98.7 ms, sys: 2.02 ms, total: 101 ms\n",
      "Wall time: 106 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "MAX_VOCAB_SIZE = 100_000\n",
    "\n",
    "TEXT.build_vocab(train_data, \n",
    "                 max_size = MAX_VOCAB_SIZE,\n",
    "                 vectors=vec)\n",
    "\n",
    "TARGET.build_vocab(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "IaAJsJviSrr2",
    "outputId": "2bd84a30-0f59-4364-848e-83243bfd0110"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4577, 100])"
      ]
     },
     "execution_count": 36,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT.vocab.vectors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "g3vwetTcG9mx",
    "outputId": "8b888140-294f-4b80-d79e-899224ac777a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torchtext.data.dataset.TabularDataset at 0x7f6534bbfa90>"
      ]
     },
     "execution_count": 37,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PrF-Wlbi5czS"
   },
   "source": [
    "## **5. Load data in batches**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uwBCojaA-bk7"
   },
   "source": [
    "We will use the BucketIterator to access the Dataloader. It sorts data according to length of text, and groups similar length text in a batch, thus reducing the amount of padding required. It pads the batch according to the max length in that particular batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "25yseUiWKwfC"
   },
   "outputs": [],
   "source": [
    "\n",
    "train_loader, val_loader, test_loader = BucketIterator.splits(datasets=(train_data, val_data, test_data), \n",
    "                                            batch_sizes=(3,3,3), \n",
    "                                            sort_key=lambda x: len(x.tweet), \n",
    "                                            device=None, \n",
    "                                            sort_within_batch=True, \n",
    "                                            repeat=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "B-xV8SAC7dZ7",
    "outputId": "6b23d716-da32-4398-be1a-be5d3a6d1715"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(538, 135, 169)"
      ]
     },
     "execution_count": 39,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_loader), len(val_loader), len(test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "d_p56mLY71w7",
    "outputId": "66aa47a0-9693-401c-cf34-e97b695b373e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torchtext.data.batch.Batch"
      ]
     },
     "execution_count": 40,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch = next(iter(train_loader))\n",
    "type(batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "dMp72yoo7-70",
    "outputId": "3dea4de7-b939-42ce-b948-30e3b4424a03"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 0, 0])"
      ]
     },
     "execution_count": 41,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 126
    },
    "colab_type": "code",
    "id": "0W5vLVNy81Lj",
    "outputId": "2bdede4f-5d0b-4db3-8d8b-e00ba13f5753"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[  28, 1512,  113],\n",
       "         [ 143,  118,  682],\n",
       "         [ 392,  670,  838],\n",
       "         [1771,    5,   13],\n",
       "         [  14, 1414,   57],\n",
       "         [1137, 1258,  160]]), tensor([6, 6, 6]))"
      ]
     },
     "execution_count": 42,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch.tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "d3KJ6U2R9Lbz",
    "outputId": "929d6c62-62d1-43af-80cc-8aeb17bcc554"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<pad>'"
      ]
     },
     "execution_count": 43,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT.vocab.itos[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NfABZ4WR9nZT"
   },
   "outputs": [],
   "source": [
    "def idxtosent(batch, idx):\n",
    "    return ' '.join([TEXT.vocab.itos[i] for i in batch.tweet[0][:,idx].cpu().data.numpy()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "fj5ZIbUC9ydr",
    "outputId": "a03b209e-4a03-4421-9d36-9330120a0b7c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'do nt worry abt it bb'"
      ]
     },
     "execution_count": 45,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idxtosent(batch,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 235
    },
    "colab_type": "code",
    "id": "kzTDu3aR90zC",
    "outputId": "732bd3a0-29be-436e-d4dc-ef6aa654f56b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'batch_size': 3,\n",
       " 'dataset': <torchtext.data.dataset.TabularDataset at 0x7f6534bbfa90>,\n",
       " 'fields': dict_keys([None, 'tweet', 'target']),\n",
       " 'input_fields': ['tweet', 'target'],\n",
       " 'target': tensor([0, 0, 0]),\n",
       " 'target_fields': [],\n",
       " 'tweet': (tensor([[  28, 1512,  113],\n",
       "          [ 143,  118,  682],\n",
       "          [ 392,  670,  838],\n",
       "          [1771,    5,   13],\n",
       "          [  14, 1414,   57],\n",
       "          [1137, 1258,  160]]), tensor([6, 6, 6]))}"
      ]
     },
     "execution_count": 46,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch.__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ClH37CUB-GXT"
   },
   "outputs": [],
   "source": [
    "class BatchGenerator:\n",
    "    def __init__(self, dl, x_field, y_field):\n",
    "        self.dl, self.x_field, self.y_field = dl, x_field, y_field\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.dl)\n",
    "    \n",
    "    def __iter__(self):\n",
    "        for batch in self.dl:\n",
    "            X = getattr(batch, self.x_field)\n",
    "            y = getattr(batch, self.y_field)\n",
    "            yield (X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 417
    },
    "colab_type": "code",
    "id": "bYWXjqv--UEF",
    "outputId": "ce4c846d-d688-4d8b-c2f3-5a135c62cde9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((tensor([[ 227,    3,  665],\n",
       "          [2795,   64, 1387],\n",
       "          [4003,    3, 1181],\n",
       "          [  94, 1038, 1760],\n",
       "          [3846,   42, 1095],\n",
       "          [   4,    7, 1179],\n",
       "          [3867,  554,   30],\n",
       "          [ 430, 3337,  230],\n",
       "          [   4,    2, 1316],\n",
       "          [1140,   37,    2],\n",
       "          [   6,    3,    8],\n",
       "          [1752,   35,  401],\n",
       "          [  16,   28, 1433],\n",
       "          [ 267,   15,   29],\n",
       "          [2268,   64,   20],\n",
       "          [3689,    3,  404],\n",
       "          [ 169,   23,  665],\n",
       "          [ 385, 3854,    2],\n",
       "          [   9,    4,  168],\n",
       "          [1368,  222,   49],\n",
       "          [ 727,  964, 1305],\n",
       "          [ 720,    2,    2]]), tensor([22, 22, 22])), tensor([0, 1, 0]))"
      ]
     },
     "execution_count": 48,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_batch_it = BatchGenerator(train_loader, 'tweet', 'target')\n",
    "next(iter(train_batch_it))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Tj_aJPx__mv5"
   },
   "source": [
    "## **6. Models and Training**\n",
    "\n",
    "For the model, we decided to follow the example in https://medium.com/@sonicboom8/sentiment-analysis-torchtext-55fb57b1fab8, but make small modifications such as adding some dropout, to prevent overfitting\n",
    "\n",
    "The model is uses a pre-trained embedding layer from glove, a bidirectional GRU and also a concat pooling method where we perform average pool and max pool and then concatenate the results.\n",
    "\n",
    "The final result was ok, with around 80% test accuracy. It was clear that the model was overfitting but we had run out of time to make further adjustments. This has been a very educational experience as it was our first time implementing NLP using PyTorch. We plan to experiment and improve the model using a varitey of methods in the future."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eMfwXhUUGBcD"
   },
   "outputs": [],
   "source": [
    "vocab_size = len(TEXT.vocab)\n",
    "embedding_dim = 100\n",
    "n_hidden = 64\n",
    "n_out = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-puOJHXXkcL0"
   },
   "outputs": [],
   "source": [
    "# Define a PyTorch module named ConcatPoolingGRUAdaptive\n",
    "class ConcatPoolingGRUAdaptive(nn.Module):\n",
    "    \n",
    "    # Constructor with the following arguments\n",
    "    def __init__(self, vocab_size, embedding_dim, n_hidden, n_out, pretrained_vec, dropout, bidirectional=True):\n",
    "        \n",
    "        # Call the constructor of the nn.Module class\n",
    "        super().__init__()\n",
    "        \n",
    "        # Initialize instance variables\n",
    "        self.vocab_size = vocab_size\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.n_hidden = n_hidden\n",
    "        self.n_out = n_out\n",
    "        self.bidirectional = bidirectional\n",
    "        \n",
    "        # Create an embedding layer with a size of vocab_size x embedding_dim\n",
    "        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)\n",
    "        \n",
    "        # Load pre-trained word embeddings into the embedding layer\n",
    "        self.emb.weight.data.copy_(pretrained_vec)\n",
    "        \n",
    "        # Freeze the embedding layer during training\n",
    "        self.emb.weight.requires_grad = False\n",
    "        \n",
    "        # Create a GRU layer with input size of embedding_dim and hidden size of n_hidden\n",
    "        self.gru = nn.GRU(self.embedding_dim, self.n_hidden, bidirectional=bidirectional)\n",
    "        \n",
    "        # Create a fully-connected linear layer to map GRU output to class scores\n",
    "        if bidirectional:\n",
    "            self.fc = nn.Linear(self.n_hidden*2*2, self.n_out)\n",
    "        else:\n",
    "            self.fc = nn.Linear(self.n_hidden*2, self.n_out)\n",
    "        \n",
    "        # Create a dropout layer with dropout probability of dropout\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "    \n",
    "    # Define the forward method for the module\n",
    "    def forward(self, seq, lengths):\n",
    "        bs = seq.size(1)\n",
    "        \n",
    "        # Initialize the hidden state of the GRU\n",
    "        self.h = self.init_hidden(bs)\n",
    "        \n",
    "        # Transpose input sequence to batch-first format\n",
    "        seq = seq.transpose(0,1)\n",
    "        \n",
    "        # Pass input sequence through the embedding layer\n",
    "        embs = self.emb(seq)\n",
    "        \n",
    "        # Transpose embeddings back to sequence-first format\n",
    "        embs = embs.transpose(0,1)\n",
    "        \n",
    "        # Pack the sequence of embeddings using lengths to avoid computing on padded elements\n",
    "        embs = pack_padded_sequence(embs, lengths)\n",
    "        \n",
    "        # Pass packed embeddings through the GRU\n",
    "        gru_out, self.h = self.gru(embs, self.h)\n",
    "        \n",
    "        # Unpack the sequence and remove padding\n",
    "        gru_out, lengths = pad_packed_sequence(gru_out)        \n",
    "        \n",
    "        # Apply adaptive max and average pooling to the GRU output along the time dimension\n",
    "        avg_pool = F.adaptive_avg_pool1d(gru_out.permute(1,2,0),1).view(bs,-1)\n",
    "        max_pool = F.adaptive_max_pool1d(gru_out.permute(1,2,0),1).view(bs,-1) \n",
    "        \n",
    "        # Concatenate the average and max pooled outputs and apply dropout\n",
    "        cat = self.dropout(torch.cat([avg_pool,max_pool],dim=1))\n",
    "        \n",
    "        # Map the concatenated output to class scores\n",
    "        outp = self.fc(cat)\n",
    "        \n",
    "        # Apply log softmax to class scores and return\n",
    "        return F.log_softmax(outp)\n",
    "    \n",
    "    # Helper method to initialize the hidden state of the GRU\n",
    "    def init_hidden(self, batch_size): \n",
    "        if self.bidirectional:\n",
    "            return torch.zeros((2,batch_size,self.n_hidden)).to(device)\n",
    "        else:\n",
    "            return torch.zeros((1,batch_size,self.n_hidden)).cuda().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WlycNBIGGtO5"
   },
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion, num_batch):\n",
    "        y_true_train = list()\n",
    "        y_pred_train = list()\n",
    "        total_loss_train = 0  \n",
    "  \n",
    "        #t = tqdm_notebook(iterator, leave=False, total=num_batch)\n",
    "    \n",
    "        for (X,lengths),y in iterator:\n",
    "\n",
    "          #t.set_description(f'Epoch {epoch}')\n",
    "          lengths = lengths.cpu().numpy()\n",
    "\n",
    "          opt.zero_grad()\n",
    "          pred = model(X, lengths)\n",
    "          loss = criterion(pred, y)\n",
    "          loss.backward()\n",
    "          opt.step()\n",
    "\n",
    "          #t.set_postfix(loss=loss.item())\n",
    "          pred_idx = torch.max(pred, dim=1)[1]\n",
    "\n",
    "          y_true_train += list(y.cpu().data.numpy())\n",
    "          y_pred_train += list(pred_idx.cpu().data.numpy())\n",
    "          total_loss_train += loss.item()\n",
    "            \n",
    "        train_acc = accuracy_score(y_true_train, y_pred_train)\n",
    "        train_loss = total_loss_train/num_batch\n",
    "        return train_loss, train_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "j74LaFc6HlRU"
   },
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, criterion, num_batch):\n",
    "            y_true_val = list()\n",
    "            y_pred_val = list()\n",
    "            total_loss_val = 0\n",
    "            for (X,lengths),y in iterator: #tqdm_notebook(iterator, leave=False): \n",
    "            \n",
    "              pred = model(X, lengths.cpu().numpy())\n",
    "              loss = criterion(pred, y)\n",
    "              pred_idx = torch.max(pred, 1)[1]\n",
    "              y_true_val += list(y.cpu().data.numpy())\n",
    "              y_pred_val += list(pred_idx.cpu().data.numpy())\n",
    "              total_loss_val += loss.item()\n",
    "            valacc = accuracy_score(y_true_val, y_pred_val)\n",
    "            valloss = total_loss_val/num_batch\n",
    "            return valloss, valacc\n",
    "         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oq402kPY_086"
   },
   "outputs": [],
   "source": [
    "train_loader, val_loader, test_loader = BucketIterator.splits(datasets=(train_data, val_data, test_data), \n",
    "                                            batch_sizes=(32,32,32), \n",
    "                                            sort_key=lambda x: len(x.tweet), \n",
    "                                            device=device, \n",
    "                                            sort_within_batch=True, \n",
    "                                            repeat=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qOoo-vYpFLOi"
   },
   "outputs": [],
   "source": [
    "train_batch_it = BatchGenerator(train_loader, 'tweet', 'target')\n",
    "val_batch_it = BatchGenerator(val_loader, 'tweet', 'target')\n",
    "test_batch_it = BatchGenerator(test_loader, 'tweet', 'target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7c8x62WYvD4o"
   },
   "outputs": [],
   "source": [
    "m = ConcatPoolingGRUAdaptive(vocab_size, embedding_dim, n_hidden, n_out, train_data.fields['tweet'].vocab.vectors, 0.5).to(device)\n",
    "opt = optim.Adam(filter(lambda p: p.requires_grad, m.parameters()), 1e-3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "APwJlE7QPSO3"
   },
   "outputs": [],
   "source": [
    "loss_fn=F.nll_loss\n",
    "epochs=10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EGtekmstQVWo"
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 217
    },
    "colab_type": "code",
    "id": "5KK44r2qHDZV",
    "outputId": "04cffe04-f4d3-4c87-b296-fa2967d91e3d"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:36: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: train_loss: 0.5438 train_acc: 0.7599 | val_loss: 0.5334 val_acc: 0.7574\n",
      "Epoch 1: train_loss: 0.4733 train_acc: 0.7829 | val_loss: 0.4746 val_acc: 0.7797\n",
      "Epoch 2: train_loss: 0.4218 train_acc: 0.8015 | val_loss: 0.4234 val_acc: 0.8020\n",
      "Epoch 3: train_loss: 0.3920 train_acc: 0.8139 | val_loss: 0.4133 val_acc: 0.8045\n",
      "Epoch 4: train_loss: 0.3640 train_acc: 0.8319 | val_loss: 0.4217 val_acc: 0.7921\n",
      "Epoch 5: train_loss: 0.3424 train_acc: 0.8406 | val_loss: 0.3862 val_acc: 0.8342\n",
      "Epoch 6: train_loss: 0.3226 train_acc: 0.8474 | val_loss: 0.4097 val_acc: 0.8045\n",
      "Epoch 7: train_loss: 0.3181 train_acc: 0.8524 | val_loss: 0.4272 val_acc: 0.7970\n",
      "Epoch 8: train_loss: 0.2910 train_acc: 0.8716 | val_loss: 0.3975 val_acc: 0.8094\n",
      "Epoch 9: train_loss: 0.2587 train_acc: 0.8877 | val_loss: 0.4128 val_acc: 0.8069\n"
     ]
    }
   ],
   "source": [
    "best_valid_loss = float('inf')\n",
    "\n",
    "epochs=10\n",
    "\n",
    "for epoch in range(epochs):      \n",
    "\n",
    "    start_time = time.time()\n",
    "    \n",
    "    train_loss, train_acc = train(m, iter(train_batch_it), opt, loss_fn, len(train_batch_it))\n",
    "    valid_loss, valid_acc = evaluate(m, iter(val_batch_it), loss_fn, len(val_batch_it))\n",
    "\n",
    "    end_time = time.time()\n",
    "\n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(m.state_dict(), 'tut4-model.pt')\n",
    "          \n",
    "        \n",
    "    print(f'Epoch {epoch}: train_loss: {train_loss:.4f} train_acc: {train_acc:.4f} | val_loss: {valid_loss:.4f} val_acc: {valid_acc:.4f}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "m0FwUXj1KfaN",
    "outputId": "916acc69-3a29-41c7-ec67-a6f894a40b85"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss: 0.409 | Test Acc: 80.59%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:36: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    }
   ],
   "source": [
    "test_loss, test_acc = evaluate(m, iter(test_batch_it), loss_fn, len(test_batch_it))\n",
    "\n",
    "print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qDsz_cIGSuKO"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "depression_detector.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}