Identify key aspects of a Review.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from torch.utils.data import TensorDataset, DataLoader,Dataset\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "import operator\n",
    "import json\n",
    "from sklearn import tree\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.impute import SimpleImputer\n",
    "from collections import Counter\n",
    "from sklearn.preprocessing import LabelEncoder, scale, PolynomialFeatures\n",
    "from sklearn.datasets import load_boston\n",
    "from sklearn.cross_decomposition import PLSRegression, PLSSVD\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_squared_error, roc_auc_score, accuracy_score, r2_score\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from pandas.plotting import scatter_matrix\n",
    "from datetime import datetime\n",
    "from scipy import stats\n",
    "import torch.utils.data\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import IsolationForest\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from torch.autograd import Variable\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "pd.set_option('display.max_columns', 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_csv = pd.read_csv(\"Dataset/train.csv\", keep_default_na=False)\n",
    "test_csv = pd.read_csv(\"Dataset/test.csv\", keep_default_na=False)\n",
    "\n",
    "data = {}\n",
    "labels = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5959, 3)"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_csv.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_data(dataset):\n",
    "    dataset['Review Text'] = dataset[\"Review Title\"].map(str) + \" \" + dataset['Review Text']    \n",
    "    dataset = dataset.drop(['Review Title'],axis=1)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IMDb reviews (combined): train = 5959, test = 2553\n"
     ]
    }
   ],
   "source": [
    "train = preprocess_data(train_csv)\n",
    "test = preprocess_data(test_csv)\n",
    "\n",
    "train_X = train[\"Review Text\"]\n",
    "train_y = train[\"topic\"]\n",
    "test_X = test[\"Review Text\"]\n",
    "\n",
    "#train_X, test_X, train_y, test_y = train_test_split(train[\"Review Text\"], train[\"topic\"], random_state=33)\n",
    "print(\"IMDb reviews (combined): train = {}, test = {}\".format(len(train_X), len(test_X)))\n",
    "#s = \" \".join(train[\"Review Text\"][0:10])\n",
    "#print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import re\n",
    "from collections import Counter\n",
    "\n",
    "def build_dict(data, vocab_size = 6300):\n",
    "    \"\"\"Construct and return a dictionary mapping each of the most frequently appearing words to a unique integer.\"\"\"\n",
    "    \n",
    "    # TODO: Determine how often each word appears in `data`. Note that `data` is a list of sentences and that a\n",
    "    #       sentence is a list of words.\n",
    "    word_counts = Counter(np.concatenate( data, axis=0 ))\n",
    "    # word_count = {} # A dict storing the words that appear in the reviews along with how often they occur\n",
    "    \n",
    "    # TODO: Sort the words found in `data` so that sorted_words[0] is the most frequently appearing word and\n",
    "    #       sorted_words[-1] is the least frequently appearing word.\n",
    "    \n",
    "    sorted_words = sorted(word_counts, key=word_counts.get, reverse=True)\n",
    "    print(len(sorted_words))\n",
    "\n",
    "    word_dict = {} # This is what we are building, a dictionary that translates words into integers\n",
    "    for idx, word in enumerate(sorted_words[:vocab_size - 2]): # The -2 is so that we save room for the 'no word'\n",
    "        word_dict[word] = idx + 2                              # 'infrequent' labels\n",
    "        \n",
    "    return word_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "#word_dict = build_dict(train_X)\n",
    "# print(word_dict)\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import *\n",
    "\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def review_to_words(review):\n",
    "    nltk.download(\"stopwords\", quiet=True)\n",
    "    stemmer = PorterStemmer()\n",
    "    \n",
    "    text = BeautifulSoup(review, \"html.parser\").get_text() # Remove HTML tags\n",
    "    text = re.sub(r\"[^a-zA-Z0-9]\", \" \", text.lower()) # Convert to lower case\n",
    "    words = text.split() # Split string into words\n",
    "    words = [w for w in words if w not in stopwords.words(\"english\")] # Remove stopwords\n",
    "    words = [PorterStemmer().stem(w) for w in words] # stem\n",
    "    \n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Useless Did nothing for me, didn't help lost even with working out and eating healthy. Didn't curb appetite or anything.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['useless',\n",
       " 'nothing',\n",
       " 'help',\n",
       " 'lost',\n",
       " 'even',\n",
       " 'working',\n",
       " 'eating',\n",
       " 'healthy',\n",
       " 'curb',\n",
       " 'appetite',\n",
       " 'anything']"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(train_X[0])\n",
    "review_to_words(train_X[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "cache_dir = os.path.join(\"cache\", \"key_analysis\")  # where to store cache files\n",
    "os.makedirs(cache_dir, exist_ok=True)  # ensure cache directory exists\n",
    "\n",
    "def preprocess_data(data_train, data_test, labels_train, cache_dir=cache_dir, cache_file=\"preprocessed_data.pkl\"):\n",
    "    \"\"\"Convert each review to words; read from cache if available.\"\"\"\n",
    "\n",
    "    # If cache_file is not None, try to read from it first\n",
    "    cache_data = None\n",
    "    print(cache_file)\n",
    "    if cache_file is not None:\n",
    "        try:\n",
    "            with open(os.path.join(cache_dir, cache_file), \"rb\") as f:\n",
    "                cache_data = pickle.load(f)\n",
    "            print(\"Read preprocessed data from cache file:\", cache_file)\n",
    "        except:\n",
    "            print(\"File not found\")\n",
    "            pass  # unable to read from cache, but that's okay\n",
    "    \n",
    "    # If cache is missing, then do the heavy lifting\n",
    "    if cache_data is None:\n",
    "        print('Preprocess training and test data to obtain words for each review')\n",
    "        # Preprocess training and test data to obtain words for each review\n",
    "        words_train = [review_to_words(review) for review in data_train]\n",
    "        words_test = [review_to_words(review) for review in data_test]\n",
    "        \n",
    "        # Write to cache file for future runs\n",
    "        print('Preprocess training and test data to obtain words for each review')\n",
    "        if cache_file is not None:\n",
    "            cache_data = dict(words_train=words_train, words_test=words_test,\n",
    "                              labels_train=labels_train)\n",
    "            with open(os.path.join(cache_dir, cache_file), \"wb\") as f:\n",
    "                pickle.dump(cache_data, f)\n",
    "            print(\"Wrote preprocessed data to cache file:\", cache_file)\n",
    "    else:\n",
    "        print('Unpack data loaded from cache file')\n",
    "        # Unpack data loaded from cache file\n",
    "        words_train, words_test, labels_train = (cache_data['words_train'],\n",
    "                cache_data['words_test'], cache_data['labels_train'])\n",
    "    \n",
    "    return words_train, words_test, labels_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preprocessed_data.pkl\n",
      "File not found\n",
      "Preprocess training and test data to obtain words for each review\n",
      "Preprocess training and test data to obtain words for each review\n",
      "Wrote preprocessed data to cache file: preprocessed_data.pkl\n"
     ]
    }
   ],
   "source": [
    "# Preprocess data\n",
    "train_X, test_X, train_y = preprocess_data(train_X, test_X, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8757\n"
     ]
    }
   ],
   "source": [
    "word_dict = build_dict(train_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['taste', 'product', 'like', 'bad', 'brand']\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "# determine the five most frequently appearing words in the training set.\n",
    "word_counts = Counter(np.concatenate( train_X, axis=0 ))\n",
    "sorted_words = sorted(word_counts, key=word_counts.get, reverse=True)\n",
    "print(sorted_words[0:5])\n",
    "print(word_counts['tast'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "#word_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('word_dict.json', 'w') as f:\n",
    "    json.dump(word_dict, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(cache_dir, 'word_dict.pkl'), \"wb\") as f:\n",
    "    pickle.dump(word_dict, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_and_pad(word_dict, sentence, pad=500):\n",
    "    NOWORD = 0 # We will use 0 to represent the 'no word' category\n",
    "    INFREQ = 1 # and we use 1 to represent the infrequent words, i.e., words not appearing in word_dict\n",
    "    \n",
    "    working_sentence = [NOWORD] * pad\n",
    "    \n",
    "    for word_index, word in enumerate(sentence[:pad]):\n",
    "        if word in word_dict:\n",
    "            working_sentence[word_index] = word_dict[word]\n",
    "        else:\n",
    "            working_sentence[word_index] = INFREQ\n",
    "            \n",
    "    return working_sentence, min(len(sentence), pad)\n",
    "\n",
    "def convert_and_pad_data(word_dict, data, pad=500):\n",
    "    result = []\n",
    "    lengths = []\n",
    "    \n",
    "    for sentence in data:\n",
    "        converted, leng = convert_and_pad(word_dict, sentence, pad)\n",
    "        result.append(converted)\n",
    "        lengths.append(leng)\n",
    "        \n",
    "    return np.array(result), np.array(lengths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, train_X_len = convert_and_pad_data(word_dict, train_X)\n",
    "test_X, test_X_len = convert_and_pad_data(word_dict, test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500\n",
      "[11]\n"
     ]
    }
   ],
   "source": [
    "print(len(train_X[0]))\n",
    "print(train_X_len[0:1]) # 500 - 322 (non zeros) = 178"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(train_y), pd.DataFrame(train_X_len), pd.DataFrame(train_X)], axis=1).to_csv('sample_train.csv', header=False, index=False)\n",
    "pd.concat([pd.DataFrame(test_X_len), pd.DataFrame(test_X)], axis=1).to_csv('sample_test.csv', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTMClassifier(nn.Module):\n",
    "    \"\"\"\n",
    "    This is the simple RNN model we will be using to perform Sentiment Analysis.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, embedding_dim, hidden_dim, vocab_size):\n",
    "        \"\"\"\n",
    "        Initialize the model by settingg up the various layers.\n",
    "        \"\"\"\n",
    "        super(LSTMClassifier, self).__init__()\n",
    "        \n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.output_size = 1\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx=0)\n",
    "        self.lstm = nn.LSTM(embedding_dim, hidden_dim)\n",
    "        self.dense = nn.Linear(in_features=hidden_dim, out_features=21)\n",
    "        self.sig = nn.Sigmoid()\n",
    "        \n",
    "        self.word_dict = None\n",
    "        self.init_weights()\n",
    "    \n",
    "    def init_weights(self):\n",
    "        initrange = 0.08\n",
    "        self.embedding.weight.data.uniform_(-initrange, initrange)\n",
    "        self.lstm.weight_ih_l0.data.uniform_(-initrange, initrange)\n",
    "        self.lstm.weight_hh_l0.data.uniform_(-initrange, initrange)\n",
    "        \n",
    "        self.lstm.bias_ih_l0.data.zero_()\n",
    "        self.lstm.bias_hh_l0.data.zero_()\n",
    "        \n",
    "        # self.fc.bias.data.zero_()\n",
    "        self.dense.bias.data.fill_(0)\n",
    "        # self.fc.weight.data.uniform_(-initrange, initrange)\n",
    "        self.dense.weight.data.normal_(0.0, (1.0 / np.sqrt(self.dense.in_features)))\n",
    "        \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        Perform a forward pass of our model on some input.\n",
    "        \"\"\"\n",
    "        \n",
    "        batch_size = x.size(0)\n",
    "        \n",
    "        x = x.t()\n",
    "        lengths = x[0,:]\n",
    "        reviews = x[1:,:]\n",
    "        \n",
    "        #print(reviews)\n",
    "        #print(batch_size)\n",
    "        \n",
    "        embeds = self.embedding(reviews)\n",
    "        lstm_out, _ = self.lstm(embeds)\n",
    "        \n",
    "        #lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim)\n",
    "        \n",
    "        out = self.dense(lstm_out)\n",
    "        #print(out)\n",
    "        #sig_out = out.view(batch_size, -1, self.output_size)\n",
    "        #sig_out = sig_out[:, -1]\n",
    "        #print(sig_out)\n",
    "        \n",
    "        out = out[lengths - 1, range(len(lengths))]\n",
    "        return self.sig(out.squeeze())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "class DNNClassifier(nn.Module):\n",
    "    \"\"\"\n",
    "    This is the simple DNN model we will be using to perform Sentiment Analysis.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, hidden_dim, input_size, output_size, dropout=0.5):\n",
    "        \"\"\"\n",
    "        Initialize the model by settingg up the various layers.\n",
    "        \"\"\"\n",
    "        super(DNNClassifier, self).__init__()\n",
    "\n",
    "        self.sig = nn.Sigmoid()        \n",
    "        # self.word_dict = None\n",
    "        \n",
    "        self.fc1 = nn.Linear(input_size, hidden_dim * 4)\n",
    "        self.fc2 = nn.Linear(hidden_dim * 4, hidden_dim * 2)\n",
    "        self.fc3 = nn.Linear(hidden_dim * 2, hidden_dim)\n",
    "        self.out = nn.Linear(hidden_dim, output_size)\n",
    "        self.dropout = nn.Dropout(p=0.5)\n",
    "        self.init_weights()\n",
    "        self.soft = nn.LogSoftmax(dim=0)\n",
    "        \n",
    "    def init_weights(m):\n",
    "        initrange = 0.08\n",
    "        classname = m.__class__.__name__\n",
    "        if classname.find('Linear') != -1:\n",
    "            # get the number of the inputs\n",
    "            n = m.in_features\n",
    "            y = 1.0/np.sqrt(n)\n",
    "            m.weight.data.normal_(0.0, y)\n",
    "            m.bias.data.fill_(0)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        Perform a forward pass of our model on some input.\n",
    "        \"\"\"\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.dropout(x)\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.dropout(x)\n",
    "        x = F.relu(self.fc3(x))\n",
    "        x = self.dropout(x)\n",
    "        out = self.out(x)\n",
    "        return self.soft(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[15 11  4 ...  1  1  1]\n",
      "[[0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " ...\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "# Read in only the first 250 rows\n",
    "train_sample = pd.read_csv('sample_train.csv', header=None, names=None)\n",
    "\n",
    "binary_label = pd.get_dummies(train_sample[[0]].astype('category'), prefix=\"\", prefix_sep=\"\")\n",
    "\n",
    "#binary_label_columns = binary_label.dot(binary_label.columns)\n",
    "#binary_label_columns = binary_label.idxmax(axis=1)\n",
    "\n",
    "#binary_label_stack = binary_label.stack()\n",
    "#print(binary_label.columns[0:20])\n",
    "#print(binary_label_columns)\n",
    "#print(binary_label_columns[5].replace(\"0_\", \"\"))\n",
    "\n",
    "labels = np.asarray(binary_label)\n",
    "\n",
    "lbl = LabelEncoder() \n",
    "lbl.fit(list(train_sample[[0]].values)) \n",
    "print(lbl.transform(train_sample[[0]].values))\n",
    "# Turn the input pandas dataframe into tensors\n",
    "\n",
    "#train_sample_y = torch.from_numpy(lbl.transform(list(train_sample[[0]].values))).float().squeeze()\n",
    "train_sample_y = labels\n",
    "train_sample_X = train_sample.drop([0], axis=1).values\n",
    "\n",
    "#train_sample_y = torch.from_numpy(labels).float().squeeze()\n",
    "#train_sample_X = torch.from_numpy(train_sample.drop([0], axis=1).values).long()\n",
    "\n",
    "print(train_sample_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameters\n",
    "batch_size = 50\n",
    "num_epochs = 10\n",
    "# Learning Rate\n",
    "learning_rate = 0.0001\n",
    "# Model parameters\n",
    "# Input size\n",
    "input_size = train_sample_X.shape[1]\n",
    "# Output size\n",
    "output_size = 1\n",
    "# Embedding Dimension\n",
    "embedding_dim = 32\n",
    "# Hidden Dimension\n",
    "hidden_dim = 200\n",
    "# Number of RNN Layers\n",
    "n_layers = 2\n",
    "#vocabulary size\n",
    "vocab_size = 6300\n",
    "\n",
    "# Show stats for every n number of batches\n",
    "show_every_n_batches = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(train_sample_X, train_sample_y, random_state=33)\n",
    "\n",
    "# Build the dataset\n",
    "train_sample_ds = torch.utils.data.TensorDataset(torch.from_numpy(X_train).long(), torch.from_numpy(y_train).float().squeeze())\n",
    "val_sample_ds = torch.utils.data.TensorDataset(torch.from_numpy(X_val).long(), torch.from_numpy(y_val).float().squeeze())\n",
    "# Build the dataloader\n",
    "train_sample_dl = torch.utils.data.DataLoader(train_sample_ds, batch_size=batch_size)\n",
    "var_sample_dl = torch.utils.data.DataLoader(val_sample_ds, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, train_loader, val_loader, epochs, optimizer, loss_fn, device):\n",
    "    train_loss = 0\n",
    "    val_loss = 0\n",
    "    for epoch in range(1, epochs + 1):\n",
    "        model.train()\n",
    "        for batch in train_loader:\n",
    "            batch_X, batch_y = batch\n",
    "            \n",
    "            batch_X = batch_X.to(device)\n",
    "            batch_y = batch_y.to(device)\n",
    "            \n",
    "            # model.zero_grad()\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            #batch_len = batch_X.t()[0,:]\n",
    "            #print(batch_len)\n",
    "            \n",
    "            # TODO: Complete this train method to train the model provided.\n",
    "            output = model.forward(batch_X)\n",
    "            #print(output.reshape(-1,1).shape)\n",
    "            #print(batch_y.shape)\n",
    "            \n",
    "            loss = loss_fn(output, batch_y)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            train_loss += loss.item()\n",
    "            #print(loss.item())\n",
    "            #break\n",
    "        model.eval()\n",
    "        for batch in val_loader:\n",
    "            batch_X, batch_y = batch\n",
    "            \n",
    "            batch_X = batch_X.to(device)\n",
    "            batch_y = batch_y.to(device)\n",
    "            \n",
    "            # model.zero_grad()\n",
    "            optimizer.zero_grad()\n",
    "            # TODO: Complete this train method to train the model provided.\n",
    "            output = model.forward(batch_X)\n",
    "            loss = loss_fn(output, batch_y)            \n",
    "            val_loss += loss.item()\n",
    "        \n",
    "        total_train_loss = train_loss / len(train_loader)\n",
    "        total_val_loss = val_loss / len(val_loader)\n",
    "        \n",
    "        print(\"Epoch: {}, BCE Train Loss: {} Valid Loss {}\".format(epoch, total_train_loss, total_val_loss))\n",
    "        val_loss = 0\n",
    "        train_loss = 0\n",
    "        #break\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "#device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "device = \"cpu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, BCE Train Loss: 0.5880884326166577 Valid Loss 0.27141678929328916\n",
      "Epoch: 2, BCE Train Loss: 0.2280863054924541 Valid Loss 0.21165577272574107\n",
      "Epoch: 3, BCE Train Loss: 0.201782432032956 Valid Loss 0.19778250753879548\n",
      "Epoch: 4, BCE Train Loss: 0.1918556244836913 Valid Loss 0.19006353120009103\n",
      "Epoch: 5, BCE Train Loss: 0.18593414028485616 Valid Loss 0.18517350852489473\n",
      "Epoch: 6, BCE Train Loss: 0.18191168473826513 Valid Loss 0.18183306405941646\n",
      "Epoch: 7, BCE Train Loss: 0.17903217126925786 Valid Loss 0.17943763335545856\n",
      "Epoch: 8, BCE Train Loss: 0.17691407716936536 Valid Loss 0.17765479336182277\n",
      "Epoch: 9, BCE Train Loss: 0.17531957113080554 Valid Loss 0.17628580232461294\n",
      "Epoch: 10, BCE Train Loss: 0.1740908076365789 Valid Loss 0.175205693145593\n"
     ]
    }
   ],
   "source": [
    "#model = DNNClassifier(hidden_dim, input_size, output_size).to(device)\n",
    "model = LSTMClassifier(embedding_dim, hidden_dim, vocab_size).to(device)\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "#loss_fn = nn.CrossEntropyLoss()\n",
    "#loss_fn = nn.NLLLoss()\n",
    "loss_fn = torch.nn.BCELoss()\n",
    "\n",
    "model_tn = train(model, train_sample_dl, var_sample_dl, num_epochs, optimizer, loss_fn, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('save/trained_rnn_new', 'wb') as pickle_file:\n",
    "    torch.save(model_tn, pickle_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in only the first 250 rows\n",
    "test_sample_X = pd.read_csv('sample_test.csv', header=None, names=None)\n",
    "\n",
    "#test_sample_ds = torch.utils.data.TensorDataset(torch.from_numpy(X_train).long(), torch.from_numpy(y_train).float().squeeze())\n",
    "#test_sample_dl = torch.utils.data.DataLoader(test_sample_ds, batch_size=batch_size)\n",
    "#print(train_sample_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LSTMClassifier(\n",
       "  (embedding): Embedding(6300, 32, padding_idx=0)\n",
       "  (lstm): LSTM(32, 200)\n",
       "  (dense): Linear(in_features=200, out_features=21, bias=True)\n",
       "  (sig): Sigmoid()\n",
       ")"
      ]
     },
     "execution_count": 265,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_tn = torch.load(\"save/trained_rnn_new\")\n",
    "model_tn.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[  11,  505,   92,   88,  387,   23,  526,  450,  625, 4009,  859,  154,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0]])\n",
      "tensor([0.0903, 0.2131, 0.0405, 0.0011, 0.0390, 0.0039, 0.0214, 0.0066, 0.0009,\n",
      "        0.0060, 0.0332, 0.0880, 0.0811, 0.0155, 0.1143, 0.0644, 0.0188, 0.0704,\n",
      "        0.0133, 0.0390, 0.0135], grad_fn=<ClampBackward>)\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "Can't call numpy() on Variable that requires grad. Use var.detach().numpy() instead.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m~/.conda/envs/deep-learning/lib/python3.7/site-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36m_wrapfunc\u001b[0;34m(obj, method, *args, **kwds)\u001b[0m\n\u001b[1;32m     55\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: argmax() got an unexpected keyword argument 'axis'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-295-912f12680b22>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0;31m#output = output.squeeze().cpu().detach().numpy().flatten()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclamp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m     \u001b[0mprecdictions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprecdictions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/deep-learning/lib/python3.7/site-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36margmax\u001b[0;34m(a, axis, out)\u001b[0m\n\u001b[1;32m   1101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1102\u001b[0m     \"\"\"\n\u001b[0;32m-> 1103\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_wrapfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'argmax'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1104\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/deep-learning/lib/python3.7/site-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36m_wrapfunc\u001b[0;34m(obj, method, *args, **kwds)\u001b[0m\n\u001b[1;32m     64\u001b[0m     \u001b[0;31m# a downstream library like 'pandas'.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mAttributeError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_wrapit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/deep-learning/lib/python3.7/site-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36m_wrapit\u001b[0;34m(obj, method, *args, **kwds)\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0mwrap\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     47\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mwrap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmu\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/deep-learning/lib/python3.7/site-packages/numpy/core/numeric.py\u001b[0m in \u001b[0;36masarray\u001b[0;34m(a, dtype, order)\u001b[0m\n\u001b[1;32m    536\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    537\u001b[0m     \"\"\"\n\u001b[0;32m--> 538\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    539\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/deep-learning/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36m__array__\u001b[0;34m(self, dtype)\u001b[0m\n\u001b[1;32m    448\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__array__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    449\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 450\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    451\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    452\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Can't call numpy() on Variable that requires grad. Use var.detach().numpy() instead."
     ]
    }
   ],
   "source": [
    "precdictions = []\n",
    "for test_x in train_sample_X[0:3]:\n",
    "    test_x = test_x.reshape(1, -1)\n",
    "    test_x = torch.from_numpy(test_x).long()\n",
    "    test_x = test_x.to(device)\n",
    "    #test_x = torch.from_numpy(test_x).long().unsqueeze(0)\n",
    "    #test_x = test_x.to(device)\n",
    "    print(test_x)\n",
    "    output = model_tn.forward(test_x)\n",
    "    #print(output.mean())\n",
    "    output = output.squeeze().cpu().detach().numpy().flatten()\n",
    "    print(output.mean())\n",
    "    precdictions.append(np.argmax(output))\n",
    "\n",
    "print(precdictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[  12,    1,   68,  233, 1121,  233,    1,  288,  479, 1121,  389, 1052,\n",
      "           48,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0]])\n",
      "tensor([0.0894, 0.2148, 0.0406, 0.0011, 0.0389, 0.0039, 0.0213, 0.0066, 0.0009,\n",
      "        0.0060, 0.0332, 0.0874, 0.0807, 0.0155, 0.1142, 0.0640, 0.0187, 0.0703,\n",
      "        0.0133, 0.0389, 0.0135])\n",
      "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "input_data = \"These are GAWD awful, they do not stick, they are uncomfortable because it doesn't stick so it moves around making it very uncomfortable and difficult to absorb. These are HORRIBLE\"\n",
    "data_X = None\n",
    "data_len = None\n",
    "input_data_words = review_to_words(input_data)\n",
    "data_X, data_len = convert_and_pad(word_dict, input_data_words)\n",
    "# data_X = pd.concat([pd.DataFrame(test_data_len), pd.DataFrame(test_data)], axis=1)\n",
    "\n",
    "# Using data_X and data_len we construct an appropriate input tensor. Remember\n",
    "# that our model expects input data of the form 'len, review[500]'.\n",
    "# data_pack = np.hstack((data_len, data_X))\n",
    "data_pack = np.hstack((data_len, data_X))\n",
    "\n",
    "data_pack = data_pack.reshape(1, -1)\n",
    "\n",
    "data = torch.from_numpy(data_pack)\n",
    "data = data.to(device)\n",
    "\n",
    "# Make sure to put the model into evaluation mode\n",
    "model_tn.eval()\n",
    "\n",
    "# TODO: Compute the result of applying the model to the input data. The variable `result` should\n",
    "#       be a numpy array which contains a single integer which is either 1 or 0\n",
    "print(data)\n",
    "with torch.no_grad():\n",
    "    output = model_tn.forward(data)\n",
    "\n",
    "print(output)\n",
    "result = np.round(output.numpy())\n",
    "# result = predictor.predict(data.values)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_csv[\"topic_test\"] = pd.Series(lbl.inverse_transform(np.array(precdictions)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Review Text</th>\n",
       "      <th>Review Title</th>\n",
       "      <th>topic</th>\n",
       "      <th>topic_test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Useless Did nothing for me, didn't help lost e...</td>\n",
       "      <td>Useless</td>\n",
       "      <td>Shipment and delivery</td>\n",
       "      <td>Bad Taste/Flavor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Useless Did nothing for me, didn't help lost e...</td>\n",
       "      <td>Useless</td>\n",
       "      <td>Not Effective</td>\n",
       "      <td>Bad Taste/Flavor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TRASH!!! Do not buy these bags it’s a waist of...</td>\n",
       "      <td>TRASH!!! Do not buy these bags it’s a waist of...</td>\n",
       "      <td>Customer Service</td>\n",
       "      <td>Bad Taste/Flavor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Do not recommend Gave me an allergic reaction ...</td>\n",
       "      <td>Do not recommend</td>\n",
       "      <td>Allergic</td>\n",
       "      <td>Bad Taste/Flavor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Can't tackle big messes These don't compare to...</td>\n",
       "      <td>Can't tackle big messes</td>\n",
       "      <td>Texture</td>\n",
       "      <td>Bad Taste/Flavor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         Review Text  \\\n",
       "0  Useless Did nothing for me, didn't help lost e...   \n",
       "1  Useless Did nothing for me, didn't help lost e...   \n",
       "2  TRASH!!! Do not buy these bags it’s a waist of...   \n",
       "3  Do not recommend Gave me an allergic reaction ...   \n",
       "4  Can't tackle big messes These don't compare to...   \n",
       "\n",
       "                                        Review Title                  topic  \\\n",
       "0                                            Useless  Shipment and delivery   \n",
       "1                                            Useless          Not Effective   \n",
       "2  TRASH!!! Do not buy these bags it’s a waist of...       Customer Service   \n",
       "3                                   Do not recommend               Allergic   \n",
       "4                            Can't tackle big messes                Texture   \n",
       "\n",
       "         topic_test  \n",
       "0  Bad Taste/Flavor  \n",
       "1  Bad Taste/Flavor  \n",
       "2  Bad Taste/Flavor  \n",
       "3  Bad Taste/Flavor  \n",
       "4  Bad Taste/Flavor  "
      ]
     },
     "execution_count": 289,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_csv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_csv.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}