run_models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c327c7d3-72a7-484b-b1be-2213a615e33a",
   "metadata": {},
   "source": [
    "# Final Output Notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98631696-f601-4776-8830-804d86a3aace",
   "metadata": {},
   "source": [
    "## Imports and Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "794e10a4-aa06-44d5-abef-3fad43981b7e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:21:01.249239Z",
     "iopub.status.busy": "2022-04-27T11:21:01.249064Z",
     "iopub.status.idle": "2022-04-27T11:21:04.025832Z",
     "shell.execute_reply": "2022-04-27T11:21:04.024679Z",
     "shell.execute_reply.started": "2022-04-27T11:21:01.249188Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-27 11:21:02.277569: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n"
     ]
    }
   ],
   "source": [
    "# Imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import project_functions as pf\n",
    "import project_models as pm\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras import optimizers\n",
    "from tensorflow.keras.callbacks import EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d921deda-ee44-41e3-a570-34831d9e47bd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:21:04.027294Z",
     "iopub.status.busy": "2022-04-27T11:21:04.027069Z",
     "iopub.status.idle": "2022-04-27T11:21:04.030785Z",
     "shell.execute_reply": "2022-04-27T11:21:04.030185Z",
     "shell.execute_reply.started": "2022-04-27T11:21:04.027256Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Plotting config\n",
    "pf.set_plot_config()\n",
    "# Hide TF messages for neater NB\n",
    "pf.tf_message_toggle(\"3\")\n",
    "# Set global seed for reproducible notebook\n",
    "pf.set_seed()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d3e32f6b-d49d-409a-a453-e8ef5fd52df1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:21:04.031659Z",
     "iopub.status.busy": "2022-04-27T11:21:04.031516Z",
     "iopub.status.idle": "2022-04-27T11:21:04.036723Z",
     "shell.execute_reply": "2022-04-27T11:21:04.036240Z",
     "shell.execute_reply.started": "2022-04-27T11:21:04.031642Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Hyperparams used during modelling\n",
    "# Compilation hyperparams\n",
    "compile_hp = dict()\n",
    "compile_hp[\"loss\"] = \"binary_crossentropy\"\n",
    "compile_hp[\"optimizer\"] = optimizers.Adam(learning_rate = 0.001)\n",
    "compile_hp[\"metrics\"] = [\"accuracy\"]\n",
    "\n",
    "# Fitting hyperparams\n",
    "fit_hp = dict()\n",
    "fit_hp[\"batch_size\"] = 64\n",
    "fit_hp[\"epochs\"] = 100\n",
    "fit_hp[\"validation_split\"] = 0.3\n",
    "# Create callback to select the best model\n",
    "fit_hp[\"callbacks\"] = EarlyStopping(monitor = \"val_loss\",\n",
    "                                         mode = \"min\",\n",
    "                                         restore_best_weights = True,\n",
    "                                         patience = 25)\n",
    "\n",
    "# Eliminate verbose to have a neater notebook \n",
    "fit_hp[\"verbose\"] = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e45fca1-4a64-46e0-96bf-5e92c22e96d5",
   "metadata": {},
   "source": [
    "## Pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b6a64fde-84dc-4d17-9020-9ed87c9729d6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:21:04.037592Z",
     "iopub.status.busy": "2022-04-27T11:21:04.037425Z",
     "iopub.status.idle": "2022-04-27T11:21:05.307654Z",
     "shell.execute_reply": "2022-04-27T11:21:05.307034Z",
     "shell.execute_reply.started": "2022-04-27T11:21:04.037573Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load in data\n",
    "text = pf.imdb_train_test_dfs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "33ced45e-4055-4558-9bac-266b4d0787d8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:21:05.309313Z",
     "iopub.status.busy": "2022-04-27T11:21:05.309054Z",
     "iopub.status.idle": "2022-04-27T11:21:05.316403Z",
     "shell.execute_reply": "2022-04-27T11:21:05.315886Z",
     "shell.execute_reply.started": "2022-04-27T11:21:05.309292Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Split into test and train data\n",
    "X = text.review.values\n",
    "y = np.array(text[\"sentiment\"], dtype = \"float32\")\n",
    "text_train, text_test, y_train, y_test = train_test_split(X, y,\n",
    "                                                    test_size = 0.2, shuffle = True,\n",
    "                                                          # reproducible split\n",
    "                                                         random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0a7a87be-2209-4c81-8d0f-f4134f2baab9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:21:05.317586Z",
     "iopub.status.busy": "2022-04-27T11:21:05.317309Z",
     "iopub.status.idle": "2022-04-27T11:21:17.350884Z",
     "shell.execute_reply": "2022-04-27T11:21:17.350212Z",
     "shell.execute_reply.started": "2022-04-27T11:21:05.317552Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Process (lower-case, no special chars etc), tokenize, pad and convert to sequences\n",
    "# Uses pre-padding and post-truncating with a max sequence length of 500\n",
    "tokenizer, X_train, X_test = pf.tokenize_padder(text_train, text_test)\n",
    "\n",
    "# Get vocab size\n",
    "vocab_size = len(tokenizer.word_index) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "87af46f1-326d-45f3-85c6-bc0d70c7362d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:21:17.351915Z",
     "iopub.status.busy": "2022-04-27T11:21:17.351751Z",
     "iopub.status.idle": "2022-04-27T11:21:17.355685Z",
     "shell.execute_reply": "2022-04-27T11:21:17.355145Z",
     "shell.execute_reply.started": "2022-04-27T11:21:17.351895Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Toggle notebook options\n",
    "display = False\n",
    "save_plots = False\n",
    "save_model = False\n",
    "save_history = False\n",
    "\n",
    "# Create dataframe to store each model's results on test set\n",
    "all_results = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28b8d5d5-5b56-439e-84d9-50af0427b7de",
   "metadata": {},
   "source": [
    "## Create Embedding Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "19b9810c-4c2b-408c-852c-ab1f89185ab6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:21:17.356586Z",
     "iopub.status.busy": "2022-04-27T11:21:17.356381Z",
     "iopub.status.idle": "2022-04-27T11:31:39.120101Z",
     "shell.execute_reply": "2022-04-27T11:31:39.119590Z",
     "shell.execute_reply.started": "2022-04-27T11:21:17.356566Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8.875 % complete\n",
      "17.749 % complete\n",
      "26.624 % complete\n",
      "35.499 % complete\n",
      "44.373 % complete\n",
      "53.248 % complete\n",
      "62.123 % complete\n",
      "70.998 % complete\n",
      "79.872 % complete\n",
      "88.747 % complete\n",
      "97.622 % complete\n"
     ]
    }
   ],
   "source": [
    "# Store in dictionary to make selection more programmatic throughout NB\n",
    "# This will take a  while\n",
    "embed_dict = dict()\n",
    "embed_dict[None] = None\n",
    "embed_dict[\"glove\"] = pm.glove_embedding(tokenizer, show_progress = False)\n",
    "# Takes the longest\n",
    "embed_dict[\"spacy\"] = pm.spacy_embedding(tokenizer, show_progress = True)\n",
    "embed_dict[\"keras\"] = pm.keras_embedding(tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e944135d-eaf8-41a6-8d55-2a93f357e77a",
   "metadata": {
    "tags": []
   },
   "source": [
    "# MLP"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "365474fa-bc4d-48a9-b148-4dee1917a607",
   "metadata": {
    "tags": []
   },
   "source": [
    "## No Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "39d2dd09-a0c9-4f18-b34e-6de2aa02047b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:31:39.121023Z",
     "iopub.status.busy": "2022-04-27T11:31:39.120874Z",
     "iopub.status.idle": "2022-04-27T11:31:39.124242Z",
     "shell.execute_reply": "2022-04-27T11:31:39.123745Z",
     "shell.execute_reply.started": "2022-04-27T11:31:39.121004Z"
    }
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict = dict()\n",
    "model_dict[\"name\"] = \"mlp\"\n",
    "model_dict[\"embedding\"] = None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da75c154-fa91-4558-8a3b-dd2b1c3bbeed",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2133891-8196-445d-9364-16e326eb520f",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-04-27T12:00:36.554686Z",
     "iopub.status.idle": "2022-04-27T12:00:36.555001Z",
     "shell.execute_reply": "2022-04-27T12:00:36.554859Z",
     "shell.execute_reply.started": "2022-04-27T12:00:36.554840Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "pf.set_seed()\n",
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bf4595e-aa50-4e77-9577-5112e613e73c",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "234b9d99-c909-4dd3-847a-63a0c3e9a61b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:31:42.294228Z",
     "iopub.status.busy": "2022-04-27T11:31:42.294067Z",
     "iopub.status.idle": "2022-04-27T11:31:44.173913Z",
     "shell.execute_reply": "2022-04-27T11:31:44.173387Z",
     "shell.execute_reply.started": "2022-04-27T11:31:42.294209Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               128256    \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 128,513\n",
      "Trainable params: 128,513\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2fddd349-bc68-40de-bdd9-0119f5ca143a",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "eb11dd5c-8879-455f-9b5e-16a4f2c6300f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:31:44.174986Z",
     "iopub.status.busy": "2022-04-27T11:31:44.174765Z",
     "iopub.status.idle": "2022-04-27T11:31:45.971163Z",
     "shell.execute_reply": "2022-04-27T11:31:45.970651Z",
     "shell.execute_reply.started": "2022-04-27T11:31:44.174964Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Linear_Dense (Dense)         (None, 256)               128256    \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 129,537\n",
      "Trainable params: 129,025\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "927b51ee-4fae-4332-855a-c17cca08bf46",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8c85447f-7c34-447d-84cf-c00328d90a92",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:31:45.972230Z",
     "iopub.status.busy": "2022-04-27T11:31:45.972006Z",
     "iopub.status.idle": "2022-04-27T11:31:47.831631Z",
     "shell.execute_reply": "2022-04-27T11:31:47.831092Z",
     "shell.execute_reply.started": "2022-04-27T11:31:45.972208Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               128256    \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 129,537\n",
      "Trainable params: 129,025\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28befc70-a636-41c5-b71a-94e6d152d346",
   "metadata": {
    "tags": []
   },
   "source": [
    "## GloVe Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d3242a41-f4dd-411d-a65b-f5c10ad2ddd9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:31:47.833933Z",
     "iopub.status.busy": "2022-04-27T11:31:47.833712Z",
     "iopub.status.idle": "2022-04-27T11:31:47.836912Z",
     "shell.execute_reply": "2022-04-27T11:31:47.836386Z",
     "shell.execute_reply.started": "2022-04-27T11:31:47.833911Z"
    }
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"glove\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d142871e-cb02-447a-a2e6-ac48a5e22aca",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9360d443-251d-475d-aa97-366bb90485e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:31:47.837917Z",
     "iopub.status.busy": "2022-04-27T11:31:47.837691Z",
     "iopub.status.idle": "2022-04-27T11:31:58.373607Z",
     "shell.execute_reply": "2022-04-27T11:31:58.372954Z",
     "shell.execute_reply.started": "2022-04-27T11:31:47.837897Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 150000)            0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               38400256  \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 72,204,513\n",
      "Trainable params: 38,400,513\n",
      "Non-trainable params: 33,804,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6a9b3d3-b3c4-422c-a72d-e33a56a94b35",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7a20f3d0-d976-45d8-baf5-009114a01f4e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:31:58.374913Z",
     "iopub.status.busy": "2022-04-27T11:31:58.374574Z",
     "iopub.status.idle": "2022-04-27T11:32:12.733961Z",
     "shell.execute_reply": "2022-04-27T11:32:12.733443Z",
     "shell.execute_reply.started": "2022-04-27T11:31:58.374883Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 150000)            0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               38400256  \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 72,204,513\n",
      "Trainable params: 38,400,513\n",
      "Non-trainable params: 33,804,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ca86dc2-649e-4419-92de-520c643e1332",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5e39e767-a533-4fb0-9f00-dc42b3599541",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:32:12.735064Z",
     "iopub.status.busy": "2022-04-27T11:32:12.734823Z",
     "iopub.status.idle": "2022-04-27T11:32:23.805067Z",
     "shell.execute_reply": "2022-04-27T11:32:23.804551Z",
     "shell.execute_reply.started": "2022-04-27T11:32:12.735043Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 150000)            0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               38400256  \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 72,205,537\n",
      "Trainable params: 38,401,025\n",
      "Non-trainable params: 33,804,512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96a583b4-9592-496d-b09b-d491bc39f1d9",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "52c08496-e4e3-4b10-b47b-0bce032594cc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:32:23.806140Z",
     "iopub.status.busy": "2022-04-27T11:32:23.805909Z",
     "iopub.status.idle": "2022-04-27T11:32:38.256352Z",
     "shell.execute_reply": "2022-04-27T11:32:38.255829Z",
     "shell.execute_reply.started": "2022-04-27T11:32:23.806119Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "flatten_3 (Flatten)          (None, 150000)            0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               38400256  \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 72,205,537\n",
      "Trainable params: 38,401,025\n",
      "Non-trainable params: 33,804,512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5c99d61-64f4-4e8c-92f1-15965e452f13",
   "metadata": {
    "tags": []
   },
   "source": [
    "## SpaCy Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ff5f520a-2a86-4ea6-b936-0d81a9db2090",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:32:38.257384Z",
     "iopub.status.busy": "2022-04-27T11:32:38.257158Z",
     "iopub.status.idle": "2022-04-27T11:32:38.260682Z",
     "shell.execute_reply": "2022-04-27T11:32:38.260072Z",
     "shell.execute_reply.started": "2022-04-27T11:32:38.257364Z"
    }
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"spacy\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7121fa25-2a44-45b8-99e6-52bce161849c",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c801bed2-bf3d-4fce-946d-e94445eec4e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:32:38.262106Z",
     "iopub.status.busy": "2022-04-27T11:32:38.261823Z",
     "iopub.status.idle": "2022-04-27T11:32:42.879911Z",
     "shell.execute_reply": "2022-04-27T11:32:42.879385Z",
     "shell.execute_reply.started": "2022-04-27T11:32:38.262061Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "flatten_4 (Flatten)          (None, 48000)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               12288256  \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 23,105,793\n",
      "Trainable params: 12,288,513\n",
      "Non-trainable params: 10,817,280\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2d58b85-d812-463b-b0b9-e5ec5de5bf54",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c5f1060a-b61b-4d1f-ba97-10d5f6423cd8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:32:42.881028Z",
     "iopub.status.busy": "2022-04-27T11:32:42.880800Z",
     "iopub.status.idle": "2022-04-27T11:32:48.515261Z",
     "shell.execute_reply": "2022-04-27T11:32:48.514676Z",
     "shell.execute_reply.started": "2022-04-27T11:32:42.881008Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "flatten_5 (Flatten)          (None, 48000)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               12288256  \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 23,105,793\n",
      "Trainable params: 12,288,513\n",
      "Non-trainable params: 10,817,280\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c27fa7a6-7ef5-4f40-8bb2-8648376fb6bb",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "733c84f0-94fa-49ba-a0cd-ebc5c12740e6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:32:48.516438Z",
     "iopub.status.busy": "2022-04-27T11:32:48.516199Z",
     "iopub.status.idle": "2022-04-27T11:32:53.464295Z",
     "shell.execute_reply": "2022-04-27T11:32:53.463757Z",
     "shell.execute_reply.started": "2022-04-27T11:32:48.516415Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "flatten_6 (Flatten)          (None, 48000)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               12288256  \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 23,106,817\n",
      "Trainable params: 12,289,025\n",
      "Non-trainable params: 10,817,792\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90e8f52e-1eef-4895-b08c-75f7434ea30d",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d4ab7fb6-e765-4fcd-9b9a-a5ecbc0597cb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:32:53.465430Z",
     "iopub.status.busy": "2022-04-27T11:32:53.465188Z",
     "iopub.status.idle": "2022-04-27T11:32:59.296782Z",
     "shell.execute_reply": "2022-04-27T11:32:59.296271Z",
     "shell.execute_reply.started": "2022-04-27T11:32:53.465408Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "flatten_7 (Flatten)          (None, 48000)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               12288256  \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 23,106,817\n",
      "Trainable params: 12,289,025\n",
      "Non-trainable params: 10,817,792\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01b966b2-4b2c-4e69-9108-4fa336909ec4",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Keras Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0094be15-3743-45ff-8f1f-0a5d860c766e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:32:59.299582Z",
     "iopub.status.busy": "2022-04-27T11:32:59.299251Z",
     "iopub.status.idle": "2022-04-27T11:32:59.302887Z",
     "shell.execute_reply": "2022-04-27T11:32:59.302366Z",
     "shell.execute_reply.started": "2022-04-27T11:32:59.299547Z"
    }
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"keras\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7b0980a-54c2-4ae4-a868-57f3264992dc",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "87ccdcfb-99aa-4423-89af-597a85c7b5d4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:32:59.303889Z",
     "iopub.status.busy": "2022-04-27T11:32:59.303675Z",
     "iopub.status.idle": "2022-04-27T11:33:15.728777Z",
     "shell.execute_reply": "2022-04-27T11:33:15.728177Z",
     "shell.execute_reply.started": "2022-04-27T11:32:59.303870Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "flatten_8 (Flatten)          (None, 128000)            0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               32768256  \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 61,614,593\n",
      "Trainable params: 61,614,593\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15360685-c7cc-48f7-8b6a-888126d71cdf",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "0d8f543a-ce2f-4af6-8ff8-ddeb2d82e065",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:33:15.730269Z",
     "iopub.status.busy": "2022-04-27T11:33:15.729792Z",
     "iopub.status.idle": "2022-04-27T11:33:34.296925Z",
     "shell.execute_reply": "2022-04-27T11:33:34.296062Z",
     "shell.execute_reply.started": "2022-04-27T11:33:15.730234Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "flatten_9 (Flatten)          (None, 128000)            0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               32768256  \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 61,614,593\n",
      "Trainable params: 61,614,593\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7321d7a-f4d2-4c55-ae87-bc1b4a62c922",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "92e5e44c-bce7-451a-885f-f7896a5aa71c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:33:34.298383Z",
     "iopub.status.busy": "2022-04-27T11:33:34.297871Z",
     "iopub.status.idle": "2022-04-27T11:33:50.167782Z",
     "shell.execute_reply": "2022-04-27T11:33:50.167247Z",
     "shell.execute_reply.started": "2022-04-27T11:33:34.298348Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "flatten_10 (Flatten)         (None, 128000)            0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               32768256  \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 61,615,617\n",
      "Trainable params: 61,615,105\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3cd61560-ecb2-49dc-a4c6-2262655ebf86",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "63080a9f-c1eb-43c2-bcb7-70895e5940f0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:33:50.168846Z",
     "iopub.status.busy": "2022-04-27T11:33:50.168599Z",
     "iopub.status.idle": "2022-04-27T11:34:09.140187Z",
     "shell.execute_reply": "2022-04-27T11:34:09.139682Z",
     "shell.execute_reply.started": "2022-04-27T11:33:50.168824Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"MLP\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "flatten_11 (Flatten)         (None, 128000)            0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               32768256  \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 61,615,617\n",
      "Trainable params: 61,615,105\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.mlp(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "332bc0d2-4a0b-4887-a3b4-3c3241c2e1f7",
   "metadata": {},
   "source": [
    "- Save results from MLPs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2cd755ea-ad94-494b-98c7-7e20ce37e89f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:34:09.141461Z",
     "iopub.status.busy": "2022-04-27T11:34:09.141106Z",
     "iopub.status.idle": "2022-04-27T11:34:09.147840Z",
     "shell.execute_reply": "2022-04-27T11:34:09.147325Z",
     "shell.execute_reply.started": "2022-04-27T11:34:09.141427Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_results.to_csv(\"all_results_table.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb770cba-9e1e-497c-9e7a-f53ea797151f",
   "metadata": {
    "tags": []
   },
   "source": [
    "# CNN"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab5dbeb5-276a-425c-aa34-3a1e96318dec",
   "metadata": {
    "tags": []
   },
   "source": [
    "## No Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "75917e67-1800-4ec0-aa72-df1d003c638d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:34:09.148807Z",
     "iopub.status.busy": "2022-04-27T11:34:09.148644Z",
     "iopub.status.idle": "2022-04-27T11:34:09.151794Z",
     "shell.execute_reply": "2022-04-27T11:34:09.151306Z",
     "shell.execute_reply.started": "2022-04-27T11:34:09.148787Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict = dict()\n",
    "model_dict[\"name\"] = \"cnn\"\n",
    "model_dict[\"embedding\"] = None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92cb4813-292a-4eb4-839b-ff22e1b587ee",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ceae06bb-2020-4414-a930-3cd3bd40e9ec",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:34:09.152939Z",
     "iopub.status.busy": "2022-04-27T11:34:09.152799Z",
     "iopub.status.idle": "2022-04-27T11:34:17.214130Z",
     "shell.execute_reply": "2022-04-27T11:34:17.213514Z",
     "shell.execute_reply.started": "2022-04-27T11:34:09.152923Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Reshaping (Reshape)          (None, 500, 1)            0         \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          2304      \n",
      "_________________________________________________________________\n",
      "max_pooling1d (MaxPooling1D) (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_12 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 16,124,673\n",
      "Trainable params: 16,124,673\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-27 11:34:09.594774: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.7\n",
      "2022-04-27 11:34:10.850932: W tensorflow/stream_executor/gpu/asm_compiler.cc:63] Running ptxas --version returned 256\n",
      "2022-04-27 11:34:10.940601: W tensorflow/stream_executor/gpu/redzone_allocator.cc:314] Internal: ptxas exited with non-zero error code 256, output: \n",
      "Relying on driver to perform ptx compilation. \n",
      "Modify $PATH to customize ptxas location.\n",
      "This message will be only logged once.\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1aa3b284-14b9-444c-a1e4-244a5f9e4348",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2ccc06f3-6678-4144-9af3-f98bbb0e78ce",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:34:17.215481Z",
     "iopub.status.busy": "2022-04-27T11:34:17.215212Z",
     "iopub.status.idle": "2022-04-27T11:34:22.293638Z",
     "shell.execute_reply": "2022-04-27T11:34:22.293131Z",
     "shell.execute_reply.started": "2022-04-27T11:34:17.215453Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Reshaping (Reshape)          (None, 500, 1)            0         \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          2304      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1 (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_13 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 16,124,673\n",
      "Trainable params: 16,124,673\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "569f56e7-8f5a-46f8-bcd7-f3557c234918",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "14158f60-4fd8-46fe-9fa5-16ed914243c8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:34:22.296078Z",
     "iopub.status.busy": "2022-04-27T11:34:22.295846Z",
     "iopub.status.idle": "2022-04-27T11:34:26.675515Z",
     "shell.execute_reply": "2022-04-27T11:34:26.674989Z",
     "shell.execute_reply.started": "2022-04-27T11:34:22.296054Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Reshaping (Reshape)          (None, 500, 1)            0         \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          2304      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1 (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_14 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 16,125,697\n",
      "Trainable params: 16,125,185\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "133f54ea-a0c8-4623-84ad-8e8f550094d0",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "2b74b653-335d-4122-ad1e-b1099a8bd46b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:34:26.676573Z",
     "iopub.status.busy": "2022-04-27T11:34:26.676342Z",
     "iopub.status.idle": "2022-04-27T11:34:31.947552Z",
     "shell.execute_reply": "2022-04-27T11:34:31.946896Z",
     "shell.execute_reply.started": "2022-04-27T11:34:26.676554Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Reshaping (Reshape)          (None, 500, 1)            0         \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          2304      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_3 (MaxPooling1 (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_15 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 16,125,697\n",
      "Trainable params: 16,125,185\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe936161-680f-4c13-a59f-ee6a20cc1cdf",
   "metadata": {
    "tags": []
   },
   "source": [
    "## GloVe Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "c8c70948-63a5-4564-931d-37990bea3f22",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:34:31.948584Z",
     "iopub.status.busy": "2022-04-27T11:34:31.948349Z",
     "iopub.status.idle": "2022-04-27T11:34:31.951570Z",
     "shell.execute_reply": "2022-04-27T11:34:31.950924Z",
     "shell.execute_reply.started": "2022-04-27T11:34:31.948562Z"
    }
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"glove\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1700b5ee-2e0a-4c33-b952-88a2ab3787ba",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "69124908-f3f1-45cb-97d3-1381ffe9bd76",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:34:31.952443Z",
     "iopub.status.busy": "2022-04-27T11:34:31.952308Z",
     "iopub.status.idle": "2022-04-27T11:34:49.327344Z",
     "shell.execute_reply": "2022-04-27T11:34:49.326610Z",
     "shell.execute_reply.started": "2022-04-27T11:34:31.952427Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          614656    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_4 (MaxPooling1 (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_16 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 50,541,025\n",
      "Trainable params: 16,737,025\n",
      "Non-trainable params: 33,804,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2e35ba4-8303-4725-9df0-050fafacbb73",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "c8b20316-d18c-446c-aaa2-5e52ce6474a2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:34:49.328716Z",
     "iopub.status.busy": "2022-04-27T11:34:49.328449Z",
     "iopub.status.idle": "2022-04-27T11:35:05.700886Z",
     "shell.execute_reply": "2022-04-27T11:35:05.700376Z",
     "shell.execute_reply.started": "2022-04-27T11:34:49.328689Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          614656    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_5 (MaxPooling1 (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_17 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 50,541,025\n",
      "Trainable params: 16,737,025\n",
      "Non-trainable params: 33,804,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4533d238-0577-4fe9-b149-1480f5c06aa6",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "958f5c0b-faf6-45c3-ad44-5c28412f3ffc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:35:05.702367Z",
     "iopub.status.busy": "2022-04-27T11:35:05.701925Z",
     "iopub.status.idle": "2022-04-27T11:35:20.941814Z",
     "shell.execute_reply": "2022-04-27T11:35:20.941148Z",
     "shell.execute_reply.started": "2022-04-27T11:35:05.702333Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          614656    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_6 (MaxPooling1 (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_18 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 50,542,049\n",
      "Trainable params: 16,737,537\n",
      "Non-trainable params: 33,804,512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eff6a9d3-7cf0-4152-b4da-ee0817dc8ebd",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "137f0324-7560-402c-be5f-83fbc15d38af",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:35:20.943224Z",
     "iopub.status.busy": "2022-04-27T11:35:20.942915Z",
     "iopub.status.idle": "2022-04-27T11:35:37.797690Z",
     "shell.execute_reply": "2022-04-27T11:35:37.797180Z",
     "shell.execute_reply.started": "2022-04-27T11:35:20.943193Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          614656    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_7 (MaxPooling1 (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_19 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 50,542,049\n",
      "Trainable params: 16,737,537\n",
      "Non-trainable params: 33,804,512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba6ed87e-91dd-40a7-9595-fc62f51bd4e4",
   "metadata": {
    "tags": []
   },
   "source": [
    "## SpaCy Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "8b5a68da-2d26-475b-ad46-8dcf9ad1168b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:35:37.798782Z",
     "iopub.status.busy": "2022-04-27T11:35:37.798521Z",
     "iopub.status.idle": "2022-04-27T11:35:37.801786Z",
     "shell.execute_reply": "2022-04-27T11:35:37.801292Z",
     "shell.execute_reply.started": "2022-04-27T11:35:37.798758Z"
    }
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"spacy\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00b3ba88-19d6-40a9-957f-b9a8f881c69f",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "54c6b14d-c034-43ab-9a36-a08fd7255733",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:35:37.806517Z",
     "iopub.status.busy": "2022-04-27T11:35:37.806121Z",
     "iopub.status.idle": "2022-04-27T11:35:48.405838Z",
     "shell.execute_reply": "2022-04-27T11:35:48.405151Z",
     "shell.execute_reply.started": "2022-04-27T11:35:37.806496Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          196864    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_8 (MaxPooling1 (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_20 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 27,136,513\n",
      "Trainable params: 16,319,233\n",
      "Non-trainable params: 10,817,280\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b1fb18a-5f98-4a15-998c-69f9bcb6adac",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "164aeeb6-9fca-4268-8772-fe9860edf16e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:35:48.407193Z",
     "iopub.status.busy": "2022-04-27T11:35:48.406914Z",
     "iopub.status.idle": "2022-04-27T11:35:57.485920Z",
     "shell.execute_reply": "2022-04-27T11:35:57.485344Z",
     "shell.execute_reply.started": "2022-04-27T11:35:48.407165Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          196864    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_9 (MaxPooling1 (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_21 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 27,136,513\n",
      "Trainable params: 16,319,233\n",
      "Non-trainable params: 10,817,280\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59e9b606-2ac2-472b-8e17-26f020281c65",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "df801163-72a4-48a2-8f99-1862464432aa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:35:57.487356Z",
     "iopub.status.busy": "2022-04-27T11:35:57.486875Z",
     "iopub.status.idle": "2022-04-27T11:36:05.493719Z",
     "shell.execute_reply": "2022-04-27T11:36:05.493212Z",
     "shell.execute_reply.started": "2022-04-27T11:35:57.487319Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          196864    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_10 (MaxPooling (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_22 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 27,137,537\n",
      "Trainable params: 16,319,745\n",
      "Non-trainable params: 10,817,792\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0d4af91-28a4-4522-851e-0966cd2c3c0e",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "7a5e3883-3e5b-4a27-96eb-86d680777045",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:36:05.494769Z",
     "iopub.status.busy": "2022-04-27T11:36:05.494538Z",
     "iopub.status.idle": "2022-04-27T11:36:14.690364Z",
     "shell.execute_reply": "2022-04-27T11:36:14.689850Z",
     "shell.execute_reply.started": "2022-04-27T11:36:05.494745Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          196864    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_11 (MaxPooling (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_23 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 27,137,537\n",
      "Trainable params: 16,319,745\n",
      "Non-trainable params: 10,817,792\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ad624ae-4918-4386-aee3-c28b3626adb8",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Keras Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "db2f4bf6-857c-4254-9067-f9f72f27ba32",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:36:14.691462Z",
     "iopub.status.busy": "2022-04-27T11:36:14.691203Z",
     "iopub.status.idle": "2022-04-27T11:36:14.694566Z",
     "shell.execute_reply": "2022-04-27T11:36:14.693952Z",
     "shell.execute_reply.started": "2022-04-27T11:36:14.691440Z"
    }
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"keras\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1244e5b3-d976-4882-87f2-6102cb8fa121",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "5d06615d-6387-4d23-ba11-a825700d75a0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:36:14.695452Z",
     "iopub.status.busy": "2022-04-27T11:36:14.695315Z",
     "iopub.status.idle": "2022-04-27T11:36:37.426288Z",
     "shell.execute_reply": "2022-04-27T11:36:37.425561Z",
     "shell.execute_reply.started": "2022-04-27T11:36:14.695434Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          524544    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_12 (MaxPooling (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_24 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 45,492,993\n",
      "Trainable params: 45,492,993\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2dc3900-895c-45ab-adb2-c42478af5349",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "0a1d8b07-f6f5-4b48-848d-5319a438a879",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:36:37.427721Z",
     "iopub.status.busy": "2022-04-27T11:36:37.427164Z",
     "iopub.status.idle": "2022-04-27T11:36:58.504322Z",
     "shell.execute_reply": "2022-04-27T11:36:58.503798Z",
     "shell.execute_reply.started": "2022-04-27T11:36:37.427691Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          524544    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_13 (MaxPooling (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_25 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 45,492,993\n",
      "Trainable params: 45,492,993\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "baa4bb37-13a6-4528-aa2b-6dbbb7a2b570",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "e79dde9b-61c9-42bc-97d2-525a362a3d86",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:36:58.505511Z",
     "iopub.status.busy": "2022-04-27T11:36:58.505339Z",
     "iopub.status.idle": "2022-04-27T11:37:17.574118Z",
     "shell.execute_reply": "2022-04-27T11:37:17.573601Z",
     "shell.execute_reply.started": "2022-04-27T11:36:58.505490Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          524544    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_14 (MaxPooling (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_26 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 45,494,017\n",
      "Trainable params: 45,493,505\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "094726cc-2597-4c31-a62b-927b9f409f45",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "0cbb2b30-4dde-4bff-8bf9-19b68e9a96fc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:37:17.575165Z",
     "iopub.status.busy": "2022-04-27T11:37:17.574942Z",
     "iopub.status.idle": "2022-04-27T11:37:38.159210Z",
     "shell.execute_reply": "2022-04-27T11:37:38.158687Z",
     "shell.execute_reply.started": "2022-04-27T11:37:17.575143Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Conv_layer (Conv1D)          (None, 493, 256)          524544    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_15 (MaxPooling (None, 246, 256)          0         \n",
      "_________________________________________________________________\n",
      "flatten_27 (Flatten)         (None, 62976)             0         \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16122112  \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 45,494,017\n",
      "Trainable params: 45,493,505\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.cnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a04c7d96-3c7c-476b-9669-c26eb451d516",
   "metadata": {},
   "source": [
    "- Save results from CNNs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "4795c8d6-12e8-48e0-a2d7-1d007f53360b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:37:38.160274Z",
     "iopub.status.busy": "2022-04-27T11:37:38.160043Z",
     "iopub.status.idle": "2022-04-27T11:37:38.167554Z",
     "shell.execute_reply": "2022-04-27T11:37:38.166903Z",
     "shell.execute_reply.started": "2022-04-27T11:37:38.160254Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_results.to_csv(\"all_results_table.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15766908-fbb9-4346-a471-3fa0e19f1d8a",
   "metadata": {
    "tags": []
   },
   "source": [
    "# RNN"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e1da29e-3c0d-4fd1-bcd2-85ad399dba83",
   "metadata": {
    "tags": []
   },
   "source": [
    "## No Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "ee3fc1e1-1a0d-4125-a0b3-c69b615184f8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:37:38.168480Z",
     "iopub.status.busy": "2022-04-27T11:37:38.168269Z",
     "iopub.status.idle": "2022-04-27T11:37:38.171906Z",
     "shell.execute_reply": "2022-04-27T11:37:38.171299Z",
     "shell.execute_reply.started": "2022-04-27T11:37:38.168459Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict = dict()\n",
    "model_dict[\"name\"] = \"rnn\"\n",
    "model_dict[\"embedding\"] = None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a25971ce-80b5-42a5-afdf-287ed20df9e6",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "8dc14767-885d-4eef-81d0-6420ee28f9ae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:37:38.172969Z",
     "iopub.status.busy": "2022-04-27T11:37:38.172651Z",
     "iopub.status.idle": "2022-04-27T11:38:54.863134Z",
     "shell.execute_reply": "2022-04-27T11:38:54.862618Z",
     "shell.execute_reply.started": "2022-04-27T11:37:38.172915Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                2176      \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 19,073\n",
      "Trainable params: 19,073\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "323d20b0-2304-4d18-afdc-1474042ec583",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "499e2910-ba98-4ead-b165-b0451ac684ae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:38:54.864230Z",
     "iopub.status.busy": "2022-04-27T11:38:54.863997Z",
     "iopub.status.idle": "2022-04-27T11:40:12.415882Z",
     "shell.execute_reply": "2022-04-27T11:40:12.415350Z",
     "shell.execute_reply.started": "2022-04-27T11:38:54.864206Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                2176      \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 19,073\n",
      "Trainable params: 19,073\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b077a9a-94f9-45de-aa19-98e54133d0de",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "98dae137-fe5b-416f-8ad7-13aea8e6680a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:40:12.417021Z",
     "iopub.status.busy": "2022-04-27T11:40:12.416756Z",
     "iopub.status.idle": "2022-04-27T11:41:29.350289Z",
     "shell.execute_reply": "2022-04-27T11:41:29.349773Z",
     "shell.execute_reply.started": "2022-04-27T11:40:12.416998Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                2176      \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 20,097\n",
      "Trainable params: 19,585\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79c30e64-dfbf-45b7-baf8-edb8a5f79cdb",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "8fcf8808-80bf-474b-967f-1663b71bc405",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:41:29.351664Z",
     "iopub.status.busy": "2022-04-27T11:41:29.351106Z",
     "iopub.status.idle": "2022-04-27T11:42:45.526326Z",
     "shell.execute_reply": "2022-04-27T11:42:45.525811Z",
     "shell.execute_reply.started": "2022-04-27T11:41:29.351638Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                2176      \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 20,097\n",
      "Trainable params: 19,585\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14a2f3f2-e697-4ad4-b8b2-f0d81598166a",
   "metadata": {
    "tags": []
   },
   "source": [
    "## GloVe Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "e249cb9b-2481-4b93-8aee-3b7eb53d083a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:42:45.527344Z",
     "iopub.status.busy": "2022-04-27T11:42:45.527164Z",
     "iopub.status.idle": "2022-04-27T11:42:45.530678Z",
     "shell.execute_reply": "2022-04-27T11:42:45.530041Z",
     "shell.execute_reply.started": "2022-04-27T11:42:45.527324Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"glove\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00f96853-b4b6-4785-9206-df393e23116d",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "a030df31-86cc-4995-9f89-eabd48cf0a48",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:42:45.532016Z",
     "iopub.status.busy": "2022-04-27T11:42:45.531581Z",
     "iopub.status.idle": "2022-04-27T11:44:16.061768Z",
     "shell.execute_reply": "2022-04-27T11:44:16.061129Z",
     "shell.execute_reply.started": "2022-04-27T11:42:45.531981Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                21312     \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 33,842,209\n",
      "Trainable params: 38,209\n",
      "Non-trainable params: 33,804,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3b85e3f-55f8-4f8d-92ee-b20f2a7cbf90",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "1e91c464-c7db-45c3-b94b-d47a0d3d9287",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:44:16.063255Z",
     "iopub.status.busy": "2022-04-27T11:44:16.062783Z",
     "iopub.status.idle": "2022-04-27T11:45:46.400933Z",
     "shell.execute_reply": "2022-04-27T11:45:46.400421Z",
     "shell.execute_reply.started": "2022-04-27T11:44:16.063221Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                21312     \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 33,842,209\n",
      "Trainable params: 38,209\n",
      "Non-trainable params: 33,804,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ffabce72-fcc4-4273-a7d3-95068e587f6e",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "d35482e8-1112-49a3-a193-4a3a6d609709",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:45:46.401866Z",
     "iopub.status.busy": "2022-04-27T11:45:46.401721Z",
     "iopub.status.idle": "2022-04-27T11:47:17.129146Z",
     "shell.execute_reply": "2022-04-27T11:47:17.128253Z",
     "shell.execute_reply.started": "2022-04-27T11:45:46.401848Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                21312     \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 33,843,233\n",
      "Trainable params: 38,721\n",
      "Non-trainable params: 33,804,512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f806ca83-0eff-4071-b336-6e7120078980",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "18ee53e1-5c93-44f1-a6ad-64c6986cc66b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:47:17.130890Z",
     "iopub.status.busy": "2022-04-27T11:47:17.130335Z",
     "iopub.status.idle": "2022-04-27T11:48:48.395358Z",
     "shell.execute_reply": "2022-04-27T11:48:48.394846Z",
     "shell.execute_reply.started": "2022-04-27T11:47:17.130853Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                21312     \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 33,843,233\n",
      "Trainable params: 38,721\n",
      "Non-trainable params: 33,804,512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df5062ff-cb2b-4117-a9f3-9ac412824bc5",
   "metadata": {
    "tags": []
   },
   "source": [
    "## SpaCy Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "3a9f7859-757d-412e-90ab-bfb6aa755f13",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:48:48.396824Z",
     "iopub.status.busy": "2022-04-27T11:48:48.396276Z",
     "iopub.status.idle": "2022-04-27T11:48:48.400060Z",
     "shell.execute_reply": "2022-04-27T11:48:48.399447Z",
     "shell.execute_reply.started": "2022-04-27T11:48:48.396790Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"spacy\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62c1fcdc-d9a0-4c03-b1a0-4889a1e2e66c",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "cdbaa6bd-24f9-44dc-8ca7-3fc977888892",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:48:48.401544Z",
     "iopub.status.busy": "2022-04-27T11:48:48.401050Z",
     "iopub.status.idle": "2022-04-27T11:50:08.291472Z",
     "shell.execute_reply": "2022-04-27T11:50:08.290858Z",
     "shell.execute_reply.started": "2022-04-27T11:48:48.401511Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 10,842,433\n",
      "Trainable params: 25,153\n",
      "Non-trainable params: 10,817,280\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89f1bc1a-f6ce-4f74-b843-04ef4813ba89",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "2751d334-9dc3-4d00-92f4-3964416b891e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:50:08.292907Z",
     "iopub.status.busy": "2022-04-27T11:50:08.292665Z",
     "iopub.status.idle": "2022-04-27T11:51:28.569526Z",
     "shell.execute_reply": "2022-04-27T11:51:28.568940Z",
     "shell.execute_reply.started": "2022-04-27T11:50:08.292876Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 10,842,433\n",
      "Trainable params: 25,153\n",
      "Non-trainable params: 10,817,280\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4233c651-1aa1-46ec-8b1b-55d64c8f8a18",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "e0411a13-fa40-4208-a270-13ba0ea2d810",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:51:28.570875Z",
     "iopub.status.busy": "2022-04-27T11:51:28.570522Z",
     "iopub.status.idle": "2022-04-27T11:52:49.476277Z",
     "shell.execute_reply": "2022-04-27T11:52:49.475767Z",
     "shell.execute_reply.started": "2022-04-27T11:51:28.570840Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 10,843,457\n",
      "Trainable params: 25,665\n",
      "Non-trainable params: 10,817,792\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bba9292-1c4d-4256-80a3-b42be31bb982",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "e41be950-ec40-4fdb-8606-341d9148d5c0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:52:49.477328Z",
     "iopub.status.busy": "2022-04-27T11:52:49.477107Z",
     "iopub.status.idle": "2022-04-27T11:54:09.910486Z",
     "shell.execute_reply": "2022-04-27T11:54:09.909977Z",
     "shell.execute_reply.started": "2022-04-27T11:52:49.477307Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 10,843,457\n",
      "Trainable params: 25,665\n",
      "Non-trainable params: 10,817,792\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b78759b9-bc6b-473a-ad53-10367f26d8e7",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Keras Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "15041ce5-28a4-425b-a589-68d48799abde",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:54:09.912086Z",
     "iopub.status.busy": "2022-04-27T11:54:09.911502Z",
     "iopub.status.idle": "2022-04-27T11:54:09.915356Z",
     "shell.execute_reply": "2022-04-27T11:54:09.914734Z",
     "shell.execute_reply.started": "2022-04-27T11:54:09.912051Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"keras\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05ec7021-1c07-4fda-8c73-f37f12d52fcb",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "55fa5879-b62d-44c5-996c-66e4dd542cb3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:54:09.916830Z",
     "iopub.status.busy": "2022-04-27T11:54:09.916284Z",
     "iopub.status.idle": "2022-04-27T11:55:45.205059Z",
     "shell.execute_reply": "2022-04-27T11:55:45.204475Z",
     "shell.execute_reply.started": "2022-04-27T11:54:09.916798Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                18496     \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 28,881,473\n",
      "Trainable params: 28,881,473\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc64d25d-68a1-4507-af56-a002b8073b71",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "8c727e5b-0995-44b0-a1b3-328b234531c5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:55:45.206160Z",
     "iopub.status.busy": "2022-04-27T11:55:45.205933Z",
     "iopub.status.idle": "2022-04-27T11:57:22.045609Z",
     "shell.execute_reply": "2022-04-27T11:57:22.044387Z",
     "shell.execute_reply.started": "2022-04-27T11:55:45.206139Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                18496     \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 28,881,473\n",
      "Trainable params: 28,881,473\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76dc1401-a962-41f1-b292-c78382c131d1",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "1f64750d-c387-4b57-9747-264ce691c458",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:57:22.047049Z",
     "iopub.status.busy": "2022-04-27T11:57:22.046590Z",
     "iopub.status.idle": "2022-04-27T11:58:57.567050Z",
     "shell.execute_reply": "2022-04-27T11:58:57.566533Z",
     "shell.execute_reply.started": "2022-04-27T11:57:22.047015Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                18496     \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 28,882,497\n",
      "Trainable params: 28,881,985\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "87237c00-bedf-446f-a358-89eed839b2ff",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "d1b879a1-29ac-43f8-8cc0-69eeec731a4b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T11:58:57.568279Z",
     "iopub.status.busy": "2022-04-27T11:58:57.568039Z",
     "iopub.status.idle": "2022-04-27T12:00:33.994042Z",
     "shell.execute_reply": "2022-04-27T12:00:33.993340Z",
     "shell.execute_reply.started": "2022-04-27T11:58:57.568249Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Bidirectional_RNN (Bidirecti (None, 64)                18496     \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 28,882,497\n",
      "Trainable params: 28,881,985\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_rnn(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d43cd9a-8d53-4f05-aca9-aadb59f3b7ea",
   "metadata": {},
   "source": [
    "- Save results from RNNs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "ba43d132-6d68-486a-aad5-cddae6235dbf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:00:33.995711Z",
     "iopub.status.busy": "2022-04-27T12:00:33.995137Z",
     "iopub.status.idle": "2022-04-27T12:00:34.000905Z",
     "shell.execute_reply": "2022-04-27T12:00:34.000287Z",
     "shell.execute_reply.started": "2022-04-27T12:00:33.995677Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_results.to_csv(\"all_results_table.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "048c797e-8ed9-4300-9a55-83826ce9314c",
   "metadata": {
    "tags": []
   },
   "source": [
    "# LSTM"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ec07b39-9ec3-46dd-a408-cf9bf2406236",
   "metadata": {
    "tags": []
   },
   "source": [
    "## No Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "d0cae2f4-d9ad-485f-909d-599ffc116bf4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:00:34.002205Z",
     "iopub.status.busy": "2022-04-27T12:00:34.001933Z",
     "iopub.status.idle": "2022-04-27T12:00:34.006154Z",
     "shell.execute_reply": "2022-04-27T12:00:34.005547Z",
     "shell.execute_reply.started": "2022-04-27T12:00:34.002174Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict = dict()\n",
    "model_dict[\"name\"] = \"lstm\"\n",
    "model_dict[\"embedding\"] = None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9bbb388e-3753-4dd9-a022-803570383026",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "6023faa7-fe9b-437c-bfe1-f207861680f1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:18:59.158992Z",
     "iopub.status.busy": "2022-04-27T12:18:59.158695Z",
     "iopub.status.idle": "2022-04-27T12:18:59.509819Z",
     "shell.execute_reply": "2022-04-27T12:18:59.509220Z",
     "shell.execute_reply.started": "2022-04-27T12:18:59.158966Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                8704      \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 25,601\n",
      "Trainable params: 25,601\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3bd00c78-2da1-4cf2-8fed-437e1beebaee",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "56c41474-5935-4bf4-b4b6-a4c2feb236c3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:22:08.385482Z",
     "iopub.status.busy": "2022-04-27T12:22:08.385174Z",
     "iopub.status.idle": "2022-04-27T12:22:08.728506Z",
     "shell.execute_reply": "2022-04-27T12:22:08.727903Z",
     "shell.execute_reply.started": "2022-04-27T12:22:08.385456Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                8704      \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 25,601\n",
      "Trainable params: 25,601\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08e5c0b1-d088-4f61-89d6-30cf6f5c554d",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "6c535ae0-601d-4569-81ad-989ae784f6d9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:22:17.140001Z",
     "iopub.status.busy": "2022-04-27T12:22:17.139712Z",
     "iopub.status.idle": "2022-04-27T12:22:17.498976Z",
     "shell.execute_reply": "2022-04-27T12:22:17.498304Z",
     "shell.execute_reply.started": "2022-04-27T12:22:17.139975Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                8704      \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 26,625\n",
      "Trainable params: 26,113\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b90c4328-687e-4830-ba8a-b9efe35ad09e",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "8ad602e7-9011-47d2-ac50-5dc55c4ca25b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:22:25.371438Z",
     "iopub.status.busy": "2022-04-27T12:22:25.371036Z",
     "iopub.status.idle": "2022-04-27T12:22:25.727454Z",
     "shell.execute_reply": "2022-04-27T12:22:25.726771Z",
     "shell.execute_reply.started": "2022-04-27T12:22:25.371403Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                8704      \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 26,625\n",
      "Trainable params: 26,113\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1632ba65-353c-449b-b4ed-946f26e082fc",
   "metadata": {
    "tags": []
   },
   "source": [
    "## GloVe Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "04a51187-e900-4437-a4f0-553d35a137b3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:22:31.950854Z",
     "iopub.status.busy": "2022-04-27T12:22:31.950567Z",
     "iopub.status.idle": "2022-04-27T12:22:31.957361Z",
     "shell.execute_reply": "2022-04-27T12:22:31.956595Z",
     "shell.execute_reply.started": "2022-04-27T12:22:31.950829Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"glove\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30374877-727b-424c-a959-734b59c39bbf",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "ee6578db-0a12-49a1-b366-e8b1d4849e03",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:22:34.993271Z",
     "iopub.status.busy": "2022-04-27T12:22:34.992979Z",
     "iopub.status.idle": "2022-04-27T12:23:02.594462Z",
     "shell.execute_reply": "2022-04-27T12:23:02.593879Z",
     "shell.execute_reply.started": "2022-04-27T12:22:34.993248Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                85248     \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 33,906,145\n",
      "Trainable params: 102,145\n",
      "Non-trainable params: 33,804,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0837417d-a610-42ac-a3a1-8d5737debdb1",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "48b4dd3f-b9d0-4771-9473-57a383b4a49d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:23:02.595833Z",
     "iopub.status.busy": "2022-04-27T12:23:02.595589Z",
     "iopub.status.idle": "2022-04-27T12:23:29.954598Z",
     "shell.execute_reply": "2022-04-27T12:23:29.954078Z",
     "shell.execute_reply.started": "2022-04-27T12:23:02.595813Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                85248     \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 33,906,145\n",
      "Trainable params: 102,145\n",
      "Non-trainable params: 33,804,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06cca9fa-8153-40a4-b733-df93db194ff0",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "08953a50-548a-43a0-a02c-1483d9f016b7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:23:29.955724Z",
     "iopub.status.busy": "2022-04-27T12:23:29.955473Z",
     "iopub.status.idle": "2022-04-27T12:23:56.507930Z",
     "shell.execute_reply": "2022-04-27T12:23:56.506930Z",
     "shell.execute_reply.started": "2022-04-27T12:23:29.955701Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                85248     \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 33,907,169\n",
      "Trainable params: 102,657\n",
      "Non-trainable params: 33,804,512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ec21ed1-7648-4f84-8541-d9150964ac1e",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "b63bfbdb-bebe-48c3-b6ed-7af645e077bf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:23:56.510038Z",
     "iopub.status.busy": "2022-04-27T12:23:56.509514Z",
     "iopub.status.idle": "2022-04-27T12:24:24.039232Z",
     "shell.execute_reply": "2022-04-27T12:24:24.038545Z",
     "shell.execute_reply.started": "2022-04-27T12:23:56.510002Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                85248     \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 33,907,169\n",
      "Trainable params: 102,657\n",
      "Non-trainable params: 33,804,512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "007ca4ad-6de7-4e4c-8014-3bce2442a4f9",
   "metadata": {
    "tags": []
   },
   "source": [
    "## SpaCy Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "0769ee6f-9fed-4e44-b2fc-7eb1af3f10ff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:24:24.040781Z",
     "iopub.status.busy": "2022-04-27T12:24:24.040204Z",
     "iopub.status.idle": "2022-04-27T12:24:24.044012Z",
     "shell.execute_reply": "2022-04-27T12:24:24.043391Z",
     "shell.execute_reply.started": "2022-04-27T12:24:24.040745Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"spacy\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba2804a6-406e-4968-966b-d52819da513c",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "bc6ecf93-863f-4c14-adb1-8e84528b3e0c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:24:24.045479Z",
     "iopub.status.busy": "2022-04-27T12:24:24.044983Z",
     "iopub.status.idle": "2022-04-27T12:24:45.080608Z",
     "shell.execute_reply": "2022-04-27T12:24:45.080021Z",
     "shell.execute_reply.started": "2022-04-27T12:24:24.045454Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                33024     \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 10,867,201\n",
      "Trainable params: 49,921\n",
      "Non-trainable params: 10,817,280\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57bfb71e-972e-48cc-9bce-d1b2e4608727",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "6746a342-723e-412a-aa0a-70296aa95f22",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:24:45.081769Z",
     "iopub.status.busy": "2022-04-27T12:24:45.081531Z",
     "iopub.status.idle": "2022-04-27T12:25:06.981684Z",
     "shell.execute_reply": "2022-04-27T12:25:06.981176Z",
     "shell.execute_reply.started": "2022-04-27T12:24:45.081746Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                33024     \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 10,867,201\n",
      "Trainable params: 49,921\n",
      "Non-trainable params: 10,817,280\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1e70b7c-54fb-4fac-953b-841daf8b0790",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "769d319d-bad4-47b3-81c8-63deb65a0dc8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:25:06.982812Z",
     "iopub.status.busy": "2022-04-27T12:25:06.982654Z",
     "iopub.status.idle": "2022-04-27T12:25:28.395091Z",
     "shell.execute_reply": "2022-04-27T12:25:28.394581Z",
     "shell.execute_reply.started": "2022-04-27T12:25:06.982792Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                33024     \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 10,868,225\n",
      "Trainable params: 50,433\n",
      "Non-trainable params: 10,817,792\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5a34f15-ad70-4919-938d-34e87a876c64",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "41039cbd-49cb-4023-93db-e5d799a24c6b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:25:28.396195Z",
     "iopub.status.busy": "2022-04-27T12:25:28.395965Z",
     "iopub.status.idle": "2022-04-27T12:25:50.311245Z",
     "shell.execute_reply": "2022-04-27T12:25:50.310551Z",
     "shell.execute_reply.started": "2022-04-27T12:25:28.396173Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                33024     \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 10,868,225\n",
      "Trainable params: 50,433\n",
      "Non-trainable params: 10,817,792\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b31945d-2f4c-46b7-a7bc-1df4c1ae521c",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Keras Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "332b6d9e-5352-4ea8-80f2-98689d81cd23",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:25:50.313984Z",
     "iopub.status.busy": "2022-04-27T12:25:50.313640Z",
     "iopub.status.idle": "2022-04-27T12:25:50.318084Z",
     "shell.execute_reply": "2022-04-27T12:25:50.317172Z",
     "shell.execute_reply.started": "2022-04-27T12:25:50.313948Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"keras\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de39a751-e3df-4346-9a56-9c140213dddd",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "bff457c3-1254-47eb-92ba-69876f0489a7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:25:50.319029Z",
     "iopub.status.busy": "2022-04-27T12:25:50.318817Z",
     "iopub.status.idle": "2022-04-27T12:26:21.246269Z",
     "shell.execute_reply": "2022-04-27T12:26:21.245669Z",
     "shell.execute_reply.started": "2022-04-27T12:25:50.318997Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                73984     \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 28,936,961\n",
      "Trainable params: 28,936,961\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d632b68e-45d6-441f-9c72-ff7c8e0d42b9",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "b05d0d02-60c0-4497-ad51-284c38a80c0e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:26:21.247440Z",
     "iopub.status.busy": "2022-04-27T12:26:21.247061Z",
     "iopub.status.idle": "2022-04-27T12:26:53.106308Z",
     "shell.execute_reply": "2022-04-27T12:26:53.105803Z",
     "shell.execute_reply.started": "2022-04-27T12:26:21.247418Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                73984     \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 28,936,961\n",
      "Trainable params: 28,936,961\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53831495-9690-4214-9e53-b9ab4a0fc2b6",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "f16bad3e-20e8-4efc-978e-bd12b6cbd1a0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:26:53.107759Z",
     "iopub.status.busy": "2022-04-27T12:26:53.107300Z",
     "iopub.status.idle": "2022-04-27T12:27:24.790444Z",
     "shell.execute_reply": "2022-04-27T12:27:24.789858Z",
     "shell.execute_reply.started": "2022-04-27T12:26:53.107724Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                73984     \n",
      "_________________________________________________________________\n",
      "Hidden_dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 28,937,985\n",
      "Trainable params: 28,937,473\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ae4aaf0-5ddb-4a2b-8429-8abdf012ad95",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "ff812d0e-1599-44c2-b7cb-3da1ade9a790",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:27:24.791793Z",
     "iopub.status.busy": "2022-04-27T12:27:24.791422Z",
     "iopub.status.idle": "2022-04-27T12:27:57.600595Z",
     "shell.execute_reply": "2022-04-27T12:27:57.599685Z",
     "shell.execute_reply.started": "2022-04-27T12:27:24.791759Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"LSTM\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Bidirectional_LSTM (Bidirect (None, 64)                73984     \n",
      "_________________________________________________________________\n",
      "Regularized_hidden_dense (De (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_normalization (BatchNo (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLu_activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 28,937,985\n",
      "Trainable params: 28,937,473\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_lstm(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89a5710a-5dc1-41b9-ab4d-87527aa916b3",
   "metadata": {},
   "source": [
    "- Save results from LSTMs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "52bf9ed6-b43c-4b0b-9687-409375c34712",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:27:57.601986Z",
     "iopub.status.busy": "2022-04-27T12:27:57.601608Z",
     "iopub.status.idle": "2022-04-27T12:27:57.608278Z",
     "shell.execute_reply": "2022-04-27T12:27:57.607670Z",
     "shell.execute_reply.started": "2022-04-27T12:27:57.601951Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_results.to_csv(\"all_results_table.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "603f3b40-29d9-4a30-9c93-329f9a4ff01c",
   "metadata": {
    "tags": []
   },
   "source": [
    "# GRU"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33161195-5d89-4463-ac0d-c2b3f6a6486f",
   "metadata": {
    "tags": []
   },
   "source": [
    "## No Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "8333c886-a298-4ed3-a375-757246821a89",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:30:37.098705Z",
     "iopub.status.busy": "2022-04-27T12:30:37.098394Z",
     "iopub.status.idle": "2022-04-27T12:30:37.102144Z",
     "shell.execute_reply": "2022-04-27T12:30:37.101503Z",
     "shell.execute_reply.started": "2022-04-27T12:30:37.098679Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict = dict()\n",
    "model_dict[\"name\"] = \"gru\"\n",
    "model_dict[\"embedding\"] = None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6159f71e-bd3b-4329-8aec-fd164cffd000",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "2980ced8-ba89-4a4d-b3b6-9fc5e67a9457",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:30:38.038605Z",
     "iopub.status.busy": "2022-04-27T12:30:38.038315Z",
     "iopub.status.idle": "2022-04-27T12:30:55.852754Z",
     "shell.execute_reply": "2022-04-27T12:30:55.852219Z",
     "shell.execute_reply.started": "2022-04-27T12:30:38.038580Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Reshaping (Reshape)          (None, 500, 1)            0         \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                6720      \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 23,617\n",
      "Trainable params: 23,617\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee83bee6-a0ff-477a-9158-6e9931097073",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "a0051552-5b16-4455-b683-47556b9a1970",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:30:55.856225Z",
     "iopub.status.busy": "2022-04-27T12:30:55.855987Z",
     "iopub.status.idle": "2022-04-27T12:31:14.719744Z",
     "shell.execute_reply": "2022-04-27T12:31:14.719186Z",
     "shell.execute_reply.started": "2022-04-27T12:30:55.856203Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Reshaping (Reshape)          (None, 500, 1)            0         \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                6720      \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 23,617\n",
      "Trainable params: 23,617\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4eabeec9-d5da-4838-8544-b37e9a9e047b",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "a016c8dd-b765-4ba3-8cea-d80328f4e10c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:31:14.721360Z",
     "iopub.status.busy": "2022-04-27T12:31:14.720892Z",
     "iopub.status.idle": "2022-04-27T12:31:32.718880Z",
     "shell.execute_reply": "2022-04-27T12:31:32.718259Z",
     "shell.execute_reply.started": "2022-04-27T12:31:14.721324Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Reshaping (Reshape)          (None, 500, 1)            0         \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                6720      \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 24,641\n",
      "Trainable params: 24,129\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19a1f608-6e1f-44ce-9bb2-c3bc031a6d23",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "03802337-1059-4caf-ac09-5af35a8b6a34",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:31:32.720514Z",
     "iopub.status.busy": "2022-04-27T12:31:32.720290Z",
     "iopub.status.idle": "2022-04-27T12:31:51.806766Z",
     "shell.execute_reply": "2022-04-27T12:31:51.806252Z",
     "shell.execute_reply.started": "2022-04-27T12:31:32.720491Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Reshaping (Reshape)          (None, 500, 1)            0         \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                6720      \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 24,641\n",
      "Trainable params: 24,129\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c43a5848-04e6-4bce-925d-3489090a7ac7",
   "metadata": {
    "tags": []
   },
   "source": [
    "## GloVe Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "dc4974a8-fd0d-43c7-bac0-da92ba5d0619",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:31:51.808057Z",
     "iopub.status.busy": "2022-04-27T12:31:51.807698Z",
     "iopub.status.idle": "2022-04-27T12:31:51.811376Z",
     "shell.execute_reply": "2022-04-27T12:31:51.810851Z",
     "shell.execute_reply.started": "2022-04-27T12:31:51.808023Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"glove\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9d9163a-a2a3-4285-8c50-f415f5f36bfb",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "ad352cc1-bd97-4eca-9c32-852ab729dd58",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:31:51.812362Z",
     "iopub.status.busy": "2022-04-27T12:31:51.812191Z",
     "iopub.status.idle": "2022-04-27T12:32:18.762079Z",
     "shell.execute_reply": "2022-04-27T12:32:18.761554Z",
     "shell.execute_reply.started": "2022-04-27T12:31:51.812339Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                64128     \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 33,885,025\n",
      "Trainable params: 81,025\n",
      "Non-trainable params: 33,804,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bbbcfa29-9dc6-497f-9770-d8923b15d879",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "268c7089-c2e6-4605-9636-b1ec2ed378a3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:32:18.763100Z",
     "iopub.status.busy": "2022-04-27T12:32:18.762920Z",
     "iopub.status.idle": "2022-04-27T12:32:47.022344Z",
     "shell.execute_reply": "2022-04-27T12:32:47.021830Z",
     "shell.execute_reply.started": "2022-04-27T12:32:18.763073Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                64128     \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 33,885,025\n",
      "Trainable params: 81,025\n",
      "Non-trainable params: 33,804,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45ca1c68-43c6-431a-ac54-de52ccfc0349",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "d624e127-141e-4ef8-851c-ed4340007b89",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:32:47.023491Z",
     "iopub.status.busy": "2022-04-27T12:32:47.023235Z",
     "iopub.status.idle": "2022-04-27T12:33:14.432989Z",
     "shell.execute_reply": "2022-04-27T12:33:14.432481Z",
     "shell.execute_reply.started": "2022-04-27T12:32:47.023468Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                64128     \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 33,886,049\n",
      "Trainable params: 81,537\n",
      "Non-trainable params: 33,804,512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "544f1a2f-51cc-4fb0-8f69-3e9dd8c3ace6",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "f12435b7-4b12-4249-bb56-302f11b49530",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:33:14.434058Z",
     "iopub.status.busy": "2022-04-27T12:33:14.433840Z",
     "iopub.status.idle": "2022-04-27T12:33:42.503827Z",
     "shell.execute_reply": "2022-04-27T12:33:42.503227Z",
     "shell.execute_reply.started": "2022-04-27T12:33:14.434036Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "glove_embedding (Embedding)  (None, 500, 300)          33804000  \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                64128     \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 33,886,049\n",
      "Trainable params: 81,537\n",
      "Non-trainable params: 33,804,512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c418aeb-8c07-40a2-a097-8305d23272e6",
   "metadata": {
    "tags": []
   },
   "source": [
    "## SpaCy Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "f5dc80cb-14ca-442d-bb51-9e864d12cbf1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:33:42.506547Z",
     "iopub.status.busy": "2022-04-27T12:33:42.506001Z",
     "iopub.status.idle": "2022-04-27T12:33:42.512277Z",
     "shell.execute_reply": "2022-04-27T12:33:42.511692Z",
     "shell.execute_reply.started": "2022-04-27T12:33:42.506510Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"spacy\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76a0edcb-4313-447c-9771-68435a210ef9",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "de15b79f-bfba-48fb-9b35-a0dc990a122a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:33:42.513203Z",
     "iopub.status.busy": "2022-04-27T12:33:42.513063Z",
     "iopub.status.idle": "2022-04-27T12:34:03.525891Z",
     "shell.execute_reply": "2022-04-27T12:34:03.525006Z",
     "shell.execute_reply.started": "2022-04-27T12:33:42.513185Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                24960     \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 10,859,137\n",
      "Trainable params: 41,857\n",
      "Non-trainable params: 10,817,280\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67eef011-1405-41fe-9a3d-bda846ee6369",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "4b99427f-4ad1-40f9-bed4-780e41ce895b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:34:03.527015Z",
     "iopub.status.busy": "2022-04-27T12:34:03.526858Z",
     "iopub.status.idle": "2022-04-27T12:34:25.497576Z",
     "shell.execute_reply": "2022-04-27T12:34:25.497058Z",
     "shell.execute_reply.started": "2022-04-27T12:34:03.526995Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                24960     \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 10,859,137\n",
      "Trainable params: 41,857\n",
      "Non-trainable params: 10,817,280\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6248e5ec-e980-4954-8bdd-73e7497d208b",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "36b71042-36d9-440d-a6b4-c51c3aaa61de",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:34:25.499200Z",
     "iopub.status.busy": "2022-04-27T12:34:25.498510Z",
     "iopub.status.idle": "2022-04-27T12:34:46.590692Z",
     "shell.execute_reply": "2022-04-27T12:34:46.590068Z",
     "shell.execute_reply.started": "2022-04-27T12:34:25.499163Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                24960     \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 10,860,161\n",
      "Trainable params: 42,369\n",
      "Non-trainable params: 10,817,792\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d69f7af-5f86-4599-b8d7-a8fa976e03fe",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "c13cd5b2-4193-4841-9bba-380048d26616",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:34:46.592162Z",
     "iopub.status.busy": "2022-04-27T12:34:46.591704Z",
     "iopub.status.idle": "2022-04-27T12:35:08.759572Z",
     "shell.execute_reply": "2022-04-27T12:35:08.759021Z",
     "shell.execute_reply.started": "2022-04-27T12:34:46.592127Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "spacy_embedding (Embedding)  (None, 500, 96)           10817280  \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                24960     \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 10,860,161\n",
      "Trainable params: 42,369\n",
      "Non-trainable params: 10,817,792\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4184b3a7-15d3-4a9c-80bc-c1074f1635b3",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Keras Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "885c59fa-4290-4582-93e9-08fa853b5284",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:35:08.760578Z",
     "iopub.status.busy": "2022-04-27T12:35:08.760423Z",
     "iopub.status.idle": "2022-04-27T12:35:08.764808Z",
     "shell.execute_reply": "2022-04-27T12:35:08.764163Z",
     "shell.execute_reply.started": "2022-04-27T12:35:08.760558Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set model architecture\n",
    "model_dict[\"embedding\"] = \"keras\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1df2406-c632-4284-8cac-e33c78217993",
   "metadata": {
    "tags": []
   },
   "source": [
    "### No Regularisation or Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "ed4bbf02-ebbb-4755-add1-67c4891f50a3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:35:08.765812Z",
     "iopub.status.busy": "2022-04-27T12:35:08.765666Z",
     "iopub.status.idle": "2022-04-27T12:35:39.840642Z",
     "shell.execute_reply": "2022-04-27T12:35:39.840096Z",
     "shell.execute_reply.started": "2022-04-27T12:35:08.765794Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                55680     \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 28,918,657\n",
      "Trainable params: 28,918,657\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "545e7660-0780-46f5-a359-d9ce5341e894",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Regularisation and No Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "3181104f-4925-48ba-9f29-6ca2c5d2ecc2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:35:39.841683Z",
     "iopub.status.busy": "2022-04-27T12:35:39.841446Z",
     "iopub.status.idle": "2022-04-27T12:36:12.240177Z",
     "shell.execute_reply": "2022-04-27T12:36:12.239567Z",
     "shell.execute_reply.started": "2022-04-27T12:35:39.841663Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                55680     \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 28,918,657\n",
      "Trainable params: 28,918,657\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = False\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fecf929-c429-404e-94a9-13ec5ae7f38f",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Normalisation and No Regularisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "061e265d-fe4d-4de9-ae77-139e906230bc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:36:12.241347Z",
     "iopub.status.busy": "2022-04-27T12:36:12.241114Z",
     "iopub.status.idle": "2022-04-27T12:36:43.294218Z",
     "shell.execute_reply": "2022-04-27T12:36:43.293708Z",
     "shell.execute_reply.started": "2022-04-27T12:36:12.241324Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                55680     \n",
      "_________________________________________________________________\n",
      "Linear_Dense (Dense)         (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 28,919,681\n",
      "Trainable params: 28,919,169\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = False\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1306b99e-331b-4396-8329-440adcad3041",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Both Regularisation and Batch Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "432173b7-65db-4153-ba8f-8ece3fec10be",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:36:43.295253Z",
     "iopub.status.busy": "2022-04-27T12:36:43.295036Z",
     "iopub.status.idle": "2022-04-27T12:37:15.713943Z",
     "shell.execute_reply": "2022-04-27T12:37:15.713347Z",
     "shell.execute_reply.started": "2022-04-27T12:36:43.295232Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"GRU\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_embedding (Embedding)  (None, 500, 256)          28846080  \n",
      "_________________________________________________________________\n",
      "Bidirectional_GRU (Bidirecti (None, 64)                55680     \n",
      "_________________________________________________________________\n",
      "Linear_Dense_Elasti (Dense)  (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "Batch_Norm1 (BatchNormalizat (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "ReLU_Activation (Activation) (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 28,919,681\n",
      "Trainable params: 28,919,169\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Tweak model architecture\n",
    "model_dict[\"regularize\"] = True\n",
    "model_dict[\"batch_normalize\"] = True\n",
    "\n",
    "# Build and compile\n",
    "model = pm.bi_gru(regularize = model_dict[\"regularize\"],\n",
    "               batch_normalize = model_dict[\"batch_normalize\"],\n",
    "               embedding = embed_dict[model_dict[\"embedding\"]],\n",
    "               **compile_hp)\n",
    "model.summary()\n",
    "# Fit model and save it and its history\n",
    "history = pm.fit_and_save(X_train, y_train, model,\n",
    "                          save_model = save_model, \n",
    "                          save_history = save_history,\n",
    "                          **model_dict, **fit_hp)\n",
    "# Evaluate on test data\n",
    "all_results = pm.get_test_metrics(model, X_test, y_test, all_results, history, **model_dict)\n",
    "# Plot and save graphs\n",
    "pf.plot_and_save(history, save = save_plots, display = display, **model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b30d8f8-59e5-41b6-9357-c2d2e3ab9b33",
   "metadata": {},
   "source": [
    "- Save results from GRUs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "163a65bb-9665-434e-90dc-92de63e8de5c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-27T12:37:15.715328Z",
     "iopub.status.busy": "2022-04-27T12:37:15.714931Z",
     "iopub.status.idle": "2022-04-27T12:37:15.721253Z",
     "shell.execute_reply": "2022-04-27T12:37:15.720655Z",
     "shell.execute_reply.started": "2022-04-27T12:37:15.715292Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_results.to_csv(\"all_results_table.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c40697ba-dc56-4689-8b64-469b74744bee",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "saturn (Python 3)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}