DetectingGeneratedTextUsingBERT.ipynb

{
  "metadata": {
    "kernelspec": {
      "language": "python",
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.13",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "kaggle": {
      "accelerator": "nvidiaTeslaT4",
      "dataSources": [
        {
          "sourceId": 61542,
          "databundleVersionId": 7516023,
          "sourceType": "competition"
        }
      ],
      "dockerImageVersionId": 30733,
      "isInternetEnabled": true,
      "language": "python",
      "sourceType": "notebook",
      "isGpuEnabled": true
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat_minor": 0,
  "nbformat": 4,
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/MaiAditya/Detecting-Generated-Text----BERT-Model/blob/main/DetectingGeneratedTextUsingBERT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load\n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the read-only \"../input/\" directory\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
        "\n",
        "import os\n",
        "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
        "    for filename in filenames:\n",
        "        print(os.path.join(dirname, filename))\n",
        "\n",
        "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\"\n",
        "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
      ],
      "metadata": {
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:18.830882Z",
          "iopub.execute_input": "2024-06-06T15:56:18.831224Z",
          "iopub.status.idle": "2024-06-06T15:56:19.207457Z",
          "shell.execute_reply.started": "2024-06-06T15:56:18.831199Z",
          "shell.execute_reply": "2024-06-06T15:56:19.206575Z"
        },
        "trusted": true,
        "id": "ofN9yE8PaDwc",
        "outputId": "030eebce-b0c4-4ea2-8ba0-eb0873fe4477"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "/kaggle/input/llm-detect-ai-generated-text/sample_submission.csv\n/kaggle/input/llm-detect-ai-generated-text/train_prompts.csv\n/kaggle/input/llm-detect-ai-generated-text/test_essays.csv\n/kaggle/input/llm-detect-ai-generated-text/train_essays.csv\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Import necessary libraries\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import re\n",
        "from nltk.corpus import stopwords\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import accuracy_score, classification_report\n",
        "from transformers import BertTokenizer, BertForSequenceClassification, AdamW\n",
        "import torch\n",
        "from torch.utils.data import DataLoader, TensorDataset, random_split"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:19.209126Z",
          "iopub.execute_input": "2024-06-06T15:56:19.209494Z",
          "iopub.status.idle": "2024-06-06T15:56:27.332523Z",
          "shell.execute_reply.started": "2024-06-06T15:56:19.209469Z",
          "shell.execute_reply": "2024-06-06T15:56:27.331548Z"
        },
        "trusted": true,
        "id": "HJsrvztcaDwf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df=pd.read_csv(\"/kaggle/input/llm-detect-ai-generated-text/train_essays.csv\")\n",
        "df.head()"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:27.333703Z",
          "iopub.execute_input": "2024-06-06T15:56:27.334230Z",
          "iopub.status.idle": "2024-06-06T15:56:27.455495Z",
          "shell.execute_reply.started": "2024-06-06T15:56:27.334197Z",
          "shell.execute_reply": "2024-06-06T15:56:27.454350Z"
        },
        "trusted": true,
        "id": "Otflcyu_aDwf",
        "outputId": "f358239e-6b75-4d59-e2d6-a62d2b803952"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 4,
          "output_type": "execute_result",
          "data": {
            "text/plain": "         id  prompt_id                                               text  \\\n0  0059830c          0  Cars. Cars have been around since they became ...   \n1  005db917          0  Transportation is a large necessity in most co...   \n2  008f63e3          0  \"America's love affair with it's vehicles seem...   \n3  00940276          0  How often do you ride in a car? Do you drive a...   \n4  00c39458          0  Cars are a wonderful thing. They are perhaps o...   \n\n   generated  \n0          0  \n1          0  \n2          0  \n3          0  \n4          0  ",
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>prompt_id</th>\n      <th>text</th>\n      <th>generated</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0059830c</td>\n      <td>0</td>\n      <td>Cars. Cars have been around since they became ...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>005db917</td>\n      <td>0</td>\n      <td>Transportation is a large necessity in most co...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>008f63e3</td>\n      <td>0</td>\n      <td>\"America's love affair with it's vehicles seem...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>00940276</td>\n      <td>0</td>\n      <td>How often do you ride in a car? Do you drive a...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>00c39458</td>\n      <td>0</td>\n      <td>Cars are a wonderful thing. They are perhaps o...</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the training and testing datasets\n",
        "train_essays = pd.read_csv(\"/kaggle/input/llm-detect-ai-generated-text/train_essays.csv\")\n",
        "test_essays = pd.read_csv(\"/kaggle/input/llm-detect-ai-generated-text/test_essays.csv\")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:27.459501Z",
          "iopub.execute_input": "2024-06-06T15:56:27.459840Z",
          "iopub.status.idle": "2024-06-06T15:56:27.513982Z",
          "shell.execute_reply.started": "2024-06-06T15:56:27.459790Z",
          "shell.execute_reply": "2024-06-06T15:56:27.512980Z"
        },
        "trusted": true,
        "id": "0yCxb5ZVaDwf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Explore the training data\n",
        "train_essays.info()"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:27.515346Z",
          "iopub.execute_input": "2024-06-06T15:56:27.515725Z",
          "iopub.status.idle": "2024-06-06T15:56:27.543015Z",
          "shell.execute_reply.started": "2024-06-06T15:56:27.515691Z",
          "shell.execute_reply": "2024-06-06T15:56:27.541972Z"
        },
        "trusted": true,
        "id": "o_DjN5gWaDwg",
        "outputId": "9e54ce02-4486-439c-f4e9-67fc402990ef"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 1378 entries, 0 to 1377\nData columns (total 4 columns):\n #   Column     Non-Null Count  Dtype \n---  ------     --------------  ----- \n 0   id         1378 non-null   object\n 1   prompt_id  1378 non-null   int64 \n 2   text       1378 non-null   object\n 3   generated  1378 non-null   int64 \ndtypes: int64(2), object(2)\nmemory usage: 43.2+ KB\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check for class balance\n",
        "sns.countplot(data=train_essays, x='generated')\n",
        "plt.show()"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:27.544353Z",
          "iopub.execute_input": "2024-06-06T15:56:27.544736Z",
          "iopub.status.idle": "2024-06-06T15:56:27.785497Z",
          "shell.execute_reply.started": "2024-06-06T15:56:27.544700Z",
          "shell.execute_reply": "2024-06-06T15:56:27.784597Z"
        },
        "trusted": true,
        "id": "ZlwKJ96waDwg",
        "outputId": "0bb9514f-1774-4a1b-ab2f-622793c91150"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<Figure size 640x480 with 1 Axes>",
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGwCAYAAABIC3rIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAArnklEQVR4nO3df1SUdaLH8c8A8kOUIVRmmMJ0u62py1pB0Wxlphzxx3rzrFvXYpVaru62UBlFxjlJv2NTM8Us1/b6o3tp+3F3s7KzJBcLvEaoGPkjQytLbjZQIUzQCghz/9j1OU26ZQTM0Pf9OmfOcZ7vd57n+3iO8j7PPDPYfD6fTwAAAAYLCfQCAAAAAo0gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxwgK9gP6gq6tLR44c0eDBg2Wz2QK9HAAAcBp8Pp+++OILuVwuhYR88zUggug0HDlyRImJiYFeBgAA6Ia6ujqdddZZ3ziHIDoNgwcPlvT3v9CYmJgArwYAAJwOr9erxMRE6+f4NyGITsOJt8liYmIIIgAA+pnTud2Fm6oBAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPECGkQVFRWaMWOGXC6XbDabNm7c+E/n/va3v5XNZtPy5cv9tjc2NiojI0MxMTGKjY1VVlaWWlpa/Obs3r1bl19+uSIjI5WYmKjFixf3wtkAAID+KqBB1NraqnHjxmnVqlXfOO+FF17Qm2++KZfLddJYRkaG9u3bp9LSUm3atEkVFRWaP3++Ne71ejV58mSdffbZqq6u1pIlS3TPPfdozZo1PX4+AACgfwro9xBNnTpVU6dO/cY5H3/8sW666Sa9+uqrmj59ut/Y/v37VVJSoh07diglJUWStHLlSk2bNk1Lly6Vy+VScXGx2tvbtXbtWoWHh2vs2LGqqanRsmXL/MLpq9ra2tTW1mY993q93/NMAQBAMAvqe4i6uro0Z84c5eXlaezYsSeNV1ZWKjY21oohSUpLS1NISIiqqqqsOePHj1d4eLg1Jz09XbW1tTp69Ogpj1tYWCi73W49+LUdAAD8sAV1ED388MMKCwvTzTfffMpxj8ej+Ph4v21hYWGKi4uTx+Ox5jgcDr85J56fmPN1+fn5am5uth51dXXf91QAAEAQC9pf3VFdXa0VK1Zo165dff4b5iMiIhQREdGnxwQAAIETtFeItm7dqoaGBg0fPlxhYWEKCwvTRx99pNtuu00jRoyQJDmdTjU0NPi97vjx42psbJTT6bTm1NfX+8058fzEHAAAYLagDaI5c+Zo9+7dqqmpsR4ul0t5eXl69dVXJUlut1tNTU2qrq62XrdlyxZ1dXUpNTXVmlNRUaGOjg5rTmlpqUaNGqUzzjijb08KAAAEpYC+ZdbS0qL33nvPen7o0CHV1NQoLi5Ow4cP15AhQ/zmDxgwQE6nU6NGjZIkjR49WlOmTNG8efO0evVqdXR0KCcnR7Nnz7Y+on/dddfp3nvvVVZWlhYuXKi9e/dqxYoVevTRR/vuRAEAQFALaBDt3LlTV155pfU8NzdXkpSZman169ef1j6Ki4uVk5OjSZMmKSQkRLNmzVJRUZE1brfbtXnzZmVnZys5OVlDhw5VQUHBP/3IPQAAMI/N5/P5Ar2IYOf1emW329Xc3KyYmJheO05y3lO9tm+gP6teMjfQSwDQD32Xn99Bew8RAABAXyGIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYL6BBVFFRoRkzZsjlcslms2njxo3WWEdHhxYuXKikpCRFR0fL5XJp7ty5OnLkiN8+GhsblZGRoZiYGMXGxiorK0stLS1+c3bv3q3LL79ckZGRSkxM1OLFi/vi9AAAQD8R0CBqbW3VuHHjtGrVqpPGvvzyS+3atUuLFi3Srl279Je//EW1tbX613/9V795GRkZ2rdvn0pLS7Vp0yZVVFRo/vz51rjX69XkyZN19tlnq7q6WkuWLNE999yjNWvW9Pr5AQCA/iEskAefOnWqpk6desoxu92u0tJSv22PPfaYLr74Yh0+fFjDhw/X/v37VVJSoh07diglJUWStHLlSk2bNk1Lly6Vy+VScXGx2tvbtXbtWoWHh2vs2LGqqanRsmXL/MIJAACYq1/dQ9Tc3CybzabY2FhJUmVlpWJjY60YkqS0tDSFhISoqqrKmjN+/HiFh4dbc9LT01VbW6ujR4+e8jhtbW3yer1+DwAA8MPVb4Lo2LFjWrhwoa699lrFxMRIkjwej+Lj4/3mhYWFKS4uTh6Px5rjcDj85px4fmLO1xUWFsput1uPxMTEnj4dAAAQRPpFEHV0dOiaa66Rz+fTE0880evHy8/PV3Nzs/Woq6vr9WMCAIDACeg9RKfjRAx99NFH2rJli3V1SJKcTqcaGhr85h8/flyNjY1yOp3WnPr6er85J56fmPN1ERERioiI6MnTAAAAQSyorxCdiKGDBw/qf/7nfzRkyBC/cbfbraamJlVXV1vbtmzZoq6uLqWmplpzKioq1NHRYc0pLS3VqFGjdMYZZ/TNiQAAgKAW0CBqaWlRTU2NampqJEmHDh1STU2NDh8+rI6ODv3yl7/Uzp07VVxcrM7OTnk8Hnk8HrW3t0uSRo8erSlTpmjevHnavn27tm3bppycHM2ePVsul0uSdN111yk8PFxZWVnat2+fnn32Wa1YsUK5ubmBOm0AABBkAvqW2c6dO3XllVdaz09ESmZmpu655x699NJLkqTzzz/f73WvvfaaJkyYIEkqLi5WTk6OJk2apJCQEM2aNUtFRUXWXLvdrs2bNys7O1vJyckaOnSoCgoK+Mg9AACwBDSIJkyYIJ/P90/Hv2nshLi4OD399NPfOOenP/2ptm7d+p3XBwAAzBDU9xABAAD0BYIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxAhpEFRUVmjFjhlwul2w2mzZu3Og37vP5VFBQoISEBEVFRSktLU0HDx70m9PY2KiMjAzFxMQoNjZWWVlZamlp8Zuze/duXX755YqMjFRiYqIWL17c26cGAAD6kYAGUWtrq8aNG6dVq1adcnzx4sUqKirS6tWrVVVVpejoaKWnp+vYsWPWnIyMDO3bt0+lpaXatGmTKioqNH/+fGvc6/Vq8uTJOvvss1VdXa0lS5bonnvu0Zo1a3r9/AAAQP8QFsiDT506VVOnTj3lmM/n0/Lly3XXXXfpqquukiQ99dRTcjgc2rhxo2bPnq39+/erpKREO3bsUEpKiiRp5cqVmjZtmpYuXSqXy6Xi4mK1t7dr7dq1Cg8P19ixY1VTU6Nly5b5hRMAADBX0N5DdOjQIXk8HqWlpVnb7Ha7UlNTVVlZKUmqrKxUbGysFUOSlJaWppCQEFVVVVlzxo8fr/DwcGtOenq6amtrdfTo0VMeu62tTV6v1+8BAAB+uII2iDwejyTJ4XD4bXc4HNaYx+NRfHy833hYWJji4uL85pxqH189xtcVFhbKbrdbj8TExO9/QgAAIGgFbRAFUn5+vpqbm61HXV1doJcEAAB6UdAGkdPplCTV19f7ba+vr7fGnE6nGhoa/MaPHz+uxsZGvzmn2sdXj/F1ERERiomJ8XsAAIAfrqANopEjR8rpdKqsrMza5vV6VVVVJbfbLUlyu91qampSdXW1NWfLli3q6upSamqqNaeiokIdHR3WnNLSUo0aNUpnnHFGH50NAAAIZgENopaWFtXU1KimpkbS32+krqmp0eHDh2Wz2bRgwQI98MADeumll7Rnzx7NnTtXLpdLM2fOlCSNHj1aU6ZM0bx587R9+3Zt27ZNOTk5mj17tlwulyTpuuuuU3h4uLKysrRv3z49++yzWrFihXJzcwN01gAAINgE9GP3O3fu1JVXXmk9PxEpmZmZWr9+ve644w61trZq/vz5ampq0mWXXaaSkhJFRkZarykuLlZOTo4mTZqkkJAQzZo1S0VFRda43W7X5s2blZ2dreTkZA0dOlQFBQV85B4AAFhsPp/PF+hFBDuv1yu73a7m5uZevZ8oOe+pXts30J9VL5kb6CUA6Ie+y8/voL2HCAAAoK8QRAAAwHgEEQAAMB5BBAAAjEcQAQAA4xFEAADAeAQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMB5BBAAAjEcQAQAA4xFEAADAeAQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMB5BBAAAjEcQAQAA4xFEAADAeAQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMB5BBAAAjBfUQdTZ2alFixZp5MiRioqK0jnnnKP7779fPp/PmuPz+VRQUKCEhARFRUUpLS1NBw8e9NtPY2OjMjIyFBMTo9jYWGVlZamlpaWvTwcAAASpoA6ihx9+WE888YQee+wx7d+/Xw8//LAWL16slStXWnMWL16soqIirV69WlVVVYqOjlZ6erqOHTtmzcnIyNC+fftUWlqqTZs2qaKiQvPnzw/EKQEAgCAUFugFfJM33nhDV111laZPny5JGjFihP70pz9p+/btkv5+dWj58uW66667dNVVV0mSnnrqKTkcDm3cuFGzZ8/W/v37VVJSoh07diglJUWStHLlSk2bNk1Lly6Vy+UKzMkBAICgEdRXiH72s5+prKxMBw4ckCS9/fbb+t///V9NnTpVknTo0CF5PB6lpaVZr7Hb7UpNTVVlZaUkqbKyUrGxsVYMSVJaWppCQkJUVVV1yuO2tbXJ6/X6PQAAwA9XUF8huvPOO+X1enXeeecpNDRUnZ2devDBB5WRkSFJ8ng8kiSHw+H3OofDYY15PB7Fx8f7jYeFhSkuLs6a83WFhYW69957e/p0AABAkArqK0TPPfeciouL9fTTT2vXrl3asGGDli5dqg0bNvTqcfPz89Xc3Gw96urqevV4AAAgsIL6ClFeXp7uvPNOzZ49W5KUlJSkjz76SIWFhcrMzJTT6ZQk1dfXKyEhwXpdfX29zj//fEmS0+lUQ0OD336PHz+uxsZG6/VfFxERoYiIiF44IwAAEIy6dYVo4sSJampqOmm71+vVxIkTv++aLF9++aVCQvyXGBoaqq6uLknSyJEj5XQ6VVZW5reGqqoqud1uSZLb7VZTU5Oqq6utOVu2bFFXV5dSU1N7bK0AAKD/6tYVotdff13t7e0nbT927Ji2bt36vRd1wowZM/Tggw9q+PDhGjt2rN566y0tW7ZMv/71ryVJNptNCxYs0AMPPKBzzz1XI0eO1KJFi+RyuTRz5kxJ0ujRozVlyhTNmzdPq1evVkdHh3JycjR79mw+YQYAACR9xyDavXu39ed33nnH76bkzs5OlZSU6Mwzz+yxxa1cuVKLFi3S7373OzU0NMjlcuk3v/mNCgoKrDl33HGHWltbNX/+fDU1Nemyyy5TSUmJIiMjrTnFxcXKycnRpEmTFBISolmzZqmoqKjH1gkAAPo3m++rX/v8LUJCQmSz2SRJp3pZVFSUVq5caV3B+aHwer2y2+1qbm5WTExMrx0nOe+pXts30J9VL5kb6CUA6Ie+y8/v73SF6NChQ/L5fPrRj36k7du3a9iwYdZYeHi44uPjFRoa2r1VAwAABMh3CqKzzz5bkqybmgEAAH4Iuv2x+4MHD+q1115TQ0PDSYH01Xt8AAAAgl23gujJJ5/UjTfeqKFDh8rpdFr3FUl//+QXQQQAAPqTbgXRAw88oAcffFALFy7s6fUAAAD0uW59MePRo0d19dVX9/RaAAAAAqJbQXT11Vdr8+bNPb0WAACAgOjWW2b/8i//okWLFunNN99UUlKSBgwY4Dd+880398jiAAAA+kK3gmjNmjUaNGiQysvLVV5e7jdms9kIIgAA0K90K4gOHTrU0+sAAAAImG7dQwQAAPBD0q0rRN/2u8rWrl3brcUAAAAEQreC6OjRo37POzo6tHfvXjU1NWnixIk9sjAAAIC+0q0geuGFF07a1tXVpRtvvFHnnHPO914UAABAX+qxe4hCQkKUm5urRx99tKd2CQAA0Cd69Kbq999/X8ePH+/JXQIAAPS6br1llpub6/fc5/Ppk08+0SuvvKLMzMweWRgAAEBf6VYQvfXWW37PQ0JCNGzYMD3yyCPf+gk0AACAYNOtIHrttdd6eh0AAAAB060gOuHTTz9VbW2tJGnUqFEaNmxYjywKAACgL3XrpurW1lb9+te/VkJCgsaPH6/x48fL5XIpKytLX375ZU+vEQAAoFd1K4hyc3NVXl6ul19+WU1NTWpqatKLL76o8vJy3XbbbT29RgAAgF7VrbfM/vznP+u///u/NWHCBGvbtGnTFBUVpWuuuUZPPPFET60PAACg13XrCtGXX34ph8Nx0vb4+HjeMgMAAP1Ot4LI7Xbr7rvv1rFjx6xtf/vb33TvvffK7Xb32OIAAAD6QrfeMlu+fLmmTJmis846S+PGjZMkvf3224qIiNDmzZt7dIEAAAC9rVtBlJSUpIMHD6q4uFjvvvuuJOnaa69VRkaGoqKienSBAAAAva1bQVRYWCiHw6F58+b5bV+7dq0+/fRTLVy4sEcWBwAA0Be6dQ/RH/7wB5133nknbR87dqxWr179vRcFAADQl7oVRB6PRwkJCSdtHzZsmD755JPvvSgAAIC+1K0gSkxM1LZt207avm3bNrlcru+9KAAAgL7UrXuI5s2bpwULFqijo0MTJ06UJJWVlemOO+7gm6oBAEC/060gysvL0+eff67f/e53am9vlyRFRkZq4cKFys/P79EFAgAA9LZuBZHNZtPDDz+sRYsWaf/+/YqKitK5556riIiInl4fAABAr+tWEJ0waNAgXXTRRT21FgAAgIDo1k3VAAAAPyQEEQAAMB5BBAAAjEcQAQAA4xFEAADAeAQRAAAwXtAH0ccff6xf/epXGjJkiKKiopSUlKSdO3da4z6fTwUFBUpISFBUVJTS0tJ08OBBv300NjYqIyNDMTExio2NVVZWllpaWvr6VAAAQJAK6iA6evSoLr30Ug0YMEB//etf9c477+iRRx7RGWecYc1ZvHixioqKtHr1alVVVSk6Olrp6ek6duyYNScjI0P79u1TaWmpNm3apIqKCs2fPz8QpwQAAIKQzefz+QK9iH/mzjvv1LZt27R169ZTjvt8PrlcLt122226/fbbJUnNzc1yOBxav369Zs+erf3792vMmDHasWOHUlJSJEklJSWaNm2a/u///u+Uv4y2ra1NbW1t1nOv16vExEQ1NzcrJiamF87075Lznuq1fQP9WfWSuYFeAoB+yOv1ym63n9bP76C+QvTSSy8pJSVFV199teLj43XBBRfoySeftMYPHTokj8ejtLQ0a5vdbldqaqoqKyslSZWVlYqNjbViSJLS0tIUEhKiqqqqUx63sLBQdrvdeiQmJvbSGQIAgGAQ1EH0wQcf6IknntC5556rV199VTfeeKNuvvlmbdiwQZLk8XgkSQ6Hw+91DofDGvN4PIqPj/cbDwsLU1xcnDXn6/Lz89Xc3Gw96urqevrUAABAEPlev8ust3V1dSklJUUPPfSQJOmCCy7Q3r17tXr1amVmZvbacSMiIvhFtQAAGCSorxAlJCRozJgxfttGjx6tw4cPS5KcTqckqb6+3m9OfX29NeZ0OtXQ0OA3fvz4cTU2NlpzAACA2YI6iC699FLV1tb6bTtw4IDOPvtsSdLIkSPldDpVVlZmjXu9XlVVVcntdkuS3G63mpqaVF1dbc3ZsmWLurq6lJqa2gdnAQAAgl1Qv2V266236mc/+5keeughXXPNNdq+fbvWrFmjNWvWSJJsNpsWLFigBx54QOeee65GjhypRYsWyeVyaebMmZL+fkVpypQpmjdvnlavXq2Ojg7l5ORo9uzZp/yEGQAAME9QB9FFF12kF154Qfn5+brvvvs0cuRILV++XBkZGdacO+64Q62trZo/f76ampp02WWXqaSkRJGRkdac4uJi5eTkaNKkSQoJCdGsWbNUVFQUiFMCAABBKKi/hyhYfJfvMfg++B4i4NT4HiIA3fGD+R4iAACAvkAQAQAA4xFEAADAeAQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMB5BBAAAjEcQAQAA4xFEAADAeAQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMB5BBAAAjEcQAQAA4xFEAADAeAQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMB5BBAAAjEcQAQAA4xFEAADAeAQRAAAwXr8Kot///vey2WxasGCBte3YsWPKzs7WkCFDNGjQIM2aNUv19fV+rzt8+LCmT5+ugQMHKj4+Xnl5eTp+/Hgfrx4AAASrfhNEO3bs0B/+8Af99Kc/9dt+66236uWXX9bzzz+v8vJyHTlyRL/4xS+s8c7OTk2fPl3t7e164403tGHDBq1fv14FBQV9fQoAACBI9YsgamlpUUZGhp588kmdccYZ1vbm5mb9x3/8h5YtW6aJEycqOTlZ69at0xtvvKE333xTkrR582a98847+q//+i+df/75mjp1qu6//36tWrVK7e3tpzxeW1ubvF6v3wMAAPxw9Ysgys7O1vTp05WWlua3vbq6Wh0dHX7bzzvvPA0fPlyVlZWSpMrKSiUlJcnhcFhz0tPT5fV6tW/fvlMer7CwUHa73XokJib2wlkBAIBgEfRB9Mwzz2jXrl0qLCw8aczj8Sg8PFyxsbF+2x0OhzwejzXnqzF0YvzE2Knk5+erubnZetTV1fXAmQAAgGAVFugFfJO6ujrdcsstKi0tVWRkZJ8dNyIiQhEREX12PAAAEFhBfYWourpaDQ0NuvDCCxUWFqawsDCVl5erqKhIYWFhcjgcam9vV1NTk9/r6uvr5XQ6JUlOp/OkT52deH5iDgAAMFtQB9GkSZO0Z88e1dTUWI+UlBRlZGRYfx4wYIDKysqs19TW1urw4cNyu92SJLfbrT179qihocGaU1paqpiYGI0ZM6bPzwkAAASfoH7LbPDgwfrJT37ity06OlpDhgyxtmdlZSk3N1dxcXGKiYnRTTfdJLfbrUsuuUSSNHnyZI0ZM0Zz5szR4sWL5fF4dNdddyk7O5u3xQAAgKQgD6LT8eijjyokJESzZs1SW1ub0tPT9fjjj1vjoaGh2rRpk2688Ua53W5FR0crMzNT9913XwBXDQAAgonN5/P5Ar2IYOf1emW329Xc3KyYmJheO05y3lO9tm+gP6teMjfQSwDQD32Xn99BfQ8RAABAXyCIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYL6iDqLCwUBdddJEGDx6s+Ph4zZw5U7W1tX5zjh07puzsbA0ZMkSDBg3SrFmzVF9f7zfn8OHDmj59ugYOHKj4+Hjl5eXp+PHjfXkqAAAgiAV1EJWXlys7O1tvvvmmSktL1dHRocmTJ6u1tdWac+utt+rll1/W888/r/Lych05ckS/+MUvrPHOzk5Nnz5d7e3teuONN7RhwwatX79eBQUFgTglAAAQhGw+n88X6EWcrk8//VTx8fEqLy/X+PHj1dzcrGHDhunpp5/WL3/5S0nSu+++q9GjR6uyslKXXHKJ/vrXv+rnP/+5jhw5IofDIUlavXq1Fi5cqE8//VTh4eHfelyv1yu73a7m5mbFxMT02vkl5z3Va/sG+rPqJXMDvQQA/dB3+fkd1FeIvq65uVmSFBcXJ0mqrq5WR0eH0tLSrDnnnXeehg8frsrKSklSZWWlkpKSrBiSpPT0dHm9Xu3bt++Ux2lra5PX6/V7AACAH65+E0RdXV1asGCBLr30Uv3kJz+RJHk8HoWHhys2NtZvrsPhkMfjseZ8NYZOjJ8YO5XCwkLZ7XbrkZiY2MNnAwAAgkm/CaLs7Gzt3btXzzzzTK8fKz8/X83Nzdajrq6u148JAAACJyzQCzgdOTk52rRpkyoqKnTWWWdZ251Op9rb29XU1OR3lai+vl5Op9Oas337dr/9nfgU2ok5XxcREaGIiIgePgsAABCsgvoKkc/nU05Ojl544QVt2bJFI0eO9BtPTk7WgAEDVFZWZm2rra3V4cOH5Xa7JUlut1t79uxRQ0ODNae0tFQxMTEaM2ZM35wIAAAIakF9hSg7O1tPP/20XnzxRQ0ePNi658dutysqKkp2u11ZWVnKzc1VXFycYmJidNNNN8ntduuSSy6RJE2ePFljxozRnDlztHjxYnk8Ht11113Kzs7mKhAAAJAU5EH0xBNPSJImTJjgt33dunW6/vrrJUmPPvqoQkJCNGvWLLW1tSk9PV2PP/64NTc0NFSbNm3SjTfeKLfbrejoaGVmZuq+++7rq9MAAABBrl99D1Gg8D1EQGDxPUQAuuMH+z1EAAAAvYEgAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPKOCaNWqVRoxYoQiIyOVmpqq7du3B3pJAAAgCBgTRM8++6xyc3N19913a9euXRo3bpzS09PV0NAQ6KUBAIAAMyaIli1bpnnz5umGG27QmDFjtHr1ag0cOFBr164N9NIAAECAhQV6AX2hvb1d1dXVys/Pt7aFhIQoLS1NlZWVJ81va2tTW1ub9by5uVmS5PV6e3WdnW1/69X9A/1Vb//b6wt1v78k0EsAglLinW/22r5P/N/h8/m+da4RQfTZZ5+ps7NTDofDb7vD4dC777570vzCwkLde++9J21PTEzstTUC+OfsK38b6CUA6C2F9l4/xBdffCG7/ZuPY0QQfVf5+fnKzc21nnd1damxsVFDhgyRzWYL4MrQF7xerxITE1VXV6eYmJhALwdAD+Lft1l8Pp+++OILuVyub51rRBANHTpUoaGhqq+v99teX18vp9N50vyIiAhFRET4bYuNje3NJSIIxcTE8B8m8APFv29zfNuVoROMuKk6PDxcycnJKisrs7Z1dXWprKxMbrc7gCsDAADBwIgrRJKUm5urzMxMpaSk6OKLL9by5cvV2tqqG264IdBLAwAAAWZMEP3bv/2bPv30UxUUFMjj8ej8889XSUnJSTdaAxEREbr77rtPetsUQP/Hv2/8Mzbf6XwWDQAA4AfMiHuIAAAAvglBBAAAjEcQAQAA4xFEAADAeAQR8DWrVq3SiBEjFBkZqdTUVG3fvj3QSwLQAyoqKjRjxgy5XC7ZbDZt3Lgx0EtCECGIgK949tlnlZubq7vvvlu7du3SuHHjlJ6eroaGhkAvDcD31NraqnHjxmnVqlWBXgqCEB+7B74iNTVVF110kR577DFJf/9G88TERN1000268847A7w6AD3FZrPphRde0MyZMwO9FAQJrhAB/9De3q7q6mqlpaVZ20JCQpSWlqbKysoArgwA0NsIIuAfPvvsM3V2dp707eUOh0MejydAqwIA9AWCCAAAGI8gAv5h6NChCg0NVX19vd/2+vp6OZ3OAK0KANAXCCLgH8LDw5WcnKyysjJrW1dXl8rKyuR2uwO4MgBAbzPmt90DpyM3N1eZmZlKSUnRxRdfrOXLl6u1tVU33HBDoJcG4HtqaWnRe++9Zz0/dOiQampqFBcXp+HDhwdwZQgGfOwe+JrHHntMS5Yskcfj0fnnn6+ioiKlpqYGelkAvqfXX39dV1555UnbMzMztX79+r5fEIIKQQQAAIzHPUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMB5BBAAAjEcQAQAA4xFEABAEJkyYoAULFgR6GYCxCCIA6CYiBvjhIIgA4Gs6OjoCvQQAfYwgAhAwX3zxhTIyMhQdHa2EhAQ9+uijfldd2tradPvtt+vMM89UdHS0UlNT9frrr1uvX79+vWJjY/Xqq69q9OjRGjRokKZMmaJPPvnE7zh//OMfNXr0aEVGRuq8887T448/bo19+OGHstlsevbZZ3XFFVcoMjJSxcXF+vzzz3XttdfqzDPP1MCBA5WUlKQ//elP1uuuv/56lZeXa8WKFbLZbLLZbPrwww8lSXv37tXUqVM1aNAgORwOzZkzR5999pn12tbWVs2dO1eDBg1SQkKCHnnkkZ7/ywXwnRBEAAImNzdX27Zt00svvaTS0lJt3bpVu3btssZzcnJUWVmpZ555Rrt379bVV1+tKVOm6ODBg9acL7/8UkuXLtV//ud/qqKiQocPH9btt99ujRcXF6ugoEAPPvig9u/fr4ceekiLFi3Shg0b/NZy55136pZbbtH+/fuVnp6uY8eOKTk5Wa+88or27t2r+fPna86cOdq+fbskacWKFXK73Zo3b54++eQTffLJJ0pMTFRTU5MmTpyoCy64QDt37lRJSYnq6+t1zTXXWMfKy8tTeXm5XnzxRW3evFmvv/6633kDCAAfAASA1+v1DRgwwPf8889b25qamnwDBw703XLLLb6PPvrIFxoa6vv444/9Xjdp0iRffn6+z+fz+datW+eT5Hvvvfes8VWrVvkcDof1/JxzzvE9/fTTfvu4//77fW632+fz+XyHDh3ySfItX778W9c8ffp032233WY9v+KKK3y33HLLSfuePHmy37a6ujqfJF9tba3viy++8IWHh/uee+45a/zzzz/3RUVFnbQvAH0nLMA9BsBQH3zwgTo6OnTxxRdb2+x2u0aNGiVJ2rNnjzo7O/XjH//Y73VtbW0aMmSI9XzgwIE655xzrOcJCQlqaGiQ9Pe3pt5//31lZWVp3rx51pzjx4/Lbrf77TclJcXveWdnpx566CE999xz+vjjj9Xe3q62tjYNHDjwG8/r7bff1muvvaZBgwadNPb+++/rb3/7m9rb25Wammptj4uLs84bQGAQRACCUktLi0JDQ1VdXa3Q0FC/sa/GxoABA/zGbDabfD6ftQ9JevLJJ/0CRNJJ+4yOjvZ7vmTJEq1YsULLly9XUlKSoqOjtWDBArW3t3/rumfMmKGHH374pLGEhAS999573/h6AIFBEAEIiB/96EcaMGCAduzYoeHDh0uSmpubdeDAAY0fP14XXHCBOjs71dDQoMsvv7xbx3A4HHK5XPrggw+UkZHxnV67bds2XXXVVfrVr34lSerq6tKBAwc0ZswYa054eLg6Ozv9XnfhhRfqz3/+s0aMGKGwsJP/iz3nnHM0YMAAVVVVWed99OhRHThwQFdcccV3PUUAPYSbqgEExODBg5WZmam8vDy99tpr2rdvn7KyshQSEiKbzaYf//jHysjI0Ny5c/WXv/xFhw4d0vbt21VYWKhXXnnltI9z7733qrCwUEVFRTpw4ID27NmjdevWadmyZd/4unPPPVelpaV64403tH//fv3mN79RfX2935wRI0aoqqpKH374oT777DN1dXUpOztbjY2Nuvbaa7Vjxw69//77evXVV3XDDTeos7NTgwYNUlZWlvLy8rRlyxbt3btX119/vUJC+O8YCCT+BQIImGXLlsntduvnP/+50tLSdOmll1ofj5ekdevWae7cubrttts0atQozZw50++K0un493//d/3xj3/UunXrlJSUpCuuuELr16/XyJEjv/F1d911ly688EKlp6drwoQJcjqdmjlzpt+c22+/XaGhoRozZoyGDRumw4cPy+Vyadu2bers7NTkyZOVlJSkBQsWKDY21oqeJUuW6PLLL9eMGTOUlpamyy67TMnJyd/tLw9Aj7L5TrzZDgAB1traqjPPPFOPPPKIsrKyAr0cAAbhHiIAAfPWW2/p3Xff1cUXX6zm5mbdd999kqSrrroqwCsDYBqCCEBALV26VLW1tQoPD1dycrK2bt2qoUOHBnpZAAzDW2YAAMB43FQNAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMN7/AwxoIxDXz0EUAAAAAElFTkSuQmCC"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_essays.info()"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:27.786628Z",
          "iopub.execute_input": "2024-06-06T15:56:27.786942Z",
          "iopub.status.idle": "2024-06-06T15:56:27.797106Z",
          "shell.execute_reply.started": "2024-06-06T15:56:27.786916Z",
          "shell.execute_reply": "2024-06-06T15:56:27.795946Z"
        },
        "trusted": true,
        "id": "E9GAdIyvaDwg",
        "outputId": "c73b4285-3fb0-4abf-efba-495d60d98873"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 3 entries, 0 to 2\nData columns (total 3 columns):\n #   Column     Non-Null Count  Dtype \n---  ------     --------------  ----- \n 0   id         3 non-null      object\n 1   prompt_id  3 non-null      int64 \n 2   text       3 non-null      object\ndtypes: int64(1), object(2)\nmemory usage: 200.0+ bytes\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "# Make sure to download the stopwords if you haven't already\n",
        "# nltk.download('stopwords')\n",
        "\n",
        "# Set of English stopwords\n",
        "stop_words = set(stopwords.words('english'))\n",
        "\n",
        "def clean_text(text):\n",
        "    \"\"\"\n",
        "    Function to clean text by removing punctuation, lowercasing,\n",
        "    removing non-alphabetic words, and removing stop words.\n",
        "\n",
        "    Parameters:\n",
        "    text (str): The input text to be cleaned.\n",
        "\n",
        "    Returns:\n",
        "    str: The cleaned text.\n",
        "    \"\"\"\n",
        "    # Remove punctuation using regex\n",
        "    text = re.sub(r'[^\\w\\s]', '', text)\n",
        "\n",
        "    # Tokenize the text by splitting into words\n",
        "    words = text.split()\n",
        "\n",
        "    # Convert words to lowercase and remove non-alphabetic words\n",
        "    words = [word.lower() for word in words if word.isalpha()]\n",
        "\n",
        "    # Remove stop words\n",
        "    words = [word for word in words if word not in stop_words]\n",
        "\n",
        "    # Join the words back into a single string and return\n",
        "    return ' '.join(words)\n",
        "\n",
        "# Apply the clean_text function to the 'text' column of the DataFrame\n",
        "train_essays['clean_text'] = train_essays['text'].apply(clean_text)\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:27.798275Z",
          "iopub.execute_input": "2024-06-06T15:56:27.798549Z",
          "iopub.status.idle": "2024-06-06T15:56:28.282009Z",
          "shell.execute_reply.started": "2024-06-06T15:56:27.798525Z",
          "shell.execute_reply": "2024-06-06T15:56:28.281162Z"
        },
        "trusted": true,
        "id": "xf1v7-bWaDwg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Split the data into training and validation sets\n",
        "X_train, X_val, y_train, y_val = train_test_split(train_essays['clean_text'], train_essays['generated'], test_size=0.2, random_state=42)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:28.283538Z",
          "iopub.execute_input": "2024-06-06T15:56:28.284332Z",
          "iopub.status.idle": "2024-06-06T15:56:28.293213Z",
          "shell.execute_reply.started": "2024-06-06T15:56:28.284299Z",
          "shell.execute_reply": "2024-06-06T15:56:28.292302Z"
        },
        "trusted": true,
        "id": "LhyvbmQXaDwg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Tokenization and Encoding for BERT\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True, padding=True, truncation=True, max_length=128)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:28.297122Z",
          "iopub.execute_input": "2024-06-06T15:56:28.297393Z",
          "iopub.status.idle": "2024-06-06T15:56:29.389871Z",
          "shell.execute_reply.started": "2024-06-06T15:56:28.297369Z",
          "shell.execute_reply": "2024-06-06T15:56:29.388956Z"
        },
        "trusted": true,
        "id": "k6ri33aVaDwg",
        "outputId": "b7d5341f-a924-4d11-adeb-0d88eb3937ca",
        "colab": {
          "referenced_widgets": [
            "5aa1357eb18d49c49ada2a597cfb8620",
            "41cd4b6831bc47d1b4d53ff6508b1b63",
            "2410c86235c84d1d84c4523f28d34f15",
            "2aed9eefc08d4b46999952b445fa73cb"
          ]
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "tokenizer_config.json:   0%|          | 0.00/48.0 [00:00<?, ?B/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5aa1357eb18d49c49ada2a597cfb8620"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "41cd4b6831bc47d1b4d53ff6508b1b63"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2410c86235c84d1d84c4523f28d34f15"
            }
          },
          "metadata": {}
        },
        {
          "name": "stderr",
          "text": "/opt/conda/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n  warnings.warn(\n",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2aed9eefc08d4b46999952b445fa73cb"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "encoded_train = tokenizer(X_train.tolist(), padding=True, truncation=True, return_tensors='pt')\n",
        "encoded_val = tokenizer(X_val.tolist(), padding=True, truncation=True, return_tensors='pt')"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:29.391226Z",
          "iopub.execute_input": "2024-06-06T15:56:29.391582Z",
          "iopub.status.idle": "2024-06-06T15:56:48.023248Z",
          "shell.execute_reply.started": "2024-06-06T15:56:29.391550Z",
          "shell.execute_reply": "2024-06-06T15:56:48.022386Z"
        },
        "trusted": true,
        "id": "xvkYbkwJaDwh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert labels to tensors\n",
        "train_labels = torch.tensor(y_train.values)\n",
        "val_labels = torch.tensor(y_val.values)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:48.024459Z",
          "iopub.execute_input": "2024-06-06T15:56:48.024770Z",
          "iopub.status.idle": "2024-06-06T15:56:48.050059Z",
          "shell.execute_reply.started": "2024-06-06T15:56:48.024742Z",
          "shell.execute_reply": "2024-06-06T15:56:48.049342Z"
        },
        "trusted": true,
        "id": "EdNMjcVWaDwh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create TensorDatasets\n",
        "train_dataset = TensorDataset(encoded_train['input_ids'], encoded_train['attention_mask'], train_labels)\n",
        "val_dataset = TensorDataset(encoded_val['input_ids'], encoded_val['attention_mask'], val_labels)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:48.051027Z",
          "iopub.execute_input": "2024-06-06T15:56:48.051296Z",
          "iopub.status.idle": "2024-06-06T15:56:48.057287Z",
          "shell.execute_reply.started": "2024-06-06T15:56:48.051274Z",
          "shell.execute_reply": "2024-06-06T15:56:48.056289Z"
        },
        "trusted": true,
        "id": "y6jMFLcsaDwh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# DataLoader for efficient processing\n",
        "train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)\n",
        "val_loader = DataLoader(val_dataset, batch_size=16, shuffle=False)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:48.058308Z",
          "iopub.execute_input": "2024-06-06T15:56:48.058570Z",
          "iopub.status.idle": "2024-06-06T15:56:48.065884Z",
          "shell.execute_reply.started": "2024-06-06T15:56:48.058547Z",
          "shell.execute_reply": "2024-06-06T15:56:48.064982Z"
        },
        "trusted": true,
        "id": "mMG0-IWcaDwh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the BERT model for sequence classification\n",
        "model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "model.to(device)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:48.067217Z",
          "iopub.execute_input": "2024-06-06T15:56:48.068072Z",
          "iopub.status.idle": "2024-06-06T15:56:51.430903Z",
          "shell.execute_reply.started": "2024-06-06T15:56:48.068046Z",
          "shell.execute_reply": "2024-06-06T15:56:51.430043Z"
        },
        "trusted": true,
        "id": "9E7OVU8OaDwh",
        "outputId": "b989c981-6401-49a3-9c65-f28cdefd6b84",
        "colab": {
          "referenced_widgets": [
            "1f00a4aae2004bc5ab96ef889f983991"
          ]
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "model.safetensors:   0%|          | 0.00/440M [00:00<?, ?B/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1f00a4aae2004bc5ab96ef889f983991"
            }
          },
          "metadata": {}
        },
        {
          "name": "stderr",
          "text": "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
          "output_type": "stream"
        },
        {
          "execution_count": 16,
          "output_type": "execute_result",
          "data": {
            "text/plain": "BertForSequenceClassification(\n  (bert): BertModel(\n    (embeddings): BertEmbeddings(\n      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n      (position_embeddings): Embedding(512, 768)\n      (token_type_embeddings): Embedding(2, 768)\n      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n      (dropout): Dropout(p=0.1, inplace=False)\n    )\n    (encoder): BertEncoder(\n      (layer): ModuleList(\n        (0-11): 12 x BertLayer(\n          (attention): BertAttention(\n            (self): BertSdpaSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (pooler): BertPooler(\n      (dense): Linear(in_features=768, out_features=768, bias=True)\n      (activation): Tanh()\n    )\n  )\n  (dropout): Dropout(p=0.1, inplace=False)\n  (classifier): Linear(in_features=768, out_features=2, bias=True)\n)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define optimizer and learning rate scheduler\n",
        "optimizer = AdamW(model.parameters(), lr=1e-5, correct_bias=False)\n",
        "epochs = 15"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:51.432168Z",
          "iopub.execute_input": "2024-06-06T15:56:51.432522Z",
          "iopub.status.idle": "2024-06-06T15:56:51.444126Z",
          "shell.execute_reply.started": "2024-06-06T15:56:51.432490Z",
          "shell.execute_reply": "2024-06-06T15:56:51.443182Z"
        },
        "trusted": true,
        "id": "PGqXyMXRaDwh",
        "outputId": "d841cff0-19e9-4ab9-bb58-8182a611e55c"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "text": "/opt/conda/lib/python3.10/site-packages/transformers/optimization.py:588: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n  warnings.warn(\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    model.train()\n",
        "    total_loss = 0\n",
        "\n",
        "    # Initialize tqdm progress bar\n",
        "    progress_bar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{epochs}\")\n",
        "\n",
        "    for batch in progress_bar:\n",
        "        input_ids, attention_mask, labels = batch\n",
        "        input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
        "        loss = outputs.loss\n",
        "        total_loss += loss.item()\n",
        "\n",
        "        loss.backward()\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)  # Gradient clipping to avoid exploding gradients\n",
        "        optimizer.step()\n",
        "\n",
        "        # Update tqdm progress bar with the current loss\n",
        "        progress_bar.set_postfix(loss=loss.item())\n",
        "\n",
        "    avg_train_loss = total_loss / len(train_loader)\n",
        "    print(f\"Average training loss for epoch {epoch+1}: {avg_train_loss:.4f}\")\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T15:56:51.445336Z",
          "iopub.execute_input": "2024-06-06T15:56:51.445594Z",
          "iopub.status.idle": "2024-06-06T16:22:03.521540Z",
          "shell.execute_reply.started": "2024-06-06T15:56:51.445571Z",
          "shell.execute_reply": "2024-06-06T16:22:03.520560Z"
        },
        "trusted": true,
        "id": "4GolxOz2aDwh",
        "outputId": "caffa3fb-a0c2-4ba5-9103-2f8190261716"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "text": "Epoch 1/15: 100%|██████████| 69/69 [01:37<00:00,  1.42s/it, loss=0.000317]\n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 1: 0.0548\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "Epoch 2/15: 100%|██████████| 69/69 [01:41<00:00,  1.47s/it, loss=0.000528]\n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 2: 0.0149\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "Epoch 3/15: 100%|██████████| 69/69 [01:41<00:00,  1.46s/it, loss=0.00048] \n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 3: 0.0109\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "Epoch 4/15: 100%|██████████| 69/69 [01:41<00:00,  1.47s/it, loss=0.000558]\n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 4: 0.0112\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "Epoch 5/15: 100%|██████████| 69/69 [01:40<00:00,  1.46s/it, loss=0.000145]\n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 5: 0.0049\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "Epoch 6/15: 100%|██████████| 69/69 [01:41<00:00,  1.46s/it, loss=0.000116]\n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 6: 0.0013\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "Epoch 7/15: 100%|██████████| 69/69 [01:40<00:00,  1.46s/it, loss=8.07e-5] \n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 7: 0.0002\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "Epoch 8/15: 100%|██████████| 69/69 [01:40<00:00,  1.46s/it, loss=6.7e-5] \n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 8: 0.0001\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "Epoch 9/15: 100%|██████████| 69/69 [01:40<00:00,  1.46s/it, loss=5.63e-5] \n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 9: 0.0001\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "Epoch 10/15: 100%|██████████| 69/69 [01:41<00:00,  1.46s/it, loss=5.47e-5] \n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 10: 0.0001\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "Epoch 11/15: 100%|██████████| 69/69 [01:41<00:00,  1.46s/it, loss=5.08e-5] \n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 11: 0.0001\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "Epoch 12/15: 100%|██████████| 69/69 [01:40<00:00,  1.46s/it, loss=3.88e-5] \n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 12: 0.0000\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "Epoch 13/15: 100%|██████████| 69/69 [01:40<00:00,  1.46s/it, loss=3.51e-5] \n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 13: 0.0000\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "Epoch 14/15: 100%|██████████| 69/69 [01:40<00:00,  1.46s/it, loss=3.89e-5] \n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 14: 0.0000\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "Epoch 15/15: 100%|██████████| 69/69 [01:40<00:00,  1.46s/it, loss=3.19e-5] ",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "Average training loss for epoch 15: 0.0000\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Validation loop\n",
        "model.eval()\n",
        "val_preds = []\n",
        "val_labels = []\n",
        "\n",
        "with torch.no_grad():\n",
        "    for batch in val_loader:\n",
        "        input_ids, attention_mask, labels = batch\n",
        "        input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)\n",
        "\n",
        "        outputs = model(input_ids, attention_mask=attention_mask)\n",
        "        logits = outputs.logits\n",
        "\n",
        "        val_preds.extend(torch.argmax(logits, dim=1).cpu().numpy())\n",
        "        val_labels.extend(labels.cpu().numpy())"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T16:22:03.524026Z",
          "iopub.execute_input": "2024-06-06T16:22:03.524327Z",
          "iopub.status.idle": "2024-06-06T16:22:11.642385Z",
          "shell.execute_reply.started": "2024-06-06T16:22:03.524301Z",
          "shell.execute_reply": "2024-06-06T16:22:11.641590Z"
        },
        "trusted": true,
        "id": "fg_WR1dQaDwh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate validation accuracy\n",
        "val_accuracy = accuracy_score(val_labels, val_preds)\n",
        "print(f\"Validation Accuracy: {val_accuracy:.2f}\")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T16:22:11.643480Z",
          "iopub.execute_input": "2024-06-06T16:22:11.643834Z",
          "iopub.status.idle": "2024-06-06T16:22:11.651248Z",
          "shell.execute_reply.started": "2024-06-06T16:22:11.643795Z",
          "shell.execute_reply": "2024-06-06T16:22:11.650203Z"
        },
        "trusted": true,
        "id": "Il1ag4TmaDwi",
        "outputId": "63b87e26-872d-4893-e21d-eb0a2a628864"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Validation Accuracy: 1.00\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Test data processing\n",
        "test_inputs = tokenizer(test_essays['text'].tolist(), padding=True, truncation=True, return_tensors='pt')\n",
        "\n",
        "# Move input tensor to the same device as the model\n",
        "test_inputs = {key: value.to(device) for key, value in test_inputs.items()}\n",
        "\n",
        "# Generate predictions using your trained model\n",
        "with torch.no_grad():\n",
        "    outputs = model(**test_inputs)\n",
        "    logits = outputs.logits\n",
        "\n",
        "# Assuming the first column of logits corresponds to the negative class (non-AI-generated)\n",
        "# and the second column corresponds to the positive class (AI-generated)\n",
        "predictions = torch.softmax(logits, dim=1)[:, 1].cpu().numpy()  # Move predictions back to CPU\n",
        "\n",
        "# Create a submission DataFrame with essay IDs and corresponding predictions\n",
        "submission = pd.DataFrame({\n",
        "    'id': test_essays['id'],\n",
        "    'generated': predictions\n",
        "})\n",
        "\n",
        "# Save the submission DataFrame to a CSV file\n",
        "submission.to_csv('/kaggle/working/submission.csv', index=False)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-06-06T16:22:11.652972Z",
          "iopub.execute_input": "2024-06-06T16:22:11.653383Z",
          "iopub.status.idle": "2024-06-06T16:22:11.690355Z",
          "shell.execute_reply.started": "2024-06-06T16:22:11.653326Z",
          "shell.execute_reply": "2024-06-06T16:22:11.689457Z"
        },
        "trusted": true,
        "id": "yMYj_TxfaDwi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "JKc4Du8yaDwi"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}