WI Validation Performance Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "74c5ea68",
   "metadata": {},
   "source": [
    "# 0. Load required modules."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a62cfb60-92db-4f36-94e5-a32badcbc640",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting numpy==1.21.6\n",
      "  Downloading numpy-1.21.6-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl.metadata (2.1 kB)\n",
      "Downloading numpy-1.21.6-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.7/15.7 MB\u001b[0m \u001b[31m33.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: numpy\n",
      "  Attempting uninstall: numpy\n",
      "    Found existing installation: numpy 1.18.4\n",
      "    Uninstalling numpy-1.18.4:\n",
      "      Successfully uninstalled numpy-1.18.4\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "jiant 2.2.0 requires attrs==19.3.0, but you have attrs 23.2.0 which is incompatible.\n",
      "jiant 2.2.0 requires numpy==1.18.4, but you have numpy 1.21.6 which is incompatible.\n",
      "torchvision 0.9.1 requires torch==1.8.1, but you have torch 2.0.1 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed numpy-1.21.6\n"
     ]
    }
   ],
   "source": [
    "!pip install numpy==1.21.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2fd8d5eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datasets\n",
    "import pickle\n",
    "from transformers import RobertaTokenizer, RobertaModel, RobertaConfig, RobertaForSequenceClassification, RobertaForMaskedLM\n",
    "import torch\n",
    "import numpy as np\n",
    "from tqdm import tqdm_notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "31589830-a9dc-42cc-ab72-064046447d18",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = './validation/wsc/'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a3ad523",
   "metadata": {},
   "source": [
    "# 1. Load the generated dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "3fe7e0f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path+'/data_pos.pickle', 'rb') as f:\n",
    "    data_pos = pickle.load(f)\n",
    "with open(path+'/data_set1.pickle', 'rb') as f:\n",
    "    data_set1 = pickle.load(f)    \n",
    "with open(path+'/data_set2.pickle', 'rb') as f:\n",
    "    data_set2 = pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "222a6025",
   "metadata": {},
   "source": [
    "# 2. Load the trained model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "7cdc351f",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_file = '/home/skhong/jiant/roberta/wsc/runs/simple/best_model.p'\n",
    "config_file = '/home/skhong/jiant/roberta/cb/models/roberta-base/model/config.json'\n",
    "\n",
    "model_version = 'roberta-base'\n",
    "config = RobertaConfig.from_json_file(config_file)\n",
    "model = RobertaForSequenceClassification(config).cuda()\n",
    "\n",
    "state_dict = torch.load(model_file)\n",
    "new_dict = dict()\n",
    "\n",
    "lst = ['encoder.embeddings.position_ids', 'encoder.embeddings.word_embeddings.weight', 'encoder.embeddings.position_embeddings.weight', 'encoder.embeddings.token_type_embeddings.weight', 'encoder.embeddings.LayerNorm.weight', 'encoder.embeddings.LayerNorm.bias']\n",
    "lst2 = [\"taskmodels_dict.wic.head.span_attention_extractor._global_attention._module.weight\", \"taskmodels_dict.wic.head.span_attention_extractor._global_attention._module.bias\"]\n",
    "lst3 = [\"taskmodels_dict.wic.head.classifier.weight\", \"taskmodels_dict.wic.head.classifier.bias\"]\n",
    "for key in state_dict.keys():\n",
    "    if 'taskmodels_dict.wsc.encoder.encoder' in key:\n",
    "        new_dict['roberta.'+key[28:]] = state_dict[key]\n",
    "    elif 'encoder.encoder' in key:\n",
    "        new_dict['roberta.'+key[8:]] = state_dict[key]\n",
    "    elif 'taskmodels_dict.wsc.head.' in key:\n",
    "        new_dict['classifier.'+key[25:]] = state_dict[key]\n",
    "    elif key in lst:\n",
    "        new_dict['roberta.'+key[8:]] = state_dict[key]\n",
    "model.load_state_dict(new_dict,strict=False)\n",
    "tokenizer = RobertaTokenizer.from_pretrained(model_version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3a737c2f-9b25-404c-8b58-7492d2cd2c87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Embedding(50265, 768, padding_idx=1)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.resize_token_embeddings(len(tokenizer))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29d51146",
   "metadata": {},
   "source": [
    "# 3. Extract attention values between tokens.\n",
    "- For models with WI added, perform the addition and extract attention values; for cases where it's not the case, input the excluded ones and extract attention values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "384817c5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_37127/941288186.py:6: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  for i, data in tqdm_notebook(enumerate(data_pos), total=len(data_pos)):\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a5c3ad7923624e6e95cfe0a37fbb789e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=180.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "X = []\n",
    "Y = []\n",
    "\n",
    "model.eval()\n",
    "\n",
    "for i, data in tqdm_notebook(enumerate(data_pos), total=len(data_pos)):\n",
    "    input_ids = data[0]\n",
    "    token_type_ids = data[1]\n",
    "    tfidf_ids = data[2]\n",
    "    \n",
    "    i_pos = data_set1[i][0]\n",
    "    j_pos = data_set1[i][1]\n",
    "    j_random_pos = data_set2[i][1]\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        pred = model(input_ids, attention_mask=token_type_ids)\n",
    "        \n",
    "    temp_x1 = []\n",
    "    temp_x2 = []\n",
    "    \n",
    "    # 128 is the maximum input length for the model's text.\n",
    "    for k in range(12):\n",
    "        temp_data = pred.attentions[k].detach().cpu().numpy().reshape(12, 512, 512)\n",
    "        try:\n",
    "            for l in range(12):\n",
    "                temp_x1.append(temp_data[l][i_pos][j_pos])\n",
    "                temp_x2.append(temp_data[l][i_pos][j_random_pos])\n",
    "        except:\n",
    "            continue\n",
    "    \n",
    "    Y.append(1)\n",
    "    X.append(temp_x1)\n",
    "    Y.append(0)\n",
    "    X.append(temp_x2)\n",
    "    \n",
    "X = np.array(X)\n",
    "Y = np.array(Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1987d6d4",
   "metadata": {},
   "source": [
    "# 4. WI Validation Model based Performance Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "a9430f22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "360\n",
      "360\n",
      "ACC :  0.8055555555555556\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "print(len(Y))\n",
    "print(len(X))\n",
    "#Y = np.array(map(int,Y))\n",
    "new_X = list()\n",
    "new_Y = list()\n",
    "x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, shuffle=True, stratify=Y, random_state=34)\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\n",
    "clf = RandomForestClassifier(\n",
    "            n_estimators=50, \n",
    "            criterion='entropy', \n",
    "            max_depth=5, \n",
    "            max_features='sqrt',\n",
    "            max_samples=0.1,\n",
    "            bootstrap=True,\n",
    "            oob_score=True, \n",
    "            random_state=100\n",
    "        ).fit(x_train,y_train)\n",
    "\n",
    "## Performance Evaluation\n",
    "print('ACC : ', clf.score(x_test,y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e01825c2-ac2a-4331-a0fa-b85a2962a1df",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}