co3d-source-code.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n#import os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n    #for filename in filenames:\n        #print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-24T13:20:20.901481Z","iopub.execute_input":"2023-05-24T13:20:20.902479Z","iopub.status.idle":"2023-05-24T13:20:20.940836Z","shell.execute_reply.started":"2023-05-24T13:20:20.902434Z","shell.execute_reply":"2023-05-24T13:20:20.939756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#Installations\n!pip install Keras-Preprocessing\n!pip install pytorch-pretrained-bert\n!pip install transformers","metadata":{"execution":{"iopub.status.busy":"2023-05-24T13:20:48.613611Z","iopub.execute_input":"2023-05-24T13:20:48.614031Z","iopub.status.idle":"2023-05-24T13:21:32.972926Z","shell.execute_reply.started":"2023-05-24T13:20:48.613995Z","shell.execute_reply":"2023-05-24T13:21:32.971764Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"code","source":"%%time\n#Library Importation\nimport random, re, pickle, random, collections\nfrom tabulate import tabulate\nfrom tqdm import trange\n\n#Data Encoding\nimport nltk, gensim\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom gensim.models import word2vec, KeyedVectors\nfrom gensim.models.word2vec import Word2Vec\nfrom sklearn.decomposition import PCA\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split\n\n#Sentiment Analysis\n#from tensorflow.keras.models import load_model\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Activation, Dense, Dropout, SimpleRNN, LSTM, Embedding\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.metrics import categorical_crossentropy\nfrom tensorflow.python.client import device_lib\nfrom keras.utils import to_categorical\nfrom keras_preprocessing.sequence import pad_sequences\nfrom keras.wrappers.scikit_learn import KerasClassifier\nfrom keras.preprocessing.text import Tokenizer\nfrom transformers import BertTokenizer, BertTokenizerFast, BertForSequenceClassification, BertModel\nfrom transformers import Trainer, TrainingArguments\nfrom transformers import AutoTokenizer, AutoModelForSequenceClassification\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset, TensorDataset, RandomSampler, SequentialSampler","metadata":{"execution":{"iopub.status.busy":"2023-05-24T13:21:37.89317Z","iopub.execute_input":"2023-05-24T13:21:37.894347Z","iopub.status.idle":"2023-05-24T13:21:52.675909Z","shell.execute_reply.started":"2023-05-24T13:21:37.8943Z","shell.execute_reply":"2023-05-24T13:21:52.674578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Get the GPU device name.\n\ndevice_name = tf.test.gpu_device_name()\n\n# The device name should look like the following:\nif device_name == '/device:GPU:0':\n    print('Found GPU at: {}'.format(device_name))\nelse:\n    raise SystemError('GPU device not found')","metadata":{"execution":{"iopub.status.busy":"2023-05-24T13:21:52.678369Z","iopub.execute_input":"2023-05-24T13:21:52.67983Z","iopub.status.idle":"2023-05-24T13:22:00.439111Z","shell.execute_reply.started":"2023-05-24T13:21:52.679783Z","shell.execute_reply":"2023-05-24T13:22:00.437755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# If there's a GPU available tell PyTorch to use the GPU\n\nif torch.cuda.is_available():  \n    device = torch.device(\"cuda\")\n    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n    print('We will use the GPU:', torch.cuda.get_device_name(0))\nelse:\n    print('No GPU available, using the CPU instead.')\n    device = torch.device(\"cpu\")","metadata":{"execution":{"iopub.status.busy":"2023-05-24T13:22:00.443233Z","iopub.execute_input":"2023-05-24T13:22:00.443561Z","iopub.status.idle":"2023-05-24T13:22:00.468963Z","shell.execute_reply.started":"2023-05-24T13:22:00.443529Z","shell.execute_reply":"2023-05-24T13:22:00.467758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Funtion Declaration\n\n#Cleaning Funtion\ndef clean_code(code):\n    \"\"\"\n    Function to clean tweet by:\n    - Changing '&' sign to and\n    - Removing newlines, carriage returns, links, emojis, handles, hashtags and punctuations\n\n    Parameters:\n        tweet (string): The tweet\n    \"\"\"      \n            \n    code = re.sub(\"@[^\\s]+\", \"\",code) # removes handles\n    code = re.sub(\"\\n\", \" \", code) # remove newlines\n    code = re.sub(\"\\r\", \"\", code) # remove carriage returns\n    code = re.sub(r\"http\\S+\", \"\", code) # removes links\n    #code = re.sub(r\"#(\\w+)\", \"\", code) # remove hashtags\n    code = re.sub(\"&\", \"and\", code) # changes & sign to and\n    #code = re.sub(r\"[^\\w\\s\\@]\",\"\",code) # removes punctuation\n    code = code.strip()\n\n    return code\n\n\n#Vector-Encoding Function\n\n#Gensim Word2Vec-Google-News-300\ndef sent_vect(series):\n    \"\"\"This function tokenizes each text and encodes each word in each text with it's vector representation\n    in the word2vec-google-news-300 GENSIM dictionary.\n    \n    This nested list/array will later be converted into a tensor, and fed directly into an RNN\"\"\"\n    \n    shape = series.shape[0]\n    series = list(series.values)\n    array = []\n    pad_array = np.zeros(300)\n    for i in range(shape):\n        word_token = word_tokenize(series[i])\n        word_token = word_token[:60]\n        sample_vector = np.array([list(wv[word]) for word in word_token if word in wv.index_to_key])\n        if sample_vector.shape[0] > 0:\n            if sample_vector.shape[0] >= 60:\n                sample_vector = sample_vector[:60,:]\n            else:\n                deficit = 60-sample_vector.shape[0]\n                for i in range(deficit):\n                    sample_vector = np.vstack((sample_vector, pad_array))\n        else:\n            sample_vector = np.zeros((60, 300))\n        array.append(sample_vector.tolist())\n    return array\n\n#Fucntion for saving the dataset in encoded format to avoid re-encoding everytime a new session is started.\ndef modified_sent_vect(series):\n    \"\"\"This function tokenizes each text and encodes each word in each text with it's vector representation\n    in the word2vec-google-news-300 GENSIM dictionary. This encoded data is then stored in a .csv file.\n    \n    This nested list/array will later be converted into a tensor in a following function, and fed directly \n    into an RNN\"\"\"\n    \n    shape = series.shape[0]\n    series = list(series.values)\n    max_length = 113\n    array = []\n    pad_array = np.zeros(300)\n    for i in range(shape):\n        word_token = word_tokenize(series[i])\n        word_token = word_token[:max_length]\n        sample_vector = np.array([list(wv[word]) for word in word_token if word in wv.index_to_key])\n        if sample_vector.shape[0] > 0:\n            if sample_vector.shape[0] >= max_length:\n                sample_vector = sample_vector[:max_length,:]\n            else:\n                deficit = max_length-sample_vector.shape[0]\n                for i in range(deficit):\n                    sample_vector = np.vstack((sample_vector, pad_array))\n        else:\n            sample_vector = np.zeros((max_length, 300))\n        array.append(sample_vector.tolist())\n        \n    dictionary = {}\n    for i in range(max_length):\n        dictionary[str(i)] = []\n    \n    for i in range(max_length):\n        for item in array:\n            temp_item = res = \" \".join([str(itm) for itm in item[i]])\n            dictionary[str(i)].append(temp_item)\n            \n    dataset = pd.DataFrame(dictionary)\n    \n    return dataset\n\n#Function to read the encoded dataset and convert it into a tensor\ndef convert_data(csv_file):\n    '''Function for importing encoded data stored in a csv file, and converting to tensor'''\n    \n    array = []\n    for i in range(csv_file.shape[0]):\n        #Loops through each row of the dataset\n        temp_list = list(csv_file.iloc[i,:])\n        new_list = []\n        \n        for item in temp_list:\n            #Loops through each column entry in the row, splits and converts to list of floats.\n            chg_list = [float(i) for i in item.split(\" \")]\n            new_list.append(chg_list) #adds each column entry to the list of vector encoded terms\n        temp_arr = np.array(new_list)\n        array.append(temp_arr.tolist()) #adds the entire row of encoded values to the overall input array\n    array = np.array(array)\n    array = tf.convert_to_tensor(array, dtype=tf.float64) #converts the entire array to a tensor.\n    return array\n\n\n#Bert Tokenizer\ndef bert_preprocessing(input_text, tokenizer):\n  '''\n  Returns <class transformers.tokenization_utils_base.BatchEncoding> with the following fields:\n    - input_ids: list of token ids\n    - token_type_ids: list of token type ids\n    - attention_mask: list of indices (0,1) specifying which tokens should considered by the model (return_attention_mask = True).\n  '''\n  return tokenizer.encode_plus(\n                        input_text,\n                        add_special_tokens = True,\n                        max_length = 113,\n                        padding='max_length',\n                        truncation = True,\n                        return_attention_mask = True,\n                        return_tensors = 'pt')\n\n#CodeBERT Tokenizer\ndef codebert_preprocessing(input_text, tokenizer):\n  '''\n  Returns <class transformers.tokenization_utils_base.BatchEncoding> with the following fields:\n    - input_ids: list of token ids\n    - token_type_ids: list of token type ids\n    - attention_mask: list of indices (0,1) specifying which tokens should considered by the model (return_attention_mask = True).\n  '''\n  return tokenizer.encode_plus(\n                        input_text,\n                        add_special_tokens = True,\n                        max_length = 113,\n                        padding='max_length',\n                        truncation = True,\n                        return_attention_mask = True,\n                        return_tensors = 'pt')\n\n\n#Defining Evaluation Metrics\ndef rnn_tp(preds, labels):\n    '''Returns True Positives (TP): count of correct predictions of actual class 1'''\n    return sum([preds == labels and preds == 'COHERENT' for preds, labels in zip(preds, labels)])\n\ndef rnn_fp(preds, labels):\n    '''Returns False Positives (FP): count of wrong predictions of actual class 1'''\n    return sum([preds != labels and preds == 'COHERENT' for preds, labels in zip(preds, labels)])\n\ndef rnn_tn(preds, labels):\n    '''Returns True Negatives (TN): count of correct predictions of actual class 0'''\n    return sum([preds == labels and preds == 'NOT_COHERENT' for preds, labels in zip(preds, labels)])\n\ndef rnn_fn(preds, labels):\n    '''Returns False Negatives (FN): count of wrong predictions of actual class 0'''\n    return sum([preds != labels and preds == 'NOT_COHERENT' for preds, labels in zip(preds, labels)])\n\ndef rnn_metrics(preds, labels):\n    '''\n    Returns the following metrics:\n    - accuracy    = (TP + TN) / N\n    - precision   = TP / (TP + FP)\n    - recall      = TP / (TP + FN)\n    - specificity = TN / (TN + FP)\n    '''\n    \n    stat_dict = {'0':'COHERENT','1':'NOT_COHERENT'}\n    prediction_class = [stat_dict[str(list(row).index(max(list(row))))] for row in preds]\n    labels = labels\n    tp = rnn_tp(prediction_class, labels)\n    tn = rnn_tn(prediction_class, labels)\n    fp = rnn_fp(prediction_class, labels)\n    fn = rnn_fn(prediction_class, labels)\n    b_accuracy = (tp + tn) / len(labels)\n    b_precision = tp / (tp + fp) if (tp + fp) > 0 else 'nan'\n    b_recall = tp / (tp + fn) if (tp + fn) > 0 else 'nan'\n    b_specificity = tn / (tn + fp) if (tn + fp) > 0 else 'nan'\n    f1_score = 2*((b_precision*b_recall)/(b_precision+b_recall)) if (b_precision != 'nan' and b_recall != 'nan') else 'nan'\n    return b_accuracy, b_precision, b_recall, b_specificity, f1_score\n\ndef b_tp(preds, labels):\n    '''Returns True Positives (TP): count of correct predictions of actual class 1'''\n    return sum([preds == labels and preds == 0 for preds, labels in zip(preds, labels)])\n\ndef b_fp(preds, labels):\n    '''Returns False Positives (FP): count of wrong predictions of actual class 1'''\n    return sum([preds != labels and preds == 0 for preds, labels in zip(preds, labels)])\n\ndef b_tn(preds, labels):\n    '''Returns True Negatives (TN): count of correct predictions of actual class 0'''\n    return sum([preds == labels and preds == 1 for preds, labels in zip(preds, labels)])\n\ndef b_fn(preds, labels):\n    '''Returns False Negatives (FN): count of wrong predictions of actual class 0'''\n    return sum([preds != labels and preds == 1 for preds, labels in zip(preds, labels)])\n\ndef b_metrics(preds, labels):\n    '''\n    Returns the following metrics:\n    - accuracy    = (TP + TN) / N\n    - precision   = TP / (TP + FP)\n    - recall      = TP / (TP + FN)\n    - specificity = TN / (TN + FP)\n    '''\n    preds = np.argmax(preds, axis = 1).flatten()\n    labels = labels.flatten()\n    tp = b_tp(preds, labels)\n    tn = b_tn(preds, labels)\n    fp = b_fp(preds, labels)\n    fn = b_fn(preds, labels)\n    b_accuracy = (tp + tn) / len(labels)\n    b_precision = tp / (tp + fp) if (tp + fp) > 0 else 'nan'\n    b_recall = tp / (tp + fn) if (tp + fn) > 0 else 'nan'\n    b_specificity = tn / (tn + fp) if (tn + fp) > 0 else 'nan'\n    f1_score = 2*((b_precision*b_recall)/(b_precision+b_recall)) if (b_precision != 'nan' and b_recall != 'nan') else 'nan'\n    return b_accuracy, b_precision, b_recall, b_specificity, f1_score","metadata":{"execution":{"iopub.status.busy":"2023-05-24T13:22:00.471989Z","iopub.execute_input":"2023-05-24T13:22:00.472429Z","iopub.status.idle":"2023-05-24T13:22:00.524998Z","shell.execute_reply.started":"2023-05-24T13:22:00.472381Z","shell.execute_reply":"2023-05-24T13:22:00.523699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"code","source":"%%time\n#Saving all datasets to in encoded format to reduce experiment time.\nfiles = ['Benchmark_Raw_Data_2.csv','CoffeeMaker_Raw_Data_2.csv','JFreeChart060_Raw_Data_2.csv',\n           'JFreeChart071_Raw_Data_2.csv','JHotDraw741_Raw_Data_2.csv']\nfolders = ['Benchmark','CoffeeMaker','JFreeChart060','JFreeChart071','JHotDraw741']\nparent_path = '/kaggle/input/sourcesniffer/SourceSniffer'\n\n#concatenating all other datasets\nfor i in range(len(files)):\n    curr_path = parent_path+'/'+folders[i]+'/'+files[i]\n    save_path = '/kaggle/working/'+folders[i]+'.csv'\n    temp_df = pd.read_csv(curr_path, encoding='latin-1')\n    temp_df.drop(['Unnamed: 0','Index'], axis=1, inplace=True)\n    \n    #Dataset Preprocessing (Cleaning)\n    temp_df['Code and Comment'] = temp_df['Code and Comment'].str.lower()\n    temp_df['Code and Comment'] = temp_df['Code and Comment'].apply(clean_code)\n    #g_codes2.head(1)\n\n    #Dataset Preprocessing (Assigning non-null entries to new variable)\n    temp_df = temp_df[[\"Code and Comment\",\"Label\"]][~temp_df['Label'].isnull()]\n    temp_comm2 = temp_df['Code and Comment']\n\n    #Dataset Preprocessing (Code-Comment Vector-Encoding)\n    temp_Dataset = modified_sent_vect(temp_comm2)\n    temp_Dataset[\"Label\"] = temp_df[\"Label\"]\n    \n    temp_Dataset.to_csv(save_path)\n    print('{} dataset has been saved!'.format(files[i]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Dataset size check\nfiles = ['Benchmark_Raw_Data_2.csv','CoffeeMaker_Raw_Data_2.csv','JFreeChart060_Raw_Data_2.csv',\n           'JFreeChart071_Raw_Data_2.csv','JHotDraw741_Raw_Data_2.csv']\nfolders = ['Benchmark','CoffeeMaker','JFreeChart060','JFreeChart071','JHotDraw741']\nparent_path = '/kaggle/input/sourcesniffer/SourceSniffer'\n\nfirst_path = parent_path+'/'+folders[0]+'/'+files[0]\ntemp_codes  = pd.read_csv(first_path, encoding='latin-1')\nprint(temp_codes.shape)\n\n#concatenating all other datasets\nfor i in range(1,len(files)):\n    curr_path = parent_path+'/'+folders[i]+'/'+files[i]\n    temp_df = pd.read_csv(curr_path, encoding='latin-1')\n    temp_codes = pd.concat([temp_codes, temp_df], axis=0)\n    print(temp_df.shape, temp_codes.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T16:35:39.442098Z","iopub.execute_input":"2023-05-11T16:35:39.442501Z","iopub.status.idle":"2023-05-11T16:35:39.63336Z","shell.execute_reply.started":"2023-05-11T16:35:39.442458Z","shell.execute_reply":"2023-05-11T16:35:39.632085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"## Gensim + LSTM Bidirectional","metadata":{}},{"cell_type":"markdown","source":"#### Load Tokenizer","metadata":{}},{"cell_type":"code","source":"%%time\n#Corpus, Dictionary etc declaration\n\nwv = KeyedVectors.load('/kaggle/input/gensimcorpus-googlenews300/word2vec-google-news-300/word2vec-google-news-300')","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:34:02.571928Z","iopub.execute_input":"2023-05-23T12:34:02.572644Z","iopub.status.idle":"2023-05-23T12:34:30.119491Z","shell.execute_reply.started":"2023-05-23T12:34:02.572602Z","shell.execute_reply":"2023-05-23T12:34:30.118369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nstop_words = set(stopwords.words('english'))\nps = nltk.PorterStemmer()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:34:30.121708Z","iopub.execute_input":"2023-05-23T12:34:30.122105Z","iopub.status.idle":"2023-05-23T12:34:30.132849Z","shell.execute_reply.started":"2023-05-23T12:34:30.122065Z","shell.execute_reply":"2023-05-23T12:34:30.131605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Benchmark","metadata":{}},{"cell_type":"markdown","source":"#### Loading Dataset and Data Preprocessing","metadata":{}},{"cell_type":"code","source":"%%time\n#Encoded Dataset Importation\ngcoh_data1 = pd.read_csv('/kaggle/input/encoded-datasets/Benchmark.csv')\ngcoh_data1.drop(['Unnamed: 0'], axis=1, inplace=True)\ngcoh_data1.shape\n\n# Creating a dataframe with 80% values of original dataframe \"Train Set\"\ng_training1 = gcoh_data1.sample(frac = 0.80, random_state=2)\ng_trainlabel1 = g_training1['Label']\n\n# Creating dataframe with rest of the 20% values \"Test Set\"\ng_testing1 = gcoh_data1.drop(g_training1.index)\ng_testlabel1 = g_testing1['Label']\n\n#Dataset Preprocessing (Label Encoding)\ng_training1 = convert_data(g_training1.iloc[:,:45])\ng_testing1 = convert_data(g_testing1.iloc[:,:45])\n\nle = preprocessing.LabelEncoder()\ngcoh_stat1 = le.fit_transform(g_trainlabel1)\ngnum_classes1 = 2\n\ng_label1 = np.array(gcoh_stat1)\ng_label1 = to_categorical(g_label1, gnum_classes1)\nprint(g_training1.shape, g_label1.shape)\nprint(g_testing1.shape, g_testlabel1.shape)\n\n#Identify which label numerical code correspond to which categories\nprint(collections.Counter(gcoh_stat1), collections.Counter(g_trainlabel1))","metadata":{"execution":{"iopub.status.busy":"2023-05-24T14:28:32.152944Z","iopub.execute_input":"2023-05-24T14:28:32.153566Z","iopub.status.idle":"2023-05-24T14:29:00.744443Z","shell.execute_reply.started":"2023-05-24T14:28:32.153519Z","shell.execute_reply":"2023-05-24T14:29:00.743242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model Training and Evaluation","metadata":{}},{"cell_type":"code","source":"%%time\n#Nodes: 100, Lr: 0.0005, Epochs: 40-50\nlr_list = [0.0005, 0.00075, 0.001]\nnodes = [100, 200, 300]\nepochs = [30, 40, 50]\n\nfor epoch in epochs:\n    for node in nodes:\n        for lr in lr_list:\n            #Model Object Initiation\n            g_model1 = Sequential([\n                SimpleRNN(node, input_shape = (g_training1.shape[1], g_training1.shape[2]), return_sequences = False),\n                Dense(gnum_classes1, activation='softmax'),\n            ])\n\n            #Model Training\n            g_epochs1 = epoch\n            g_model1.compile(loss='categorical_crossentropy', optimizer = Adam(learning_rate=lr), metrics = ['accuracy'])\n            g_hist1 = g_model1.fit(g_training1, g_label1, epochs = g_epochs1, batch_size = 50, validation_split=False, verbose=False)\n            print('Epochs Complete: Epochs:{}, Hidden Nodes:{}, Learning Rate:{}.'.format(epoch, node, lr))\n            print('47/47 [==============================] - loss: {} - accuracy: {}'.format(g_hist1.history[\"loss\"][g_epochs1 - 1], g_hist1.history[\"accuracy\"][g_epochs1 - 1]))\n            print('\\n')\n\n            #Model Evaluation\n            pred1 = g_model1.predict(g_testing1, verbose=False)\n            rnn_accuracy, rnn_precision, rnn_recall, rnn_specificity, f1_score = rnn_metrics(pred1, g_testlabel1)\n            print('Validation Accuracy: {:.4f}'.format(rnn_accuracy))\n            print('Validation Precision: {:.4f}'.format(rnn_precision))\n            print('Validation Recall: {:.4f}'.format(rnn_recall))\n            print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{"execution":{"iopub.status.busy":"2023-05-24T14:29:00.746597Z","iopub.execute_input":"2023-05-24T14:29:00.747317Z","iopub.status.idle":"2023-05-24T15:08:24.227562Z","shell.execute_reply.started":"2023-05-24T14:29:00.747267Z","shell.execute_reply":"2023-05-24T15:08:24.225989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#Nodes: 100, Lr: 0.0005, Epochs: 40-50\nlr_list = [0.0005, 0.00075, 0.001]\nnodes = [100, 200, 300]\nepochs = [30, 40, 50]\n\nfor epoch in epochs:\n    for node in nodes:\n        for lr in lr_list:\n            #Model Object Initiation\n            g_model1_5 = Sequential([\n                LSTM(node, input_shape = (g_training1.shape[1], g_training1.shape[2]), return_sequences = False),\n                Dense(gnum_classes1, activation='softmax'),\n            ])\n\n            #Model Training\n            g_epochs1_5 = epoch\n            g_model1_5.compile(loss='categorical_crossentropy', optimizer = Adam(learning_rate=lr), metrics = ['accuracy'])\n            g_hist1_5 = g_model1_5.fit(g_training1, g_label1, epochs = g_epochs1_5, batch_size = 50, validation_split=False, verbose=False)\n            print('Epochs Complete: Epochs:{}, Hidden Nodes:{}, Learning Rate:{}.'.format(epoch, node, lr))\n            print('47/47 [==============================] - loss: {} - accuracy: {}'.format(g_hist1_5.history[\"loss\"][g_epochs1_5 - 1], g_hist1_5.history[\"accuracy\"][g_epochs1_5 - 1]))\n            print('\\n')\n\n            #Model Evaluation\n            pred1_5 = g_model1_5.predict(g_testing1, verbose=False)\n            rnn_accuracy, rnn_precision, rnn_recall, rnn_specificity, f1_score = rnn_metrics(pred1_5, g_testlabel1)\n            print('Validation Accuracy: {:.4f}'.format(rnn_accuracy))\n            print('Validation Precision: {:.4f}'.format(rnn_precision))\n            print('Validation Recall: {:.4f}'.format(rnn_recall))\n            print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{"execution":{"iopub.status.busy":"2023-05-24T15:08:24.230245Z","iopub.execute_input":"2023-05-24T15:08:24.231605Z","iopub.status.idle":"2023-05-24T15:20:18.861972Z","shell.execute_reply.started":"2023-05-24T15:08:24.231553Z","shell.execute_reply":"2023-05-24T15:20:18.859624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CoffeeMaker","metadata":{}},{"cell_type":"markdown","source":"#### Loading Dataset and Data Preprocessing","metadata":{}},{"cell_type":"code","source":"%%time\n#Encoded Dataset Importation\ngcoh_data2 = pd.read_csv('/kaggle/input/encoded-datasets-75maxlength/Max_Length_75/CoffeeMaker.csv')\ngcoh_data2.drop(['Unnamed: 0'], axis=1, inplace=True)\ngcoh_data2.shape\n\n# Creating a dataframe with 80% values of original dataframe \"Train Set\"\ng_training2 = gcoh_data2.sample(frac = 0.80)\ng_trainlabel2 = g_training2['Label']\n\n# Creating dataframe with rest of the 20% values \"Test Set\"\ng_testing2 = gcoh_data2.drop(g_training2.index)\ng_testlabel2 = g_testing2['Label']\n\n#Dataset Preprocessing (Label Encoding)\ng_training2 = convert_data(g_training2.iloc[:,:45])\ng_testing2 = convert_data(g_testing2.iloc[:,:45])\n\nle = preprocessing.LabelEncoder()\ngcoh_stat2 = le.fit_transform(g_trainlabel2)\ngnum_classes2 = 2\n\ng_label2 = np.array(gcoh_stat2)\ng_label2 = to_categorical(g_label2, gnum_classes2)\nprint(g_training2.shape, g_label2.shape)\nprint(g_testing2.shape, g_testlabel2.shape)\n\n#Identify which label numerical code correspond to which categories\nprint(collections.Counter(gcoh_stat2), collections.Counter(g_trainlabel2))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T20:17:30.417783Z","iopub.execute_input":"2023-05-23T20:17:30.419295Z","iopub.status.idle":"2023-05-23T20:17:30.994698Z","shell.execute_reply.started":"2023-05-23T20:17:30.419247Z","shell.execute_reply":"2023-05-23T20:17:30.993239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model Training and Evaluation","metadata":{}},{"cell_type":"code","source":"%%time\n#Nodes: 100, Lr: 0.0005, Epochs: 40-50\nlr_list = [0.0005, 0.00075, 0.001]\nnodes = [100, 200, 300]\nepochs = [30, 40, 50]\n\nfor epoch in epochs:\n    for node in nodes:\n        for lr in lr_list:\n            #Model Object Initiation\n            g_model2 = Sequential([\n                SimpleRNN(node, input_shape = (g_training2.shape[1], g_training2.shape[2]), return_sequences = False),\n                Dense(gnum_classes2, activation='softmax'),\n            ])\n\n            #Model Training and Evaluation\n            g_epochs2 = epoch\n            g_model2.compile(loss='categorical_crossentropy', optimizer = Adam(learning_rate=lr), metrics = ['accuracy'])\n            g_hist2 = g_model2.fit(g_training2, g_label2, epochs = g_epochs2, batch_size = 50, validation_split=False, verbose=False)\n            print('Epochs Complete: Epochs:{}, Hidden Nodes:{}, Learning Rate:{}.'.format(epoch, node, lr))\n            print('47/47 [==============================] - loss: {} - accuracy: {}'.format(g_hist2.history[\"loss\"][g_epochs2 - 1], g_hist2.history[\"accuracy\"][g_epochs2 - 1]))\n            print('\\n')\n\n            #Model Evaluation\n            pred2 = g_model2.predict(g_testing2, verbose=False)\n            rnn_accuracy, rnn_precision, rnn_recall, rnn_specificity, f1_score = rnn_metrics(pred2, g_testlabel2)\n            print('Validation Accuracy: {:.4f}'.format(rnn_accuracy))\n            print('Validation Precision: {:.4f}'.format(rnn_precision))\n            print('Validation Recall: {:.4f}'.format(rnn_recall))\n            print('Validation Specificity: {:.4f}'.format(rnn_specificity))\n            print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T20:22:56.656253Z","iopub.execute_input":"2023-05-23T20:22:56.657082Z","iopub.status.idle":"2023-05-23T20:24:39.37886Z","shell.execute_reply.started":"2023-05-23T20:22:56.657037Z","shell.execute_reply":"2023-05-23T20:24:39.377597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#Nodes: 100, Lr: 0.0005, Epochs: 40-50\nlr_list = [0.0005, 0.00075, 0.001]\nnodes = [100, 200, 300]\nepochs = [30, 40, 50]\n\nfor epoch in epochs:\n    for node in nodes:\n        for lr in lr_list:\n            #Model Object Initiation\n            g_model2_5 = Sequential([\n                LSTM(node, input_shape = (g_training2.shape[1], g_training2.shape[2]), return_sequences = False),\n                Dense(gnum_classes2, activation='softmax'),\n            ])\n            #Model Training and Evaluation\n            g_epochs2_5 = epoch\n            g_model2_5.compile(loss='categorical_crossentropy', optimizer = Adam(learning_rate=lr), metrics = ['accuracy'])\n            g_hist2_5 = g_model2_5.fit(g_training2, g_label2, epochs = g_epochs2_5, batch_size = 50, validation_split=False, verbose=False)\n            print('Epochs Complete: Epochs:{}, Hidden Nodes:{}, Learning Rate:{}.'.format(epoch, node, lr))\n            print('47/47 [==============================] - loss: {} - accuracy: {}'.format(g_hist2_5.history[\"loss\"][g_epochs2_5 - 1], g_hist2_5.history[\"accuracy\"][g_epochs2_5 - 1]))\n            print('\\n')\n\n            #Model Evaluation\n            pred2_5 = g_model2_5.predict(g_testing2, verbose=False)\n            rnn_accuracy, rnn_precision, rnn_recall, rnn_specificity, f1_score = rnn_metrics(pred2_5, g_testlabel2)\n            print('Validation Accuracy: {:.4f}'.format(rnn_accuracy))\n            print('Validation Precision: {:.4f}'.format(rnn_precision))\n            print('Validation Recall: {:.4f}'.format(rnn_recall))\n            print('Validation Specificity: {:.4f}'.format(rnn_specificity))\n            print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T20:24:39.381753Z","iopub.execute_input":"2023-05-23T20:24:39.38213Z","iopub.status.idle":"2023-05-23T20:26:22.556581Z","shell.execute_reply.started":"2023-05-23T20:24:39.382089Z","shell.execute_reply":"2023-05-23T20:26:22.55545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### JFreeChart060","metadata":{}},{"cell_type":"markdown","source":"#### Loading Dataset and Data Preprocessing","metadata":{}},{"cell_type":"code","source":"%%time\n#Encoded Dataset Importation\ngcoh_data3 = pd.read_csv('/kaggle/input/encoded-datasets-75maxlength/Max_Length_75/JFreeChart060.csv')\ngcoh_data3.drop(['Unnamed: 0'], axis=1, inplace=True)\ngcoh_data3.shape\n\n# Creating a dataframe with 80% values of original dataframe \"Train Set\"\ng_training3 = gcoh_data3.sample(frac = 0.80, random_state=2)\ng_trainlabel3 = g_training3['Label']\n\n# Creating dataframe with rest of the 20% values \"Test Set\"\ng_testing3 = gcoh_data3.drop(g_training3.index)\ng_testlabel3 = g_testing3['Label']\n\n#Dataset Preprocessing (Label Encoding)\ng_training3 = convert_data(g_training3.iloc[:,:45])\ng_testing3 = convert_data(g_testing3.iloc[:,:45])\n\nle = preprocessing.LabelEncoder()\ngcoh_stat3 = le.fit_transform(g_trainlabel3)\ngnum_classes3 = 2\n\ng_label3 = np.array(gcoh_stat3)\ng_label3 = to_categorical(g_label3, gnum_classes3)\nprint(g_training3.shape, g_label3.shape)\nprint(g_testing3.shape, g_testlabel3.shape)\n\n#Identify which label numerical code correspond to which categories\nprint(collections.Counter(gcoh_stat3), collections.Counter(g_trainlabel3))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T19:38:36.429833Z","iopub.execute_input":"2023-05-23T19:38:36.430299Z","iopub.status.idle":"2023-05-23T19:38:40.793063Z","shell.execute_reply.started":"2023-05-23T19:38:36.430261Z","shell.execute_reply":"2023-05-23T19:38:40.791864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model Training and Evaluation","metadata":{}},{"cell_type":"code","source":"%%time\n#Nodes: 100, Lr: 0.0005, Epochs: 40-50\nlr_list = [0.0005, 0.00075, 0.001]\nnodes = [100, 200, 300]\nepochs = [30, 40, 50]\n\nfor epoch in epochs:\n    for node in nodes:\n        for lr in lr_list:\n            #Model Object Initiation\n            g_model3 = Sequential([\n                SimpleRNN(node, input_shape = (g_training3.shape[1], g_training3.shape[2]), return_sequences = False),\n                Dense(gnum_classes3, activation='softmax'),\n            ])\n\n            #Model Training and Evaluation\n            g_epochs3 = epoch\n            g_model3.compile(loss='categorical_crossentropy', optimizer = Adam(learning_rate=lr), metrics = ['accuracy'])\n            g_hist3 = g_model3.fit(g_training3, g_label3, epochs = g_epochs3, batch_size = 50, validation_split=False, verbose=False)\n            print('Epochs Complete: Epochs:{}, Hidden Nodes:{}, Learning Rate:{}.'.format(epoch, node, lr))\n            print('47/47 [==============================] - loss: {} - accuracy: {}'.format(g_hist3.history[\"loss\"][g_epochs3 - 1], g_hist3.history[\"accuracy\"][g_epochs3 - 1]))\n            print('\\n')\n\n            #Model Evaluation\n            pred3 = g_model3.predict(g_testing3, verbose=False)\n            rnn_accuracy, rnn_precision, rnn_recall, rnn_specificity, f1_score = rnn_metrics(pred3, g_testlabel3)\n            print('Validation Accuracy: {:.4f}'.format(rnn_accuracy))\n            print('Validation Precision: {:.4f}'.format(rnn_precision))\n            print('Validation Recall: {:.4f}'.format(rnn_recall))\n            print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T19:40:35.165838Z","iopub.execute_input":"2023-05-23T19:40:35.16661Z","iopub.status.idle":"2023-05-23T19:48:03.064342Z","shell.execute_reply.started":"2023-05-23T19:40:35.166571Z","shell.execute_reply":"2023-05-23T19:48:03.063261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#Nodes: 100, Lr: 0.0005, Epochs: 40-50\nlr_list = [0.0005, 0.00075, 0.001]\nnodes = [100, 200, 300]\nepochs = [30, 40, 50]\n\nfor epoch in epochs:\n    for node in nodes:\n        for lr in lr_list:\n            #Model Object Initiation\n            g_model3_5 = Sequential([\n                LSTM(node, input_shape = (g_training3.shape[1], g_training3.shape[2]), return_sequences = False),\n                Dense(gnum_classes3, activation='softmax'),\n            ])\n\n            #Model Training and Evaluation\n            g_epochs3_5 = epoch\n            g_model3_5.compile(loss='categorical_crossentropy', optimizer = Adam(learning_rate=lr), metrics = ['accuracy'])\n            g_hist3_5 = g_model3_5.fit(g_training3, g_label3, epochs = g_epochs3_5, batch_size = 50, validation_split=False, verbose=False)\n            print('Epochs Complete: Epochs:{}, Hidden Nodes:{}, Learning Rate:{}.'.format(epoch, node, lr))\n            print('47/47 [==============================] - loss: {} - accuracy: {}'.format(g_hist3_5.history[\"loss\"][g_epochs3_5 - 1], g_hist3_5.history[\"accuracy\"][g_epochs3_5 - 1]))\n            print('\\n')\n\n            #Model Evaluation\n            pred3_5 = g_model3_5.predict(g_testing3, verbose=False)\n            rnn_accuracy, rnn_precision, rnn_recall, rnn_specificity, f1_score = rnn_metrics(pred3_5, g_testlabel3)\n            print('Validation Accuracy: {:.4f}'.format(rnn_accuracy))\n            print('Validation Precision: {:.4f}'.format(rnn_precision))\n            print('Validation Recall: {:.4f}'.format(rnn_recall))\n            print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T19:48:03.06679Z","iopub.execute_input":"2023-05-23T19:48:03.067478Z","iopub.status.idle":"2023-05-23T19:51:02.320357Z","shell.execute_reply.started":"2023-05-23T19:48:03.067424Z","shell.execute_reply":"2023-05-23T19:51:02.319188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### JFreeChart071","metadata":{}},{"cell_type":"markdown","source":"#### Loading Dataset and Data Preprocessing","metadata":{}},{"cell_type":"code","source":"%%time\n#Encoded Dataset Importation\ngcoh_data4 = pd.read_csv('/kaggle/input/encoded-datasets-75maxlength/Max_Length_75/JFreeChart071.csv')\ngcoh_data4.drop(['Unnamed: 0'], axis=1, inplace=True)\ngcoh_data4.shape\n\n# Creating a dataframe with 80% values of original dataframe \"Train Set\"\ng_training4 = gcoh_data4.sample(frac = 0.80, random_state=10)\ng_trainlabel4 = g_training4['Label']\n\n# Creating dataframe with rest of the 20% values \"Test Set\"\ng_testing4 = gcoh_data4.drop(g_training4.index)\ng_testlabel4 = g_testing4['Label']\n\n#Dataset Preprocessing (Label Encoding)\ng_training4 = convert_data(g_training4.iloc[:,:45])\ng_testing4 = convert_data(g_testing4.iloc[:,:45])\n\nle = preprocessing.LabelEncoder()\ngcoh_stat4 = le.fit_transform(g_trainlabel4)\ngnum_classes4 = 2\n\ng_label4 = np.array(gcoh_stat4)\ng_label4 = to_categorical(g_label4, gnum_classes4)\nprint(g_training4.shape, g_label4.shape)\nprint(g_testing4.shape, g_testlabel4.shape)\n\n#Identify which label numerical code correspond to which categories\nprint(collections.Counter(gcoh_stat4), collections.Counter(g_trainlabel4))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T21:25:29.421356Z","iopub.execute_input":"2023-05-23T21:25:29.422343Z","iopub.status.idle":"2023-05-23T21:25:34.921546Z","shell.execute_reply.started":"2023-05-23T21:25:29.422278Z","shell.execute_reply":"2023-05-23T21:25:34.920366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model Training and Evaluation","metadata":{}},{"cell_type":"code","source":"%%time\n#Nodes: 100, Lr: 0.0005, Epochs: 40-50\nlr_list = [0.0005, 0.00075, 0.001]\nnodes = [100, 200, 300]\nepochs = [30, 40, 50]\n\nfor epoch in epochs:\n    for node in nodes:\n        for lr in lr_list:\n            #Model Object Initiation\n            g_model4 = Sequential([\n                SimpleRNN(node, input_shape = (g_training4.shape[1], g_training4.shape[2]), return_sequences = False),\n                Dense(gnum_classes4, activation='softmax'),\n            ])\n\n            #Model Training and Evaluation\n            g_epochs4 = epoch\n            g_model4.compile(loss='categorical_crossentropy', optimizer = Adam(learning_rate=lr), metrics = ['accuracy'])\n            g_hist4 = g_model4.fit(g_training4, g_label4, epochs = g_epochs4, batch_size = 50, validation_split=False, verbose=False)\n            print('Epochs Complete: Epochs:{}, Hidden Nodes:{}, Learning Rate:{}.'.format(epoch, node, lr))\n            print('47/47 [==============================] - loss: {} - accuracy: {}'.format(g_hist4.history[\"loss\"][g_epochs4 - 1], g_hist4.history[\"accuracy\"][g_epochs4 - 1]))\n            print('\\n')\n\n            #Model Evaluation\n            pred4 = g_model4.predict(g_testing4, verbose=False)\n            rnn_accuracy, rnn_precision, rnn_recall, rnn_specificity, f1_score = rnn_metrics(pred4, g_testlabel4)\n            print('Validation Accuracy: {:.4f}'.format(rnn_accuracy))\n            print('Validation Precision: {:.4f}'.format(rnn_precision))\n            print('Validation Recall: {:.4f}'.format(rnn_recall))\n            print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T20:43:36.04522Z","iopub.execute_input":"2023-05-23T20:43:36.046517Z","iopub.status.idle":"2023-05-23T20:53:16.037697Z","shell.execute_reply.started":"2023-05-23T20:43:36.046464Z","shell.execute_reply":"2023-05-23T20:53:16.036348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#Nodes: 100, Lr: 0.0005, Epochs: 40-50\nlr_list = [0.0005, 0.00075, 0.001]\nnodes = [100, 200, 300]\nepochs = [10, 20, 30]\n\nfor epoch in epochs:\n    for node in nodes:\n        for lr in lr_list:\n            #Model Object Initiation\n            g_model4_5 = Sequential([\n                LSTM(node, input_shape = (g_training4.shape[1], g_training4.shape[2]), return_sequences = False),\n                Dense(gnum_classes4, activation='softmax'),\n            ])\n\n            #Model Training and Evaluation\n            g_epochs4_5 = epoch\n            g_model4_5.compile(loss='categorical_crossentropy', optimizer = Adam(learning_rate=lr), metrics = ['accuracy'])\n            g_hist4_5 = g_model4_5.fit(g_training4, g_label4, epochs = g_epochs4_5, batch_size = 50, validation_split=False, verbose=False)\n            print('Epochs Complete: Epochs:{}, Hidden Nodes:{}, Learning Rate:{}.'.format(epoch, node, lr))\n            print('47/47 [==============================] - loss: {} - accuracy: {}'.format(g_hist4_5.history[\"loss\"][g_epochs4_5 - 1], g_hist4_5.history[\"accuracy\"][g_epochs4_5 - 1]))\n            print('\\n')\n\n            #Model Evaluation\n            pred4_5 = g_model4_5.predict(g_testing4, verbose=False)\n            rnn_accuracy, rnn_precision, rnn_recall, rnn_specificity, f1_score = rnn_metrics(pred4_5, g_testlabel4)\n            print('Validation Accuracy: {:.4f}'.format(rnn_accuracy))\n            print('Validation Precision: {:.4f}'.format(rnn_precision))\n            print('Validation Recall: {:.4f}'.format(rnn_recall))\n            print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T21:26:32.917846Z","iopub.execute_input":"2023-05-23T21:26:32.918856Z","iopub.status.idle":"2023-05-23T21:28:56.302525Z","shell.execute_reply.started":"2023-05-23T21:26:32.918797Z","shell.execute_reply":"2023-05-23T21:28:56.301425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### JHotDraw741","metadata":{}},{"cell_type":"markdown","source":"#### Loading Dataset and Data Preprocessing","metadata":{}},{"cell_type":"code","source":"%%time\n#Encoded Dataset Importation\ngcoh_data5 = pd.read_csv('/kaggle/input/encoded-datasets/JHotDraw741.csv')\ngcoh_data5.drop(['Unnamed: 0'], axis=1, inplace=True)\ngcoh_data5.shape\n\n# Creating a dataframe with 80% values of original dataframe \"Train Set\"\ng_training5 = gcoh_data5.sample(frac = 0.80, random_state = 2)\ng_trainlabel5 = g_training5['Label']\n\n# Creating dataframe with rest of the 20% values \"Test Set\"\ng_testing5 = gcoh_data5.drop(g_training5.index)\ng_testlabel5 = g_testing5['Label']\n\n#Dataset Preprocessing (Label Encoding)\ng_training5 = convert_data(g_training5.iloc[:,:45])\ng_testing5 = convert_data(g_testing5.iloc[:,:45])\n\nle = preprocessing.LabelEncoder()\ngcoh_stat5 = le.fit_transform(g_trainlabel5)\ngnum_classes5 = 2\n\ng_label5 = np.array(gcoh_stat5)\ng_label5 = to_categorical(g_label5, gnum_classes5)\nprint(g_training5.shape, g_label5.shape)\nprint(g_testing5.shape, g_testlabel5.shape)\n\n#Identify which label numerical code correspond to which categories\nprint(collections.Counter(gcoh_stat5), collections.Counter(g_trainlabel5))","metadata":{"execution":{"iopub.status.busy":"2023-05-24T15:43:16.943Z","iopub.execute_input":"2023-05-24T15:43:16.943494Z","iopub.status.idle":"2023-05-24T15:43:36.746718Z","shell.execute_reply.started":"2023-05-24T15:43:16.943443Z","shell.execute_reply":"2023-05-24T15:43:36.745329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model Training and Evaluation","metadata":{}},{"cell_type":"code","source":"%%time\n#Nodes: 100, Lr: 0.0005, Epochs: 40-50\nlr_list = [0.0005, 0.00075, 0.001]\nnodes = [100, 200, 300]\nepochs = [30, 40, 50]\n\nfor epoch in epochs:\n    for node in nodes:\n        for lr in lr_list:\n            #Model Object Initiation\n            g_model5 = Sequential([\n                SimpleRNN(node, input_shape = (g_training5.shape[1], g_training5.shape[2]), return_sequences = False),\n                Dense(gnum_classes5, activation='softmax'),\n            ])\n\n            #Model Training and Evaluation\n            g_epochs5 = epoch\n            g_model5.compile(loss='categorical_crossentropy', optimizer = Adam(learning_rate=lr), metrics = ['accuracy'])\n            g_hist5 = g_model5.fit(g_training5, g_label5, epochs = g_epochs5, batch_size = 50, validation_split=False, verbose=False)\n            print('Epochs Complete: Epochs:{}, Hidden Nodes:{}, Learning Rate:{}.'.format(epoch, node, lr))\n            print('47/47 [==============================] - loss: {} - accuracy: {}'.format(g_hist5.history[\"loss\"][g_epochs5 - 1], g_hist5.history[\"accuracy\"][g_epochs5 - 1]))\n            print('\\n')\n\n            #Model Evaluation\n            pred5 = g_model5.predict(g_testing5, verbose=False)\n            rnn_accuracy, rnn_precision, rnn_recall, rnn_specificity, f1_score = rnn_metrics(pred5, g_testlabel5)\n            print('Validation Accuracy: {:.4f}'.format(rnn_accuracy))\n            print('Validation Precision: {:.4f}'.format(rnn_precision))\n            print('Validation Recall: {:.4f}'.format(rnn_recall))\n            print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{"execution":{"iopub.status.busy":"2023-05-24T15:45:21.823589Z","iopub.execute_input":"2023-05-24T15:45:21.824422Z","iopub.status.idle":"2023-05-24T16:12:29.092335Z","shell.execute_reply.started":"2023-05-24T15:45:21.824382Z","shell.execute_reply":"2023-05-24T16:12:29.090873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#Nodes: 100, Lr: 0.0005, Epochs: 40-50\nlr_list = [0.0005, 0.00075, 0.001]\nnodes = [100, 200, 300]\nepochs = [20, 30, 40, 50]\n\nfor epoch in epochs:\n    for node in nodes:\n        for lr in lr_list:\n            #Model Object Initiation\n            g_model5_5 = Sequential([\n                LSTM(node, input_shape = (g_training5.shape[1], g_training5.shape[2]), return_sequences = False),\n                Dense(gnum_classes5, activation='softmax'),\n            ])\n\n            #Model Training and Evaluation\n            g_epochs5_5 = epoch\n            g_model5_5.compile(loss='categorical_crossentropy', optimizer = Adam(learning_rate=lr), metrics = ['accuracy'])\n            g_hist5_5 = g_model5_5.fit(g_training5, g_label5, epochs = g_epochs5_5, batch_size = 50, validation_split=False, verbose=False)\n            print('Epochs Complete: Epochs:{}, Hidden Nodes:{}, Learning Rate:{}.'.format(epoch, node, lr))\n            print('47/47 [==============================] - loss: {} - accuracy: {}'.format(g_hist5_5.history[\"loss\"][g_epochs5_5 - 1], g_hist5_5.history[\"accuracy\"][g_epochs5_5 - 1]))\n            print('\\n')\n\n            #Model Evaluation\n            pred5_5 = g_model5_5.predict(g_testing5, verbose=False)\n            rnn_accuracy, rnn_precision, rnn_recall, rnn_specificity, f1_score = rnn_metrics(pred5_5, g_testlabel5)\n            print('Validation Accuracy: {:.4f}'.format(rnn_accuracy))\n            print('Validation Precision: {:.4f}'.format(rnn_precision))\n            print('Validation Recall: {:.4f}'.format(rnn_recall))\n            print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{"execution":{"iopub.status.busy":"2023-05-24T16:12:29.094967Z","iopub.execute_input":"2023-05-24T16:12:29.095981Z","iopub.status.idle":"2023-05-24T16:21:29.30462Z","shell.execute_reply.started":"2023-05-24T16:12:29.095932Z","shell.execute_reply":"2023-05-24T16:21:29.303251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### All Datasets Combined","metadata":{}},{"cell_type":"markdown","source":"#### Loading Dataset and Data Preprocessing","metadata":{}},{"cell_type":"code","source":"%%time\n#Encoded Dataset Importation\nfiles = ['Benchmark.csv','CoffeeMaker.csv','JFreeChart060.csv','JFreeChart071.csv','JHotDraw741.csv']\nparent_path = '/kaggle/input/encoded-datasets'\nfirst_path = parent_path+'/'+files[0]\ng_codes6  = pd.read_csv(first_path)\n\n#concatenating all other datasets\nfor i in range(1,len(files)):\n    curr_path = parent_path+'/'+files[i]\n    temp_df = pd.read_csv(curr_path)\n    g_codes6 = pd.concat([g_codes6, temp_df], axis=0)   \ng_codes6.drop(['Unnamed: 0'], axis=1, inplace=True)\n\n# Creating a dataframe with 80% values of original dataframe \"Train Set\"\ng_training6 = g_codes6.sample(frac = 0.80, random_state = 2)\ng_trainlabel6 = g_training6['Label']\n\n# Creating dataframe with rest of the 20% values \"Test Set\"\ng_testing6 = g_codes6.drop(g_training6.index)\ng_testlabel6 = g_testing6['Label']\n\n#Dataset Preprocessing (Label Encoding)\ng_training6 = convert_data(g_training6.iloc[:,:45])\ng_testing6 = convert_data(g_testing6.iloc[:,:45])\n\nle = preprocessing.LabelEncoder()\ngcoh_stat6 = le.fit_transform(g_trainlabel6)\ngnum_classes6 = 2\n\ng_label6 = np.array(gcoh_stat6)\ng_label6 = to_categorical(g_label6, gnum_classes6)\nprint(g_training6.shape, g_label6.shape)\nprint(g_testing6.shape, g_testlabel6.shape)\n\n#Identify which label numerical code correspond to which categories\nprint(collections.Counter(gcoh_stat6), collections.Counter(g_trainlabel6))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T22:00:50.476083Z","iopub.execute_input":"2023-05-23T22:00:50.476694Z","iopub.status.idle":"2023-05-23T22:01:43.309499Z","shell.execute_reply.started":"2023-05-23T22:00:50.476654Z","shell.execute_reply":"2023-05-23T22:01:43.308198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model Training and Evaluation","metadata":{}},{"cell_type":"code","source":"%%time\n#Nodes: 100, Lr: 0.0005, Epochs: 40-50\nlr_list = [0.0005, 0.00075, 0.001]\nnodes = [100, 200, 300]\nepochs = [30, 40, 50]\n\nfor epoch in epochs:\n    for node in nodes:\n        for lr in lr_list:\n            #Model Object Initiation\n            g_model6 = Sequential([\n                SimpleRNN(node, input_shape = (g_training6.shape[1], g_training6.shape[2]), return_sequences = False),\n                Dense(gnum_classes6, activation='softmax'),\n            ])\n\n            #Model Training and Evaluation\n            g_epochs6 = epoch\n            g_model6.compile(loss='categorical_crossentropy', optimizer = Adam(learning_rate=lr), metrics = ['accuracy'])\n            g_hist6 = g_model6.fit(g_training6, g_label6, epochs = g_epochs6, batch_size = 50, validation_split=False, verbose=False)\n            print('Epochs Complete: Epochs:{}, Hidden Nodes:{}, Learning Rate:{}.'.format(epoch, node, lr))\n            print('47/47 [==============================] - loss: {} - accuracy: {}'.format(g_hist6.history[\"loss\"][g_epochs6 - 1], g_hist6.history[\"accuracy\"][g_epochs6 - 1]))\n            print('\\n')\n\n            #Model Evaluation\n            pred6 = g_model6.predict(g_testing6, verbose=False)\n            rnn_accuracy, rnn_precision, rnn_recall, rnn_specificity, f1_score = rnn_metrics(pred6, g_testlabel6)\n            print('Validation Accuracy: {:.4f}'.format(rnn_accuracy))\n            print('Validation Precision: {:.4f}'.format(rnn_precision))\n            print('Validation Recall: {:.4f}'.format(rnn_recall))\n            print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T22:02:03.913433Z","iopub.execute_input":"2023-05-23T22:02:03.914233Z","iopub.status.idle":"2023-05-23T23:08:02.657753Z","shell.execute_reply.started":"2023-05-23T22:02:03.914192Z","shell.execute_reply":"2023-05-23T23:08:02.656672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#Nodes: 100, Lr: 0.0005, Epochs: 40-50\nlr_list = [0.0005, 0.00075, 0.001]\nnodes = [100, 200, 300]\nepochs = [30, 40, 50]\n\nfor epoch in epochs:\n    for node in nodes:\n        for lr in lr_list:\n            #Model Object Initiation\n            g_model6_5 = Sequential([\n                LSTM(node, input_shape = (g_training6.shape[1], g_training6.shape[2]), return_sequences = False),\n                Dense(gnum_classes6, activation='softmax'),\n            ])\n\n            #Model Training and Evaluation\n            g_epochs6_5 = epoch\n            g_model6_5.compile(loss='categorical_crossentropy', optimizer = Adam(learning_rate=lr), metrics = ['accuracy'])\n            g_hist6_5 = g_model6_5.fit(g_training6, g_label6, epochs = g_epochs6_5, batch_size = 50, validation_split=False, verbose=False)\n            print('Epochs Complete: Epochs:{}, Hidden Nodes:{}, Learning Rate:{}.'.format(epoch, node, lr))\n            print('47/47 [==============================] - loss: {} - accuracy: {}'.format(g_hist6_5.history[\"loss\"][g_epochs6_5 - 1], g_hist6_5.history[\"accuracy\"][g_epochs6_5 - 1]))\n            print('\\n')\n\n            #Model Evaluation\n            pred6_5 = g_model6_5.predict(g_testing6, verbose=False)\n            rnn_accuracy, rnn_precision, rnn_recall, rnn_specificity, f1_score = rnn_metrics(pred6_5, g_testlabel6)\n            print('Validation Accuracy: {:.4f}'.format(rnn_accuracy))\n            print('Validation Precision: {:.4f}'.format(rnn_precision))\n            print('Validation Recall: {:.4f}'.format(rnn_recall))\n            print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T21:31:30.260356Z","iopub.execute_input":"2023-05-23T21:31:30.261431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"## BERT and CodeBERT Sentence Prediction","metadata":{}},{"cell_type":"markdown","source":"#### Load Tokenizer","metadata":{}},{"cell_type":"code","source":"%%time\n#Load the BERT tokenizer.\n\nb_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\ncb_tokenizer = AutoTokenizer.from_pretrained(\"microsoft/codebert-base\")","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:28:43.475594Z","iopub.execute_input":"2023-05-11T20:28:43.478292Z","iopub.status.idle":"2023-05-11T20:28:45.761907Z","shell.execute_reply.started":"2023-05-11T20:28:43.478243Z","shell.execute_reply":"2023-05-11T20:28:45.760518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Benchmark","metadata":{}},{"cell_type":"markdown","source":"#### Load Dataset, Data Preprocessing and Model Training and Evaluation","metadata":{}},{"cell_type":"code","source":"%%time\n#Dataset Importation\ncodes1  = pd.read_csv(\"/kaggle/input/sourcesniffer/SourceSniffer/Benchmark/Benchmark_Raw_Data_2.csv\", encoding='latin-1')\ncodes1.drop(['Unnamed: 0','Index'], axis=1, inplace=True)\ncodes1.head(1)\n\n\n#Dataset Preprocessing (Cleaning)\ncodes1['Code and Comment'] = codes1['Code and Comment'].str.lower()\ncodes1['Code and Comment'] = codes1['Code and Comment'].apply(clean_code)\ncodes1.head(1)\n\n#Dataset Preprocessing (Assigning non-null entries to new variable)\ncoh_data1 = codes1[[\"Code and Comment\",\"Label\"]][~codes1['Label'].isnull()]\ncoh_data1.shape\n\n#Dataset Preprocessing (Label Encoding)\nle = preprocessing.LabelEncoder()\ncoh_stat1 = le.fit_transform(coh_data1['Label'])\n\ncode_comm1 = coh_data1['Code and Comment']\n\n#Identify which label numerical code correspond to which categories\nprint(collections.Counter(coh_stat1), collections.Counter(coh_data1['Label']))","metadata":{"execution":{"iopub.status.busy":"2023-05-11T19:18:11.02458Z","iopub.execute_input":"2023-05-11T19:18:11.02534Z","iopub.status.idle":"2023-05-11T19:18:11.145767Z","shell.execute_reply.started":"2023-05-11T19:18:11.025297Z","shell.execute_reply":"2023-05-11T19:18:11.144546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### BERT","metadata":{}},{"cell_type":"code","source":"%%time\n#BERT\n#Dataset Preprocessing (Code-Comment Vector-Encoding)\n\nb_token_id1 = []\nb_attention_masks1 = []\n\nfor sample in code_comm1:\n    b_encoding_dict1 = bert_preprocessing(sample, b_tokenizer)\n    b_token_id1.append(b_encoding_dict1['input_ids']) \n    b_attention_masks1.append(b_encoding_dict1['attention_mask'])\n\nb_token_id1 = torch.cat(b_token_id1, dim = 0)\nb_attention_masks1 = torch.cat(b_attention_masks1, dim = 0)\nb_labels1 = torch.tensor(coh_stat1)\n\n#BERT\n#Splitting Dataset into training and validation set, and loading into batches.\nb_val_ratio1 = 0.2\n# Recommended batch size: 16, 32. See: https://arxiv.org/pdf/1810.04805.pdf\nb_batch_size1 = 16\n\n# Indices of the train and validation splits stratified by labels\nb_train_idx1, b_val_idx1 = train_test_split(\n    np.arange(len(b_labels1)),\n    test_size = b_val_ratio1,\n    shuffle = True,\n    stratify = b_labels1)\n\n# Train and validation sets\nb_train_set1 = TensorDataset(b_token_id1[b_train_idx1], \n                          b_attention_masks1[b_train_idx1], \n                          b_labels1[b_train_idx1])\n\nb_val_set1 = TensorDataset(b_token_id1[b_val_idx1], \n                        b_attention_masks1[b_val_idx1], \n                        b_labels1[b_val_idx1])\n\n# Prepare DataLoader\nb_train_dataloader1 = DataLoader(\n            b_train_set1,\n            sampler = RandomSampler(b_train_set1),\n            batch_size = b_batch_size1)\n\nb_validation_dataloader1 = DataLoader(\n            b_val_set1,\n            sampler = SequentialSampler(b_val_set1),\n            batch_size = b_batch_size1)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T19:18:11.148349Z","iopub.execute_input":"2023-05-11T19:18:11.149431Z","iopub.status.idle":"2023-05-11T19:18:26.531304Z","shell.execute_reply.started":"2023-05-11T19:18:11.149383Z","shell.execute_reply":"2023-05-11T19:18:26.530046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#BERT\n#Model Object Initiation\n#Load the BertForSequenceClassification model\nb_model1 = BertForSequenceClassification.from_pretrained(\n    'bert-base-uncased',\n    num_labels = 2,\n    output_attentions = False,\n    output_hidden_states = False,)\n\n#Recommended learning rates (Adam): 5e-5, 3e-5, 2e-5. See: https://arxiv.org/pdf/1810.04805.pdf\nb_optimizer1 = torch.optim.AdamW(b_model1.parameters(), \n                              lr = 5e-5,\n                              eps = 1e-08)\n\n#Run on GPU\nb_model1.cuda()\n\n#BERT\n#Model Training and Evaluation\n# Recommended number of epochs: 2, 3, 4. See: https://arxiv.org/pdf/1810.04805.pdf\nb_epochs1 = 3\n\nfor _ in trange(b_epochs1, desc = 'Epoch'):\n    \n    # ========== Training ==========\n    # Set model to training mode\n    b_model1.train()\n    \n    # Tracking variables\n    b_tr_loss1 = 0\n    nb_tr_examples1, nb_tr_steps1 = 0, 0\n\n    for step, batch in enumerate(b_train_dataloader1):\n        batch = tuple(t.to(device) for t in batch)\n        b_input_ids1, b_input_mask1, b_labels1 = batch\n        b_optimizer1.zero_grad()\n        # Forward pass\n        b_train_output1 = b_model1(b_input_ids1, \n                             token_type_ids = None, \n                             attention_mask = b_input_mask1, \n                             labels = b_labels1)\n        # Backward pass\n        b_train_output1.loss.backward()\n        b_optimizer1.step()\n        # Update tracking variables\n        b_tr_loss1 += b_train_output1.loss.item()\n        nb_tr_examples1 += b_input_ids1.size(0)\n        nb_tr_steps1 += 1\n\n    # ========== Validation ==========\n    # Set model to evaluation mode\n    b_model1.eval()\n\n    # Tracking variables \n    b_val_accuracy1, b_val_precision1, b_val_recall1, b_val_specificity1, f1_val_score1 = [], [], [], [], []\n\n    for batch in b_validation_dataloader1:\n        batch = tuple(t.to(device) for t in batch)\n        b_input_ids1, b_input_mask1, b_labels1 = batch\n        with torch.no_grad():\n          # Forward pass\n          b_eval_output1 = b_model1(b_input_ids1, \n                              token_type_ids = None, \n                              attention_mask = b_input_mask1)\n        logits1 = b_eval_output1.logits.detach().cpu().numpy()\n        label_ids1 = b_labels1.to('cpu').numpy()\n        # Calculate validation metrics\n        b_accuracy1, b_precision1, b_recall1, b_specificity1, f1_score1 = b_metrics(logits1, label_ids1)\n        b_val_accuracy1.append(b_accuracy1)\n        # Update precision only when (tp + fp) !=0; ignore nan\n        if b_precision1 != 'nan': b_val_precision1.append(b_precision1)\n        # Update recall only when (tp + fn) !=0; ignore nan\n        if b_recall1 != 'nan': b_val_recall1.append(b_recall1)\n        # Update specificity only when (tn + fp) !=0; ignore nan\n        if b_specificity1 != 'nan': b_val_specificity1.append(b_specificity1)\n        # Update f1_score only when recall and specificity != nan; ignore nan\n        if f1_score1 != 'nan': f1_val_score1.append(f1_score1)\n\n    print('\\n\\t - Train loss: {:.4f}'.format(b_tr_loss1 / nb_tr_steps1))\n    print('\\t - Validation Accuracy: {:.4f}'.format(sum(b_val_accuracy1)/len(b_val_accuracy1)))\n    print('\\t - Validation Precision: {:.4f}'.format(sum(b_val_precision1)/len(b_val_precision1)) if len(b_val_precision1)>0 else '\\t - Validation Precision: NaN')\n    print('\\t - Validation Recall: {:.4f}'.format(sum(b_val_recall1)/len(b_val_recall1)) if len(b_val_recall1)>0 else '\\t - Validation Recall: NaN')\n    print('\\t - Validation Specificity: {:.4f}'.format(sum(b_val_specificity1)/len(b_val_specificity1)) if len(b_val_specificity1)>0 else '\\t - Validation Specificity: NaN')\n    print('\\t - F1 Score: {:.4f}\\n'.format(sum(f1_val_score1)/len(f1_val_score1)) if len(f1_val_score1)>0 else '\\t - F1 Score: NaN')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T19:27:32.199834Z","iopub.execute_input":"2023-05-11T19:27:32.200303Z","iopub.status.idle":"2023-05-11T19:30:00.796591Z","shell.execute_reply.started":"2023-05-11T19:27:32.200265Z","shell.execute_reply":"2023-05-11T19:30:00.795333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### CodeBERT","metadata":{}},{"cell_type":"code","source":"%%time\n#CodeBERT\n#Dataset Preprocessing (Code-Comment Vector-Encoding)\ncb_token_id1 = []\ncb_attention_masks1 = []\n\nfor sample in code_comm1:\n    cb_encoding_dict1 = codebert_preprocessing(sample, cb_tokenizer)\n    cb_token_id1.append(cb_encoding_dict1['input_ids']) \n    cb_attention_masks1.append(cb_encoding_dict1['attention_mask'])\n\ncb_token_id1 = torch.cat(cb_token_id1, dim = 0)\ncb_attention_masks1 = torch.cat(cb_attention_masks1, dim = 0)\ncb_labels1 = torch.tensor(coh_stat1)\n\n#CodeBERT\n#Splitting Dataset into training and validation set, and loading into batches.\ncb_val_ratio1 = 0.2\n#Recommended batch size: 16, 32. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_batch_size1 = 16\n\n#Indices of the train and validation splits stratified by labels\ncb_train_idx1, cb_val_idx1 = train_test_split(\n    np.arange(len(cb_labels1)),\n    test_size = cb_val_ratio1,\n    shuffle = True,\n    stratify = cb_labels1)\n\n#Train and validation sets\ncb_train_set1 = TensorDataset(cb_token_id1[cb_train_idx1], \n                          cb_attention_masks1[cb_train_idx1], \n                          cb_labels1[cb_train_idx1])\n\ncb_val_set1 = TensorDataset(cb_token_id1[cb_val_idx1], \n                        cb_attention_masks1[cb_val_idx1], \n                        cb_labels1[cb_val_idx1])\n\n#Prepare DataLoader\ncb_train_dataloader1 = DataLoader(\n            cb_train_set1,\n            sampler = RandomSampler(cb_train_set1),\n            batch_size = cb_batch_size1)\n\ncb_validation_dataloader1 = DataLoader(\n            cb_val_set1,\n            sampler = SequentialSampler(cb_val_set1),\n            batch_size = cb_batch_size1)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T19:31:54.5183Z","iopub.execute_input":"2023-05-11T19:31:54.519266Z","iopub.status.idle":"2023-05-11T19:31:57.093787Z","shell.execute_reply.started":"2023-05-11T19:31:54.519222Z","shell.execute_reply":"2023-05-11T19:31:57.092674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#CodeBERT\n#Model Object Initiation\n# Load the RobertaForSequenceClassification model\ncb_model1 = AutoModelForSequenceClassification.from_pretrained(\"microsoft/codebert-base\")\n\n# Recommended learning rates (Adam): 5e-5, 3e-5, 2e-5. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_optimizer1 = torch.optim.AdamW(cb_model1.parameters(), \n                              lr = 5e-5,\n                              eps = 1e-08)\n\n# Run on GPU\ncb_model1.cuda()\n\n#CodeBERT\n#Model Training and Evaluation\n#Recommended number of epochs: 2, 3, 4. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_epochs1 = 3\n\nfor _ in trange(cb_epochs1, desc = 'Epoch'):\n    \n    # ========== Training ==========\n    # Set model to training mode\n    cb_model1.train()\n    \n    # Tracking variables\n    cb_tr_loss1 = 0\n    nb_tr_examples1, nb_tr_steps1 = 0, 0\n\n    for step, batch in enumerate(cb_train_dataloader1):\n        batch = tuple(t.to(device) for t in batch)\n        cb_input_ids1, cb_input_mask1, cb_labels1 = batch\n        cb_optimizer1.zero_grad()\n        # Forward pass\n        cb_train_output1 = cb_model1(cb_input_ids1, \n                             token_type_ids = None, \n                             attention_mask = cb_input_mask1, \n                             labels = cb_labels1)\n        # Backward pass\n        cb_train_output1.loss.backward()\n        cb_optimizer1.step()\n        # Update tracking variables\n        cb_tr_loss1 += cb_train_output1.loss.item()\n        nb_tr_examples1 += cb_input_ids1.size(0)\n        nb_tr_steps1 += 1\n\n    # ========== Validation ==========\n    # Set model to evaluation mode\n    cb_model1.eval()\n\n    # Tracking variables \n    cb_val_accuracy1, cb_val_precision1, cb_val_recall1, cb_val_specificity1, cf1_val_score1 = [], [], [], [], []\n\n    for batch in cb_validation_dataloader1:\n        batch = tuple(t.to(device) for t in batch)\n        cb_input_ids1, cb_input_mask1, cb_labels1 = batch\n        with torch.no_grad():\n          # Forward pass\n          cb_eval_output1 = cb_model1(cb_input_ids1, \n                              token_type_ids = None, \n                              attention_mask = cb_input_mask1)\n        logits1 = cb_eval_output1.logits.detach().cpu().numpy()\n        label_ids1 = cb_labels1.to('cpu').numpy()\n        # Calculate validation metrics\n        cb_accuracy1, cb_precision1, cb_recall1, cb_specificity1, cf1_score1 = b_metrics(logits1, label_ids1)\n        cb_val_accuracy1.append(cb_accuracy1)\n        # Update precision only when (tp + fp) !=0; ignore nan\n        if cb_precision1 != 'nan': cb_val_precision1.append(cb_precision1)\n        # Update recall only when (tp + fn) !=0; ignore nan\n        if cb_recall1 != 'nan': cb_val_recall1.append(cb_recall1)\n        # Update specificity only when (tn + fp) !=0; ignore nan\n        if cb_specificity1 != 'nan': cb_val_specificity1.append(cb_specificity1)\n        # Update f1_score only when recall and specificity != nan; ignore nan\n        if cf1_score1 != 'nan': cf1_val_score1.append(cf1_score1)\n\n    print('\\n\\t - Train loss: {:.4f}'.format(cb_tr_loss1 / nb_tr_steps1))\n    print('\\t - Validation Accuracy: {:.4f}'.format(sum(cb_val_accuracy1)/len(cb_val_accuracy1)))\n    print('\\t - Validation Precision: {:.4f}'.format(sum(cb_val_precision1)/len(cb_val_precision1)) if len(cb_val_precision1)>0 else '\\t - Validation Precision: NaN')\n    print('\\t - Validation Recall: {:.4f}'.format(sum(cb_val_recall1)/len(cb_val_recall1)) if len(cb_val_recall1)>0 else '\\t - Validation Recall: NaN')\n    print('\\t - Validation Specificity: {:.4f}'.format(sum(cb_val_specificity1)/len(cb_val_specificity1)) if len(cb_val_specificity1)>0 else '\\t - Validation Specificity: NaN')\n    print('\\t - F1 Score: {:.4f}\\n'.format(sum(cf1_val_score1)/len(cf1_val_score1)) if len(cf1_val_score1)>0 else '\\t - F1 Score: NaN')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T19:32:03.599579Z","iopub.execute_input":"2023-05-11T19:32:03.599965Z","iopub.status.idle":"2023-05-11T19:34:37.65767Z","shell.execute_reply.started":"2023-05-11T19:32:03.599928Z","shell.execute_reply":"2023-05-11T19:34:37.656279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CoffeeMaker","metadata":{}},{"cell_type":"markdown","source":"#### Load Dataset, Data Preprocessing and Model Training and Evaluation","metadata":{}},{"cell_type":"code","source":"%%time\n#Dataset Importation\ncodes2  = pd.read_csv(\"/kaggle/input/sourcesniffer/SourceSniffer/CoffeeMaker/CoffeeMaker_Raw_Data_2.csv\", encoding='latin-1')\ncodes2.drop(['Unnamed: 0','Index'], axis=1, inplace=True)\n#codes2.head(1)\n\n#Dataset Preprocessing (Cleaning)\ncodes2['Code and Comment'] = codes2['Code and Comment'].str.lower()\ncodes2['Code and Comment'] = codes2['Code and Comment'].apply(clean_code)\n#codes2.head(1)\n\n#Dataset Preprocessing (Assigning non-null entries to new variable)\ncoh_data2 = codes2[[\"Code and Comment\",\"Label\"]][~codes2['Label'].isnull()]\n#coh_data2.shape\n\n#Dataset Preprocessing (Label Encoding)\nle = preprocessing.LabelEncoder()\ncoh_stat2 = le.fit_transform(coh_data2['Label'])\ncode_comm2 = coh_data2['Code and Comment']\n\n#Identify which label numerical code correspond to which categories\nprint(collections.Counter(coh_stat2), collections.Counter(coh_data2['Label']))","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:00:43.351216Z","iopub.execute_input":"2023-05-11T20:00:43.351947Z","iopub.status.idle":"2023-05-11T20:00:43.417272Z","shell.execute_reply.started":"2023-05-11T20:00:43.351907Z","shell.execute_reply":"2023-05-11T20:00:43.416226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### BERT","metadata":{}},{"cell_type":"code","source":"%%time\n#BERT\n#Dataset Preprocessing (Code-Comment Vector-Encoding)\n\nb_token_id2 = []\nb_attention_masks2 = []\n\nfor sample in code_comm2:\n    b_encoding_dict2 = bert_preprocessing(sample, b_tokenizer)\n    b_token_id2.append(b_encoding_dict2['input_ids']) \n    b_attention_masks2.append(b_encoding_dict2['attention_mask'])\n\nb_token_id2 = torch.cat(b_token_id2, dim = 0)\nb_attention_masks2 = torch.cat(b_attention_masks2, dim = 0)\nb_labels2 = torch.tensor(coh_stat2)\n\n#BERT\n#Splitting Dataset into training and validation set, and loading into batches.\nb_val_ratio2 = 0.2\n# Recommended batch size: 16, 32. See: https://arxiv.org/pdf/1810.04805.pdf\nb_batch_size2 = 4\n\n# Indices of the train and validation splits stratified by labels\nb_train_idx2, b_val_idx2 = train_test_split(\n    np.arange(len(b_labels2)),\n    test_size = b_val_ratio2,\n    shuffle = True,\n    stratify = b_labels2)\n\n# Train and validation sets\nb_train_set2 = TensorDataset(b_token_id2[b_train_idx2], \n                          b_attention_masks2[b_train_idx2], \n                          b_labels2[b_train_idx2])\n\nb_val_set2 = TensorDataset(b_token_id2[b_val_idx2], \n                        b_attention_masks2[b_val_idx2], \n                        b_labels2[b_val_idx2])\n\n# Prepare DataLoader\nb_train_dataloader2 = DataLoader(\n            b_train_set2,\n            sampler = RandomSampler(b_train_set2),\n            batch_size = b_batch_size2)\n\nb_validation_dataloader2 = DataLoader(\n            b_val_set2,\n            sampler = SequentialSampler(b_val_set2),\n            batch_size = b_batch_size2)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:00:50.123472Z","iopub.execute_input":"2023-05-11T20:00:50.123902Z","iopub.status.idle":"2023-05-11T20:00:50.399394Z","shell.execute_reply.started":"2023-05-11T20:00:50.123862Z","shell.execute_reply":"2023-05-11T20:00:50.398196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#BERT\n#Model Object Initiation\n#Load the BertForSequenceClassification model\nb_model2 = BertForSequenceClassification.from_pretrained(\n    'bert-base-uncased',\n    num_labels = 2,\n    output_attentions = False,\n    output_hidden_states = False,)\n\n#Recommended learning rates (Adam): 5e-5, 3e-5, 2e-5. See: https://arxiv.org/pdf/1810.04805.pdf\nb_optimizer2 = torch.optim.AdamW(b_model2.parameters(), \n                              lr = 5e-5,\n                              eps = 1e-08)\n\n#Run on GPU\nb_model2.cuda()\n\n#Model Training and Evaluation\n# Recommended number of epochs: 2, 3, 4. See: https://arxiv.org/pdf/1810.04805.pdf\nb_epochs2 = 3\n\nfor _ in trange(b_epochs2, desc = 'Epoch'):\n    \n    # ========== Training ==========\n    # Set model to training mode\n    b_model2.train()\n    \n    # Tracking variables\n    b_tr_loss2 = 0\n    nb_tr_examples2, nb_tr_steps2 = 0, 0\n\n    for step, batch in enumerate(b_train_dataloader2):\n        batch = tuple(t.to(device) for t in batch)\n        b_input_ids2, b_input_mask2, b_labels2 = batch\n        b_optimizer2.zero_grad()\n        # Forward pass\n        b_train_output2 = b_model2(b_input_ids2, \n                             token_type_ids = None, \n                             attention_mask = b_input_mask2, \n                             labels = b_labels2)\n        # Backward pass\n        b_train_output2.loss.backward()\n        b_optimizer2.step()\n        # Update tracking variables\n        b_tr_loss2 += b_train_output2.loss.item()\n        nb_tr_examples2 += b_input_ids2.size(0)\n        nb_tr_steps2 += 1\n\n    # ========== Validation ==========\n    # Set model to evaluation mode\n    b_model2.eval()\n\n    # Tracking variables \n    b_val_accuracy2, b_val_precision2, b_val_recall2, b_val_specificity2, f1_val_score2 = [], [], [], [], []\n\n    for batch in b_validation_dataloader2:\n        batch = tuple(t.to(device) for t in batch)\n        b_input_ids2, b_input_mask2, b_labels2 = batch\n        with torch.no_grad():\n          # Forward pass\n          b_eval_output2 = b_model2(b_input_ids2, \n                              token_type_ids = None, \n                              attention_mask = b_input_mask2)\n        logits2 = b_eval_output2.logits.detach().cpu().numpy()\n        label_ids2 = b_labels2.to('cpu').numpy()\n        # Calculate validation metrics\n        b_accuracy2, b_precision2, b_recall2, b_specificity2, f1_score2 = b_metrics(logits2, label_ids2)\n        b_val_accuracy2.append(b_accuracy2)\n        # Update precision only when (tp + fp) !=0; ignore nan\n        if b_precision2 != 'nan': b_val_precision2.append(b_precision2)\n        # Update recall only when (tp + fn) !=0; ignore nan\n        if b_recall2 != 'nan': b_val_recall2.append(b_recall2)\n        # Update specificity only when (tn + fp) !=0; ignore nan\n        if b_specificity2 != 'nan': b_val_specificity2.append(b_specificity2)\n        # Update f1_score only when recall and specificity != nan; ignore nan\n        if f1_score2 != 'nan': f1_val_score2.append(f1_score2)\n\n    print('\\n\\t - Train loss: {:.4f}'.format(b_tr_loss2 / nb_tr_steps2))\n    print('\\t - Validation Accuracy: {:.4f}'.format(sum(b_val_accuracy2)/len(b_val_accuracy2)))\n    print('\\t - Validation Precision: {:.4f}'.format(sum(b_val_precision2)/len(b_val_precision2)) if len(b_val_precision2)>0 else '\\t - Validation Precision: NaN')\n    print('\\t - Validation Recall: {:.4f}'.format(sum(b_val_recall2)/len(b_val_recall2)) if len(b_val_recall2)>0 else '\\t - Validation Recall: NaN')\n    print('\\t - Validation Specificity: {:.4f}'.format(sum(b_val_specificity2)/len(b_val_specificity2)) if len(b_val_specificity2)>0 else '\\t - Validation Specificity: NaN')\n    print('\\t - F1 Score: {:.4f}\\n'.format(sum(f1_val_score2)/len(f1_val_score2)) if len(f1_val_score2)>0 else '\\t - F1 Score: NaN')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:00:55.3111Z","iopub.execute_input":"2023-05-11T20:00:55.311567Z","iopub.status.idle":"2023-05-11T20:01:06.593777Z","shell.execute_reply.started":"2023-05-11T20:00:55.311528Z","shell.execute_reply":"2023-05-11T20:01:06.592669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### CodeBERT","metadata":{}},{"cell_type":"code","source":"%%time\n#CodeBERT\n#Dataset Preprocessing (Code-Comment Vector-Encoding)\n\ncb_token_id2 = []\ncb_attention_masks2 = []\n\nfor sample in code_comm2:\n    cb_encoding_dict2 = codebert_preprocessing(sample, cb_tokenizer)\n    cb_token_id2.append(cb_encoding_dict2['input_ids']) \n    cb_attention_masks2.append(cb_encoding_dict2['attention_mask'])\n\ncb_token_id2 = torch.cat(cb_token_id2, dim = 0)\ncb_attention_masks2 = torch.cat(cb_attention_masks2, dim = 0)\ncb_labels2 = torch.tensor(coh_stat2)\n\n#CodeBERT\n#Splitting Dataset into training and validation set, and loading into batches.\ncb_val_ratio2 = 0.2\n#Recommended batch size: 16, 32. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_batch_size2 = 4\n\n#Indices of the train and validation splits stratified by labels\ncb_train_idx2, cb_val_idx2 = train_test_split(\n    np.arange(len(cb_labels2)),\n    test_size = cb_val_ratio2,\n    shuffle = True,\n    stratify = cb_labels2)\n\n#Train and validation sets\ncb_train_set2 = TensorDataset(cb_token_id2[cb_train_idx2], \n                          cb_attention_masks2[cb_train_idx2], \n                          cb_labels2[cb_train_idx2])\n\ncb_val_set2 = TensorDataset(cb_token_id2[cb_val_idx2], \n                        cb_attention_masks2[cb_val_idx2], \n                        cb_labels2[cb_val_idx2])\n\n#Prepare DataLoader\ncb_train_dataloader2 = DataLoader(\n            cb_train_set2,\n            sampler = RandomSampler(cb_train_set2),\n            batch_size = cb_batch_size2)\n\ncb_validation_dataloader2 = DataLoader(\n            cb_val_set2,\n            sampler = SequentialSampler(cb_val_set2),\n            batch_size = cb_batch_size2)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:01:31.81791Z","iopub.execute_input":"2023-05-11T20:01:31.818742Z","iopub.status.idle":"2023-05-11T20:01:31.871373Z","shell.execute_reply.started":"2023-05-11T20:01:31.818697Z","shell.execute_reply":"2023-05-11T20:01:31.870062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#CodeBERT\n#Model Object Initiation\n#Load the RobertaForSequenceClassification model\ncb_model2 = AutoModelForSequenceClassification.from_pretrained(\"microsoft/codebert-base\")\n\n#Recommended learning rates (Adam): 5e-5, 3e-5, 2e-5. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_optimizer2 = torch.optim.AdamW(cb_model2.parameters(), \n                              lr = 5e-5,\n                              eps = 1e-08)\n\n#Run on GPU\ncb_model2.cuda()\n\n#Model Training and Evaluation\n#Recommended number of epochs: 2, 3, 4. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_epochs2 = 3\n\nfor _ in trange(cb_epochs2, desc = 'Epoch'):\n    \n    # ========== Training ==========\n    # Set model to training mode\n    cb_model2.train()\n    \n    # Tracking variables\n    cb_tr_loss2 = 0\n    nb_tr_examples2, nb_tr_steps2 = 0, 0\n\n    for step, batch in enumerate(cb_train_dataloader2):\n        batch = tuple(t.to(device) for t in batch)\n        cb_input_ids2, cb_input_mask2, cb_labels2 = batch\n        cb_optimizer2.zero_grad()\n        # Forward pass\n        cb_train_output2 = cb_model2(cb_input_ids2, \n                             token_type_ids = None, \n                             attention_mask = cb_input_mask2, \n                             labels = cb_labels2)\n        # Backward pass\n        cb_train_output2.loss.backward()\n        cb_optimizer2.step()\n        # Update tracking variables\n        cb_tr_loss2 += cb_train_output2.loss.item()\n        nb_tr_examples2 += cb_input_ids2.size(0)\n        nb_tr_steps2 += 1\n\n    # ========== Validation ==========\n    # Set model to evaluation mode\n    cb_model2.eval()\n\n    # Tracking variables \n    cb_val_accuracy2, cb_val_precision2, cb_val_recall2, cb_val_specificity2, cf1_val_score2 = [], [], [], [], []\n\n    for batch in cb_validation_dataloader2:\n        batch = tuple(t.to(device) for t in batch)\n        cb_input_ids2, cb_input_mask2, cb_labels2 = batch\n        with torch.no_grad():\n          # Forward pass\n          cb_eval_output2 = cb_model2(cb_input_ids2, \n                              token_type_ids = None, \n                              attention_mask = cb_input_mask2)\n        logits2 = cb_eval_output2.logits.detach().cpu().numpy()\n        label_ids2 = cb_labels2.to('cpu').numpy()\n        # Calculate validation metrics\n        cb_accuracy2, cb_precision2, cb_recall2, cb_specificity2, cf1_score2 = b_metrics(logits2, label_ids2)\n        cb_val_accuracy2.append(cb_accuracy2)\n        # Update precision only when (tp + fp) !=0; ignore nan\n        if cb_precision2 != 'nan': cb_val_precision2.append(cb_precision2)\n        # Update recall only when (tp + fn) !=0; ignore nan\n        if cb_recall2 != 'nan': cb_val_recall2.append(cb_recall2)\n        # Update specificity only when (tn + fp) !=0; ignore nan\n        if cb_specificity2 != 'nan': cb_val_specificity2.append(cb_specificity2)\n        # Update f1_score only when recall and specificity != nan; ignore nan\n        if cf1_score2 != 'nan': cf1_val_score2.append(cf1_score2)\n\n    print('\\n\\t - Train loss: {:.4f}'.format(cb_tr_loss2 / nb_tr_steps2))\n    print('\\t - Validation Accuracy: {:.4f}'.format(sum(cb_val_accuracy2)/len(cb_val_accuracy2)))\n    print('\\t - Validation Precision: {:.4f}'.format(sum(cb_val_precision2)/len(cb_val_precision2)) if len(cb_val_precision2)>0 else '\\t - Validation Precision: NaN')\n    print('\\t - Validation Recall: {:.4f}'.format(sum(cb_val_recall2)/len(cb_val_recall2)) if len(cb_val_recall2)>0 else '\\t - Validation Recall: NaN')\n    print('\\t - Validation Specificity: {:.4f}'.format(sum(cb_val_specificity2)/len(cb_val_specificity2)) if len(cb_val_specificity2)>0 else '\\t - Validation Specificity: NaN')\n    print('\\t - F1 Score: {:.4f}\\n'.format(sum(cf1_val_score2)/len(cf1_val_score2)) if len(cf1_val_score2)>0 else '\\t - F1 Score: NaN')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:01:36.110069Z","iopub.execute_input":"2023-05-11T20:01:36.11057Z","iopub.status.idle":"2023-05-11T20:01:47.726883Z","shell.execute_reply.started":"2023-05-11T20:01:36.110523Z","shell.execute_reply":"2023-05-11T20:01:47.725507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### JFreeChart060","metadata":{}},{"cell_type":"markdown","source":"#### Load Dataset, Data Preprocessing and Model Training and Evaluation","metadata":{}},{"cell_type":"code","source":"%%time\n#Dataset Importation\ncodes3  = pd.read_csv(\"/kaggle/input/sourcesniffer/SourceSniffer/JFreeChart060/JFreeChart060_Raw_Data_2.csv\", encoding='latin-1')\ncodes3.drop(['Unnamed: 0','Index'], axis=1, inplace=True)\n#codes3.head(1)\n\n#Dataset Preprocessing (Cleaning)\ncodes3['Code and Comment'] = codes3['Code and Comment'].str.lower()\ncodes3['Code and Comment'] = codes3['Code and Comment'].apply(clean_code)\n#codes3.head(1)\n\n#Dataset Preprocessing (Assigning non-null entries to new variable)\ncoh_data3 = codes3[[\"Code and Comment\",\"Label\"]][~codes3['Label'].isnull()]\n#coh_data3.shape\n\n#Dataset Preprocessing (Label Encoding)\nle = preprocessing.LabelEncoder()\ncoh_stat3 = le.fit_transform(coh_data3['Label'])\ncode_comm3 = coh_data3['Code and Comment']\n\n#Identify which label numerical code correspond to which categories\nprint(collections.Counter(coh_stat3), collections.Counter(coh_data3['Label']))\n","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:07:43.953933Z","iopub.execute_input":"2023-05-11T20:07:43.954367Z","iopub.status.idle":"2023-05-11T20:07:44.009349Z","shell.execute_reply.started":"2023-05-11T20:07:43.954329Z","shell.execute_reply":"2023-05-11T20:07:44.008031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### BERT","metadata":{}},{"cell_type":"code","source":"%%time\n#BERT\n#Dataset Preprocessing (Code-Comment Vector-Encoding)\n\nb_token_id3 = []\nb_attention_masks3 = []\n\nfor sample in code_comm3:\n    b_encoding_dict3 = bert_preprocessing(sample, b_tokenizer)\n    b_token_id3.append(b_encoding_dict3['input_ids']) \n    b_attention_masks3.append(b_encoding_dict3['attention_mask'])\n\nb_token_id3 = torch.cat(b_token_id3, dim = 0)\nb_attention_masks3 = torch.cat(b_attention_masks3, dim = 0)\nb_labels3 = torch.tensor(coh_stat3)\n\n#BERT\n#Splitting Dataset into training and validation set, and loading into batches.\nb_val_ratio3 = 0.2\n# Recommended batch size: 16, 32. See: https://arxiv.org/pdf/1810.04805.pdf\nb_batch_size3 = 16\n\n# Indices of the train and validation splits stratified by labels\nb_train_idx3, b_val_idx3 = train_test_split(\n    np.arange(len(b_labels3)),\n    test_size = b_val_ratio3,\n    shuffle = True,\n    stratify = b_labels3)\n\n# Train and validation sets\nb_train_set3 = TensorDataset(b_token_id3[b_train_idx3], \n                          b_attention_masks3[b_train_idx3], \n                          b_labels3[b_train_idx3])\n\nb_val_set3 = TensorDataset(b_token_id3[b_val_idx3], \n                        b_attention_masks3[b_val_idx3], \n                        b_labels3[b_val_idx3])\n\n# Prepare DataLoader\nb_train_dataloader3 = DataLoader(\n            b_train_set3,\n            sampler = RandomSampler(b_train_set3),\n            batch_size = b_batch_size3)\n\nb_validation_dataloader3 = DataLoader(\n            b_val_set3,\n            sampler = SequentialSampler(b_val_set3),\n            batch_size = b_batch_size3)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:07:49.292089Z","iopub.execute_input":"2023-05-11T20:07:49.292745Z","iopub.status.idle":"2023-05-11T20:07:52.071762Z","shell.execute_reply.started":"2023-05-11T20:07:49.292686Z","shell.execute_reply":"2023-05-11T20:07:52.07046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#BERT\n#Model Object Initiation\n#Load the BertForSequenceClassification model\nb_model3 = BertForSequenceClassification.from_pretrained(\n    'bert-base-uncased',\n    num_labels = 2,\n    output_attentions = False,\n    output_hidden_states = False,)\n\n#Recommended learning rates (Adam): 5e-5, 3e-5, 2e-5. See: https://arxiv.org/pdf/1810.04805.pdf\nb_optimizer3 = torch.optim.AdamW(b_model3.parameters(), \n                              lr = 5e-5,\n                              eps = 1e-08)\n\n#Run on GPU\nb_model3.cuda()\n\n#Model Training and Evaluation\n# Recommended number of epochs: 2, 3, 4. See: https://arxiv.org/pdf/1810.04805.pdf\nb_epochs3 = 3\n\nfor _ in trange(b_epochs3, desc = 'Epoch'):\n    \n    # ========== Training ==========\n    # Set model to training mode\n    b_model3.train()\n    \n    # Tracking variables\n    b_tr_loss3 = 0\n    nb_tr_examples3, nb_tr_steps3 = 0, 0\n\n    for step, batch in enumerate(b_train_dataloader3):\n        batch = tuple(t.to(device) for t in batch)\n        b_input_ids3, b_input_mask3, b_labels3 = batch\n        b_optimizer3.zero_grad()\n        # Forward pass\n        b_train_output3 = b_model3(b_input_ids3, \n                             token_type_ids = None, \n                             attention_mask = b_input_mask3, \n                             labels = b_labels3)\n        # Backward pass\n        b_train_output3.loss.backward()\n        b_optimizer3.step()\n        # Update tracking variables\n        b_tr_loss3 += b_train_output3.loss.item()\n        nb_tr_examples3 += b_input_ids3.size(0)\n        nb_tr_steps3 += 1\n\n    # ========== Validation ==========\n    # Set model to evaluation mode\n    b_model3.eval()\n\n    # Tracking variables \n    b_val_accuracy3, b_val_precision3, b_val_recall3, b_val_specificity3, f1_val_score3 = [], [], [], [], []\n\n    for batch in b_validation_dataloader3:\n        batch = tuple(t.to(device) for t in batch)\n        b_input_ids3, b_input_mask3, b_labels3 = batch\n        with torch.no_grad():\n          # Forward pass\n          b_eval_output3 = b_model3(b_input_ids3, \n                              token_type_ids = None, \n                              attention_mask = b_input_mask3)\n        logits3 = b_eval_output3.logits.detach().cpu().numpy()\n        label_ids3 = b_labels3.to('cpu').numpy()\n        # Calculate validation metrics\n        b_accuracy3, b_precision3, b_recall3, b_specificity3, f1_score3 = b_metrics(logits3, label_ids3)\n        b_val_accuracy3.append(b_accuracy3)\n        # Update precision only when (tp + fp) !=0; ignore nan\n        if b_precision3 != 'nan': b_val_precision3.append(b_precision3)\n        # Update recall only when (tp + fn) !=0; ignore nan\n        if b_recall3 != 'nan': b_val_recall3.append(b_recall3)\n        # Update specificity only when (tn + fp) !=0; ignore nan\n        if b_specificity3 != 'nan': b_val_specificity3.append(b_specificity3)\n        # Update f1_score only when recall and specificity != nan; ignore nan\n        if f1_score3 != 'nan': f1_val_score3.append(f1_score3)\n\n    print('\\n\\t - Train loss: {:.4f}'.format(b_tr_loss3 / nb_tr_steps3))\n    print('\\t - Validation Accuracy: {:.4f}'.format(sum(b_val_accuracy3)/len(b_val_accuracy3)))\n    print('\\t - Validation Precision: {:.4f}'.format(sum(b_val_precision3)/len(b_val_precision3)) if len(b_val_precision3)>0 else '\\t - Validation Precision: NaN')\n    print('\\t - Validation Recall: {:.4f}'.format(sum(b_val_recall3)/len(b_val_recall3)) if len(b_val_recall3)>0 else '\\t - Validation Recall: NaN')\n    print('\\t - Validation Specificity: {:.4f}'.format(sum(b_val_specificity3)/len(b_val_specificity3)) if len(b_val_specificity3)>0 else '\\t - Validation Specificity: NaN')\n    print('\\t - F1 Score: {:.4f}\\n'.format(sum(f1_val_score3)/len(f1_val_score3)) if len(f1_val_score3)>0 else '\\t - F1 Score: NaN')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:07:56.300075Z","iopub.execute_input":"2023-05-11T20:07:56.300561Z","iopub.status.idle":"2023-05-11T20:08:25.716719Z","shell.execute_reply.started":"2023-05-11T20:07:56.300521Z","shell.execute_reply":"2023-05-11T20:08:25.715533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### CodeBERT","metadata":{}},{"cell_type":"code","source":"%%time\n#CodeBERT\n#Dataset Preprocessing (Code-Comment Vector-Encoding)\n\ncb_token_id3 = []\ncb_attention_masks3 = []\n\nfor sample in code_comm3:\n    cb_encoding_dict3 = codebert_preprocessing(sample, cb_tokenizer)\n    cb_token_id3.append(cb_encoding_dict3['input_ids']) \n    cb_attention_masks3.append(cb_encoding_dict3['attention_mask'])\n\ncb_token_id3 = torch.cat(cb_token_id3, dim = 0)\ncb_attention_masks3 = torch.cat(cb_attention_masks3, dim = 0)\ncb_labels3 = torch.tensor(coh_stat3)\n\n#CodeBERT\n#Splitting Dataset into training and validation set, and loading into batches.\ncb_val_ratio3 = 0.2\n#Recommended batch size: 16, 32. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_batch_size3 = 16\n\n#Indices of the train and validation splits stratified by labels\ncb_train_idx3, cb_val_idx3 = train_test_split(\n    np.arange(len(cb_labels3)),\n    test_size = cb_val_ratio3,\n    shuffle = True,\n    stratify = cb_labels3)\n\n#Train and validation sets\ncb_train_set3 = TensorDataset(cb_token_id3[cb_train_idx3], \n                          cb_attention_masks3[cb_train_idx3], \n                          cb_labels3[cb_train_idx3])\n\ncb_val_set3 = TensorDataset(cb_token_id3[cb_val_idx3], \n                        cb_attention_masks3[cb_val_idx3], \n                        cb_labels3[cb_val_idx3])\n\n#Prepare DataLoader\ncb_train_dataloader3 = DataLoader(\n            cb_train_set3,\n            sampler = RandomSampler(cb_train_set3),\n            batch_size = cb_batch_size3)\n\ncb_validation_dataloader3 = DataLoader(\n            cb_val_set3,\n            sampler = SequentialSampler(cb_val_set3),\n            batch_size = cb_batch_size3)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:08:37.233609Z","iopub.execute_input":"2023-05-11T20:08:37.234812Z","iopub.status.idle":"2023-05-11T20:08:37.590328Z","shell.execute_reply.started":"2023-05-11T20:08:37.234752Z","shell.execute_reply":"2023-05-11T20:08:37.589007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#CodeBERT\n#Model Object Initiation\n#Load the RobertaForSequenceClassification model\ncb_model3 = AutoModelForSequenceClassification.from_pretrained(\"microsoft/codebert-base\")\n\n#Recommended learning rates (Adam): 5e-5, 3e-5, 2e-5. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_optimizer3 = torch.optim.AdamW(cb_model3.parameters(), \n                              lr = 5e-5,\n                              eps = 1e-08)\n\n#Run on GPU\ncb_model3.cuda()\n\n#Model Training and Evaluation\n#Recommended number of epochs: 2, 3, 4. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_epochs3 = 3\n\nfor _ in trange(cb_epochs3, desc = 'Epoch'):\n    \n    # ========== Training ==========\n    # Set model to training mode\n    cb_model3.train()\n    \n    # Tracking variables\n    cb_tr_loss3 = 0\n    nb_tr_examples3, nb_tr_steps3 = 0, 0\n\n    for step, batch in enumerate(cb_train_dataloader3):\n        batch = tuple(t.to(device) for t in batch)\n        cb_input_ids3, cb_input_mask3, cb_labels3 = batch\n        cb_optimizer3.zero_grad()\n        # Forward pass\n        cb_train_output3 = cb_model3(cb_input_ids3, \n                             token_type_ids = None, \n                             attention_mask = cb_input_mask3, \n                             labels = cb_labels3)\n        # Backward pass\n        cb_train_output3.loss.backward()\n        cb_optimizer3.step()\n        # Update tracking variables\n        cb_tr_loss3 += cb_train_output3.loss.item()\n        nb_tr_examples3 += cb_input_ids3.size(0)\n        nb_tr_steps3 += 1\n\n    # ========== Validation ==========\n    # Set model to evaluation mode\n    cb_model3.eval()\n\n    # Tracking variables \n    cb_val_accuracy3, cb_val_precision3, cb_val_recall3, cb_val_specificity3, cf1_val_score3 = [], [], [], [], []\n\n    for batch in cb_validation_dataloader3:\n        batch = tuple(t.to(device) for t in batch)\n        cb_input_ids3, cb_input_mask3, cb_labels3 = batch\n        with torch.no_grad():\n          # Forward pass\n          cb_eval_output3 = cb_model3(cb_input_ids3, \n                              token_type_ids = None, \n                              attention_mask = cb_input_mask3)\n        logits3 = cb_eval_output3.logits.detach().cpu().numpy()\n        label_ids3 = cb_labels3.to('cpu').numpy()\n        # Calculate validation metrics\n        cb_accuracy3, cb_precision3, cb_recall3, cb_specificity3, cf1_score3 = b_metrics(logits3, label_ids3)\n        cb_val_accuracy3.append(cb_accuracy3)\n        # Update precision only when (tp + fp) !=0; ignore nan\n        if cb_precision3 != 'nan': cb_val_precision3.append(cb_precision3)\n        # Update recall only when (tp + fn) !=0; ignore nan\n        if cb_recall3 != 'nan': cb_val_recall3.append(cb_recall3)\n        # Update specificity only when (tn + fp) !=0; ignore nan\n        if cb_specificity3 != 'nan': cb_val_specificity3.append(cb_specificity3)\n        # Update f1_score only when recall and specificity != nan; ignore nan\n        if cf1_score3 != 'nan': cf1_val_score3.append(cf1_score3)\n\n    print('\\n\\t - Train loss: {:.4f}'.format(cb_tr_loss3 / nb_tr_steps3))\n    print('\\t - Validation Accuracy: {:.4f}'.format(sum(cb_val_accuracy3)/len(cb_val_accuracy3)))\n    print('\\t - Validation Precision: {:.4f}'.format(sum(cb_val_precision3)/len(cb_val_precision3)) if len(cb_val_precision3)>0 else '\\t - Validation Precision: NaN')\n    print('\\t - Validation Recall: {:.4f}'.format(sum(cb_val_recall3)/len(cb_val_recall3)) if len(cb_val_recall3)>0 else '\\t - Validation Recall: NaN')\n    print('\\t - Validation Specificity: {:.4f}'.format(sum(cb_val_specificity3)/len(cb_val_specificity3)) if len(cb_val_specificity3)>0 else '\\t - Validation Specificity: NaN')\n    print('\\t - F1 Score: {:.4f}\\n'.format(sum(cf1_val_score3)/len(cf1_val_score3)) if len(cf1_val_score3)>0 else '\\t - F1 Score: NaN')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:08:42.37339Z","iopub.execute_input":"2023-05-11T20:08:42.373877Z","iopub.status.idle":"2023-05-11T20:09:11.280964Z","shell.execute_reply.started":"2023-05-11T20:08:42.373836Z","shell.execute_reply":"2023-05-11T20:09:11.27979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### JFreeChart071","metadata":{}},{"cell_type":"markdown","source":"#### Load Dataset, Data Preprocessing and Model Training and Evaluation","metadata":{}},{"cell_type":"code","source":"%%time\n#Dataset Importation\ncodes4  = pd.read_csv(\"/kaggle/input/sourcesniffer/SourceSniffer/JFreeChart071/JFreeChart071_Raw_Data_2.csv\", encoding='latin-1')\ncodes4.drop(['Unnamed: 0','Index'], axis=1, inplace=True)\n#codes4.head(1)\n\n#Dataset Preprocessing (Cleaning)\ncodes4['Code and Comment'] = codes4['Code and Comment'].str.lower()\ncodes4['Code and Comment'] = codes4['Code and Comment'].apply(clean_code)\n#codes4.head(1)\n\n#Dataset Preprocessing (Assigning non-null entries to new variable)\ncoh_data4 = codes4[[\"Code and Comment\",\"Label\"]][~codes4['Label'].isnull()]\n#coh_data4.shape\n\n#Dataset Preprocessing (Label Encoding)\nle = preprocessing.LabelEncoder()\ncoh_stat4 = le.fit_transform(coh_data4['Label'])\ncode_comm4 = coh_data4['Code and Comment']\n\n#Identify which label numerical code correspond to which categories\nprint(collections.Counter(coh_stat4), collections.Counter(coh_data4['Label']))","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:12:26.437428Z","iopub.execute_input":"2023-05-11T20:12:26.438221Z","iopub.status.idle":"2023-05-11T20:12:26.49826Z","shell.execute_reply.started":"2023-05-11T20:12:26.438148Z","shell.execute_reply":"2023-05-11T20:12:26.497128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### BERT","metadata":{}},{"cell_type":"code","source":"%%time\n#BERT\n#Dataset Preprocessing (Code-Comment Vector-Encoding)\n\nb_token_id4 = []\nb_attention_masks4 = []\n\nfor sample in code_comm4:\n    b_encoding_dict4 = bert_preprocessing(sample, b_tokenizer)\n    b_token_id4.append(b_encoding_dict4['input_ids']) \n    b_attention_masks4.append(b_encoding_dict4['attention_mask'])\n\nb_token_id4 = torch.cat(b_token_id4, dim = 0)\nb_attention_masks4 = torch.cat(b_attention_masks4, dim = 0)\nb_labels4 = torch.tensor(coh_stat4)\n\n#BERT\n#Splitting Dataset into training and validation set, and loading into batches.\nb_val_ratio4 = 0.2\n# Recommended batch size: 16, 32. See: https://arxiv.org/pdf/1810.04805.pdf\nb_batch_size4 = 16\n\n# Indices of the train and validation splits stratified by labels\nb_train_idx4, b_val_idx4 = train_test_split(\n    np.arange(len(b_labels4)),\n    test_size = b_val_ratio4,\n    shuffle = True,\n    stratify = b_labels4)\n\n# Train and validation sets\nb_train_set4 = TensorDataset(b_token_id4[b_train_idx4], \n                          b_attention_masks4[b_train_idx4], \n                          b_labels4[b_train_idx4])\n\nb_val_set4 = TensorDataset(b_token_id4[b_val_idx4], \n                        b_attention_masks4[b_val_idx4], \n                        b_labels4[b_val_idx4])\n\n# Prepare DataLoader\nb_train_dataloader4 = DataLoader(\n            b_train_set4,\n            sampler = RandomSampler(b_train_set4),\n            batch_size = b_batch_size4)\n\nb_validation_dataloader4 = DataLoader(\n            b_val_set4,\n            sampler = SequentialSampler(b_val_set4),\n            batch_size = b_batch_size4)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:12:35.534792Z","iopub.execute_input":"2023-05-11T20:12:35.535191Z","iopub.status.idle":"2023-05-11T20:12:38.247368Z","shell.execute_reply.started":"2023-05-11T20:12:35.535154Z","shell.execute_reply":"2023-05-11T20:12:38.2462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#BERT\n#Model Object Initiation\n#Load the BertForSequenceClassification model\nb_model4 = BertForSequenceClassification.from_pretrained(\n    'bert-base-uncased',\n    num_labels = 2,\n    output_attentions = False,\n    output_hidden_states = False,)\n\n#Recommended learning rates (Adam): 5e-5, 3e-5, 2e-5. See: https://arxiv.org/pdf/1810.04805.pdf\nb_optimizer4 = torch.optim.AdamW(b_model4.parameters(), \n                              lr = 5e-5,\n                              eps = 1e-08)\n\n#Run on GPU\nb_model4.cuda()\n\n#Model Training and Evaluation\n# Recommended number of epochs: 2, 3, 4. See: https://arxiv.org/pdf/1810.04805.pdf\nb_epochs4 = 3\n\nfor _ in trange(b_epochs4, desc = 'Epoch'):\n    \n    # ========== Training ==========\n    # Set model to training mode\n    b_model4.train()\n    \n    # Tracking variables\n    b_tr_loss4 = 0\n    nb_tr_examples4, nb_tr_steps4 = 0, 0\n\n    for step, batch in enumerate(b_train_dataloader4):\n        batch = tuple(t.to(device) for t in batch)\n        b_input_ids4, b_input_mask4, b_labels4 = batch\n        b_optimizer4.zero_grad()\n        # Forward pass\n        b_train_output4 = b_model4(b_input_ids4, \n                             token_type_ids = None, \n                             attention_mask = b_input_mask4, \n                             labels = b_labels4)\n        # Backward pass\n        b_train_output4.loss.backward()\n        b_optimizer4.step()\n        # Update tracking variables\n        b_tr_loss4 += b_train_output4.loss.item()\n        nb_tr_examples4 += b_input_ids4.size(0)\n        nb_tr_steps4 += 1\n\n    # ========== Validation ==========\n    # Set model to evaluation mode\n    b_model4.eval()\n\n    # Tracking variables \n    b_val_accuracy4, b_val_precision4, b_val_recall4, b_val_specificity4, f1_val_score4 = [], [], [], [], []\n\n    for batch in b_validation_dataloader4:\n        batch = tuple(t.to(device) for t in batch)\n        b_input_ids4, b_input_mask4, b_labels4 = batch\n        with torch.no_grad():\n          # Forward pass\n          b_eval_output4 = b_model4(b_input_ids4, \n                              token_type_ids = None, \n                              attention_mask = b_input_mask4)\n        logits4 = b_eval_output4.logits.detach().cpu().numpy()\n        label_ids4 = b_labels4.to('cpu').numpy()\n        # Calculate validation metrics\n        b_accuracy4, b_precision4, b_recall4, b_specificity4, f1_score4 = b_metrics(logits4, label_ids4)\n        b_val_accuracy4.append(b_accuracy4)\n        # Update precision only when (tp + fp) !=0; ignore nan\n        if b_precision4 != 'nan': b_val_precision4.append(b_precision4)\n        # Update recall only when (tp + fn) !=0; ignore nan\n        if b_recall4 != 'nan': b_val_recall4.append(b_recall4)\n        # Update specificity only when (tn + fp) !=0; ignore nan\n        if b_specificity4 != 'nan': b_val_specificity4.append(b_specificity4)\n        # Update f1_score only when recall and specificity != nan; ignore nan\n        if f1_score4 != 'nan': f1_val_score4.append(f1_score4)\n\n    print('\\n\\t - Train loss: {:.4f}'.format(b_tr_loss4 / nb_tr_steps4))\n    print('\\t - Validation Accuracy: {:.4f}'.format(sum(b_val_accuracy4)/len(b_val_accuracy4)))\n    print('\\t - Validation Precision: {:.4f}'.format(sum(b_val_precision4)/len(b_val_precision4)) if len(b_val_precision4)>0 else '\\t - Validation Precision: NaN')\n    print('\\t - Validation Recall: {:.4f}'.format(sum(b_val_recall4)/len(b_val_recall4)) if len(b_val_recall4)>0 else '\\t - Validation Recall: NaN')\n    print('\\t - Validation Specificity: {:.4f}'.format(sum(b_val_specificity4)/len(b_val_specificity4)) if len(b_val_specificity4)>0 else '\\t - Validation Specificity: NaN')\n    print('\\t - F1 Score: {:.4f}\\n'.format(sum(f1_val_score4)/len(f1_val_score4)) if len(f1_val_score4)>0 else '\\t - F1 Score: NaN')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:12:47.759001Z","iopub.execute_input":"2023-05-11T20:12:47.759407Z","iopub.status.idle":"2023-05-11T20:13:22.388831Z","shell.execute_reply.started":"2023-05-11T20:12:47.759367Z","shell.execute_reply":"2023-05-11T20:13:22.387667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### CodeBERT","metadata":{}},{"cell_type":"code","source":"%%time\n#CodeBERT\n#Dataset Preprocessing (Code-Comment Vector-Encoding)\n\ncb_token_id4 = []\ncb_attention_masks4 = []\n\nfor sample in code_comm4:\n    cb_encoding_dict4 = codebert_preprocessing(sample, cb_tokenizer)\n    cb_token_id4.append(cb_encoding_dict4['input_ids']) \n    cb_attention_masks4.append(cb_encoding_dict4['attention_mask'])\n\ncb_token_id4 = torch.cat(cb_token_id4, dim = 0)\ncb_attention_masks4 = torch.cat(cb_attention_masks4, dim = 0)\ncb_labels4 = torch.tensor(coh_stat4)\n\n#CodeBERT\n#Splitting Dataset into training and validation set, and loading into batches.\ncb_val_ratio4 = 0.2\n#Recommended batch size: 16, 32. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_batch_size4 = 16\n\n#Indices of the train and validation splits stratified by labels\ncb_train_idx4, cb_val_idx4 = train_test_split(\n    np.arange(len(cb_labels4)),\n    test_size = cb_val_ratio4,\n    shuffle = True,\n    stratify = cb_labels4)\n\n#Train and validation sets\ncb_train_set4 = TensorDataset(cb_token_id4[cb_train_idx4], \n                          cb_attention_masks4[cb_train_idx4], \n                          cb_labels4[cb_train_idx4])\n\ncb_val_set4 = TensorDataset(cb_token_id4[cb_val_idx4], \n                        cb_attention_masks4[cb_val_idx4], \n                        cb_labels4[cb_val_idx4])\n\n#Prepare DataLoader\ncb_train_dataloader4 = DataLoader(\n            cb_train_set4,\n            sampler = RandomSampler(cb_train_set4),\n            batch_size = cb_batch_size4)\n\ncb_validation_dataloader4 = DataLoader(\n            cb_val_set4,\n            sampler = SequentialSampler(cb_val_set4),\n            batch_size = cb_batch_size4)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:13:34.635673Z","iopub.execute_input":"2023-05-11T20:13:34.636072Z","iopub.status.idle":"2023-05-11T20:13:35.175054Z","shell.execute_reply.started":"2023-05-11T20:13:34.636037Z","shell.execute_reply":"2023-05-11T20:13:35.173849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#CodeBERT\n#Model Object Initiation\n#Load the RobertaForSequenceClassification model\ncb_model4 = AutoModelForSequenceClassification.from_pretrained(\"microsoft/codebert-base\")\n\n#Recommended learning rates (Adam): 5e-5, 3e-5, 2e-5. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_optimizer4 = torch.optim.AdamW(cb_model4.parameters(), \n                              lr = 5e-5,\n                              eps = 1e-08)\n\n#Run on GPU\ncb_model4.cuda()\n\n#Model Training and Evaluation\n#Recommended number of epochs: 2, 3, 4. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_epochs4 = 3\n\nfor _ in trange(cb_epochs4, desc = 'Epoch'):\n    \n    # ========== Training ==========\n    # Set model to training mode\n    cb_model4.train()\n    \n    # Tracking variables\n    cb_tr_loss4 = 0\n    nb_tr_examples4, nb_tr_steps4 = 0, 0\n\n    for step, batch in enumerate(cb_train_dataloader4):\n        batch = tuple(t.to(device) for t in batch)\n        cb_input_ids4, cb_input_mask4, cb_labels4 = batch\n        cb_optimizer4.zero_grad()\n        # Forward pass\n        cb_train_output4 = cb_model4(cb_input_ids4, \n                             token_type_ids = None, \n                             attention_mask = cb_input_mask4, \n                             labels = cb_labels4)\n        # Backward pass\n        cb_train_output4.loss.backward()\n        cb_optimizer4.step()\n        # Update tracking variables\n        cb_tr_loss4 += cb_train_output4.loss.item()\n        nb_tr_examples4 += cb_input_ids4.size(0)\n        nb_tr_steps4 += 1\n\n    # ========== Validation ==========\n    # Set model to evaluation mode\n    cb_model4.eval()\n\n    # Tracking variables \n    cb_val_accuracy4, cb_val_precision4, cb_val_recall4, cb_val_specificity4, cf1_val_score4 = [], [], [], [], []\n\n    for batch in cb_validation_dataloader4:\n        batch = tuple(t.to(device) for t in batch)\n        cb_input_ids4, cb_input_mask4, cb_labels4 = batch\n        with torch.no_grad():\n          # Forward pass\n          cb_eval_output4 = cb_model4(cb_input_ids4, \n                              token_type_ids = None, \n                              attention_mask = cb_input_mask4)\n        logits4 = cb_eval_output4.logits.detach().cpu().numpy()\n        label_ids4 = cb_labels4.to('cpu').numpy()\n        # Calculate validation metrics\n        cb_accuracy4, cb_precision4, cb_recall4, cb_specificity4, cf1_score4 = b_metrics(logits4, label_ids4)\n        cb_val_accuracy4.append(cb_accuracy4)\n        # Update precision only when (tp + fp) !=0; ignore nan\n        if cb_precision4 != 'nan': cb_val_precision4.append(cb_precision4)\n        # Update recall only when (tp + fn) !=0; ignore nan\n        if cb_recall4 != 'nan': cb_val_recall4.append(cb_recall4)\n        # Update specificity only when (tn + fp) !=0; ignore nan\n        if cb_specificity4 != 'nan': cb_val_specificity4.append(cb_specificity4)\n        # Update f1_score only when recall and specificity != nan; ignore nan\n        if cf1_score4 != 'nan': cf1_val_score4.append(cf1_score4)\n\n    print('\\n\\t - Train loss: {:.4f}'.format(cb_tr_loss4 / nb_tr_steps4))\n    print('\\t - Validation Accuracy: {:.4f}'.format(sum(cb_val_accuracy4)/len(cb_val_accuracy4)))\n    print('\\t - Validation Precision: {:.4f}'.format(sum(cb_val_precision4)/len(cb_val_precision4)) if len(cb_val_precision4)>0 else '\\t - Validation Precision: NaN')\n    print('\\t - Validation Recall: {:.4f}'.format(sum(cb_val_recall4)/len(cb_val_recall4)) if len(cb_val_recall4)>0 else '\\t - Validation Recall: NaN')\n    print('\\t - Validation Specificity: {:.4f}'.format(sum(cb_val_specificity4)/len(cb_val_specificity4)) if len(cb_val_specificity4)>0 else '\\t - Validation Specificity: NaN')\n    print('\\t - F1 Score: {:.4f}\\n'.format(sum(cf1_val_score4)/len(cf1_val_score4)) if len(cf1_val_score4)>0 else '\\t - F1 Score: NaN')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:13:39.291769Z","iopub.execute_input":"2023-05-11T20:13:39.292159Z","iopub.status.idle":"2023-05-11T20:14:13.322674Z","shell.execute_reply.started":"2023-05-11T20:13:39.292123Z","shell.execute_reply":"2023-05-11T20:14:13.321562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### JHotDraw741","metadata":{}},{"cell_type":"markdown","source":"#### Load Dataset, Data Preprocessing and Model Training and Evaluation","metadata":{}},{"cell_type":"code","source":"%%time\n#Dataset Importation\ncodes5  = pd.read_csv(\"/kaggle/input/sourcesniffer/SourceSniffer/JHotDraw741/JHotDraw741_Raw_Data_2.csv\", encoding='latin-1')\ncodes5.drop(['Unnamed: 0','Index'], axis=1, inplace=True)\n#codes5.head(1)\n\n#Dataset Preprocessing (Cleaning)\ncodes5['Code and Comment'] = codes5['Code and Comment'].str.lower()\ncodes5['Code and Comment'] = codes5['Code and Comment'].apply(clean_code)\n#codes5.head(1)\n\n#Dataset Preprocessing (Assigning non-null entries to new variable)\ncoh_data5 = codes5[[\"Code and Comment\",\"Label\"]][~codes5['Label'].isnull()]\n#coh_data5.shape\n\n#Dataset Preprocessing (Label Encoding)\nle = preprocessing.LabelEncoder()\ncoh_stat5 = le.fit_transform(coh_data5['Label'])\ncode_comm5 = coh_data5['Code and Comment']\n\n#Identify which label numerical code correspond to which categories\nprint(collections.Counter(coh_stat5), collections.Counter(coh_data5['Label']))\n","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:17:15.752205Z","iopub.execute_input":"2023-05-11T20:17:15.753003Z","iopub.status.idle":"2023-05-11T20:17:15.848965Z","shell.execute_reply.started":"2023-05-11T20:17:15.752962Z","shell.execute_reply":"2023-05-11T20:17:15.84777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### BERT","metadata":{}},{"cell_type":"code","source":"%%time\n#BERT\n#Dataset Preprocessing (Code-Comment Vector-Encoding)\n\nb_token_id5 = []\nb_attention_masks5 = []\n\nfor sample in code_comm5:\n    b_encoding_dict5 = bert_preprocessing(sample, b_tokenizer)\n    b_token_id5.append(b_encoding_dict5['input_ids']) \n    b_attention_masks5.append(b_encoding_dict5['attention_mask'])\n\nb_token_id5 = torch.cat(b_token_id5, dim = 0)\nb_attention_masks5 = torch.cat(b_attention_masks5, dim = 0)\nb_labels5 = torch.tensor(coh_stat5)\n\n#BERT\n#Splitting Dataset into training and validation set, and loading into batches.\nb_val_ratio5 = 0.2\n# Recommended batch size: 16, 32. See: https://arxiv.org/pdf/1810.04805.pdf\nb_batch_size5 = 16\n\n# Indices of the train and validation splits stratified by labels\nb_train_idx5, b_val_idx5 = train_test_split(\n    np.arange(len(b_labels5)),\n    test_size = b_val_ratio5,\n    shuffle = True,\n    stratify = b_labels5)\n\n# Train and validation sets\nb_train_set5 = TensorDataset(b_token_id5[b_train_idx5], \n                          b_attention_masks5[b_train_idx5], \n                          b_labels5[b_train_idx5])\n\nb_val_set5 = TensorDataset(b_token_id5[b_val_idx5], \n                        b_attention_masks5[b_val_idx5], \n                        b_labels5[b_val_idx5])\n\n# Prepare DataLoader\nb_train_dataloader5 = DataLoader(\n            b_train_set5,\n            sampler = RandomSampler(b_train_set5),\n            batch_size = b_batch_size5)\n\nb_validation_dataloader5 = DataLoader(\n            b_val_set5,\n            sampler = SequentialSampler(b_val_set5),\n            batch_size = b_batch_size5)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:17:24.591788Z","iopub.execute_input":"2023-05-11T20:17:24.592419Z","iopub.status.idle":"2023-05-11T20:17:33.588125Z","shell.execute_reply.started":"2023-05-11T20:17:24.59238Z","shell.execute_reply":"2023-05-11T20:17:33.586908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#BERT\n#Model Object Initiation\n#Load the BertForSequenceClassification model\nb_model5 = BertForSequenceClassification.from_pretrained(\n    'bert-base-uncased',\n    num_labels = 2,\n    output_attentions = False,\n    output_hidden_states = False,)\n\n#Recommended learning rates (Adam): 5e-5, 3e-5, 2e-5. See: https://arxiv.org/pdf/1810.04805.pdf\nb_optimizer5 = torch.optim.AdamW(b_model5.parameters(), \n                              lr = 5e-5,\n                              eps = 1e-08)\n\n#Run on GPU\nb_model5.cuda()\n\n#Model Training and Evaluation\n# Recommended number of epochs: 2, 3, 4. See: https://arxiv.org/pdf/1810.04805.pdf\nb_epochs5 = 3\n\nfor _ in trange(b_epochs5, desc = 'Epoch'):\n    \n    # ========== Training ==========\n    # Set model to training mode\n    b_model5.train()\n    \n    # Tracking variables\n    b_tr_loss5 = 0\n    nb_tr_examples5, nb_tr_steps5 = 0, 0\n\n    for step, batch in enumerate(b_train_dataloader5):\n        batch = tuple(t.to(device) for t in batch)\n        b_input_ids5, b_input_mask5, b_labels5 = batch\n        b_optimizer5.zero_grad()\n        # Forward pass\n        b_train_output5 = b_model5(b_input_ids5, \n                             token_type_ids = None, \n                             attention_mask = b_input_mask5, \n                             labels = b_labels5)\n        # Backward pass\n        b_train_output5.loss.backward()\n        b_optimizer5.step()\n        # Update tracking variables\n        b_tr_loss5 += b_train_output5.loss.item()\n        nb_tr_examples5 += b_input_ids5.size(0)\n        nb_tr_steps5 += 1\n\n    # ========== Validation ==========\n    # Set model to evaluation mode\n    b_model5.eval()\n\n    # Tracking variables \n    b_val_accuracy5, b_val_precision5, b_val_recall5, b_val_specificity5, f1_val_score5 = [], [], [], [], []\n\n    for batch in b_validation_dataloader5:\n        batch = tuple(t.to(device) for t in batch)\n        b_input_ids5, b_input_mask5, b_labels5 = batch\n        with torch.no_grad():\n          # Forward pass\n          b_eval_output5 = b_model5(b_input_ids5, \n                              token_type_ids = None, \n                              attention_mask = b_input_mask5)\n        logits5 = b_eval_output5.logits.detach().cpu().numpy()\n        label_ids5 = b_labels5.to('cpu').numpy()\n        # Calculate validation metrics\n        b_accuracy5, b_precision5, b_recall5, b_specificity5, f1_score5 = b_metrics(logits5, label_ids5)\n        b_val_accuracy5.append(b_accuracy5)\n        # Update precision only when (tp + fp) !=0; ignore nan\n        if b_precision5 != 'nan': b_val_precision5.append(b_precision5)\n        # Update recall only when (tp + fn) !=0; ignore nan\n        if b_recall5 != 'nan': b_val_recall5.append(b_recall5)\n        # Update specificity only when (tn + fp) !=0; ignore nan\n        if b_specificity5 != 'nan': b_val_specificity5.append(b_specificity5)\n        # Update f1_score only when recall and specificity != nan; ignore nan\n        if f1_score5 != 'nan': f1_val_score5.append(f1_score5)\n\n    print('\\n\\t - Train loss: {:.4f}'.format(b_tr_loss5 / nb_tr_steps5))\n    print('\\t - Validation Accuracy: {:.4f}'.format(sum(b_val_accuracy5)/len(b_val_accuracy5)))\n    print('\\t - Validation Precision: {:.4f}'.format(sum(b_val_precision5)/len(b_val_precision5)) if len(b_val_precision5)>0 else '\\t - Validation Precision: NaN')\n    print('\\t - Validation Recall: {:.4f}'.format(sum(b_val_recall5)/len(b_val_recall5)) if len(b_val_recall5)>0 else '\\t - Validation Recall: NaN')\n    print('\\t - Validation Specificity: {:.4f}'.format(sum(b_val_specificity5)/len(b_val_specificity5)) if len(b_val_specificity5)>0 else '\\t - Validation Specificity: NaN')\n    print('\\t - F1 Score: {:.4f}\\n'.format(sum(f1_val_score5)/len(f1_val_score5)) if len(f1_val_score5)>0 else '\\t - F1 Score: NaN')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:17:40.63004Z","iopub.execute_input":"2023-05-11T20:17:40.630506Z","iopub.status.idle":"2023-05-11T20:19:19.690761Z","shell.execute_reply.started":"2023-05-11T20:17:40.630469Z","shell.execute_reply":"2023-05-11T20:19:19.689668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### CodeBERT","metadata":{}},{"cell_type":"code","source":"%%time\n#CodeBERT\n#Dataset Preprocessing (Code-Comment Vector-Encoding)\n\ncb_token_id5 = []\ncb_attention_masks5 = []\n\nfor sample in code_comm5:\n    cb_encoding_dict5 = codebert_preprocessing(sample, cb_tokenizer)\n    cb_token_id5.append(cb_encoding_dict5['input_ids']) \n    cb_attention_masks5.append(cb_encoding_dict5['attention_mask'])\n\ncb_token_id5 = torch.cat(cb_token_id5, dim = 0)\ncb_attention_masks5 = torch.cat(cb_attention_masks5, dim = 0)\ncb_labels5 = torch.tensor(coh_stat5)\n\n#CodeBERT\n#Splitting Dataset into training and validation set, and loading into batches.\ncb_val_ratio5 = 0.2\n#Recommended batch size: 16, 32. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_batch_size5 = 16\n\n#Indices of the train and validation splits stratified by labels\ncb_train_idx5, cb_val_idx5 = train_test_split(\n    np.arange(len(cb_labels5)),\n    test_size = cb_val_ratio5,\n    shuffle = True,\n    stratify = cb_labels5)\n\n#Train and validation sets\ncb_train_set5 = TensorDataset(cb_token_id5[cb_train_idx5], \n                          cb_attention_masks5[cb_train_idx5], \n                          cb_labels5[cb_train_idx5])\n\ncb_val_set5 = TensorDataset(cb_token_id5[cb_val_idx5], \n                        cb_attention_masks5[cb_val_idx5], \n                        cb_labels5[cb_val_idx5])\n\n#Prepare DataLoader\ncb_train_dataloader5 = DataLoader(\n            cb_train_set5,\n            sampler = RandomSampler(cb_train_set5),\n            batch_size = cb_batch_size5)\n\ncb_validation_dataloader5 = DataLoader(\n            cb_val_set5,\n            sampler = SequentialSampler(cb_val_set5),\n            batch_size = cb_batch_size5)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:24:18.009477Z","iopub.execute_input":"2023-05-11T20:24:18.010343Z","iopub.status.idle":"2023-05-11T20:24:19.269892Z","shell.execute_reply.started":"2023-05-11T20:24:18.010303Z","shell.execute_reply":"2023-05-11T20:24:19.268775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#CodeBERT\n#Model Object Initiation\n#Load the RobertaForSequenceClassification model\ncb_model5 = AutoModelForSequenceClassification.from_pretrained(\"microsoft/codebert-base\")\n\n#Recommended learning rates (Adam): 5e-5, 3e-5, 2e-5. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_optimizer5 = torch.optim.AdamW(cb_model5.parameters(), \n                              lr = 5e-5,\n                              eps = 1e-08)\n\n#Run on GPU\ncb_model5.cuda()\n\n#Model Training and Evaluation\n#Recommended number of epochs: 2, 3, 4. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_epochs5 = 3\n\nfor _ in trange(cb_epochs5, desc = 'Epoch'):\n    \n    # ========== Training ==========\n    # Set model to training mode\n    cb_model5.train()\n    \n    # Tracking variables\n    cb_tr_loss5 = 0\n    nb_tr_examples5, nb_tr_steps5 = 0, 0\n\n    for step, batch in enumerate(cb_train_dataloader5):\n        batch = tuple(t.to(device) for t in batch)\n        cb_input_ids5, cb_input_mask5, cb_labels5 = batch\n        cb_optimizer5.zero_grad()\n        # Forward pass\n        cb_train_output5 = cb_model5(cb_input_ids5, \n                             token_type_ids = None, \n                             attention_mask = cb_input_mask5, \n                             labels = cb_labels5)\n        # Backward pass\n        cb_train_output5.loss.backward()\n        cb_optimizer5.step()\n        # Update tracking variables\n        cb_tr_loss5 += cb_train_output5.loss.item()\n        nb_tr_examples5 += cb_input_ids5.size(0)\n        nb_tr_steps5 += 1\n\n    # ========== Validation ==========\n    # Set model to evaluation mode\n    cb_model5.eval()\n\n    # Tracking variables \n    cb_val_accuracy5, cb_val_precision5, cb_val_recall5, cb_val_specificity5, cf1_val_score5 = [], [], [], [], []\n\n    for batch in cb_validation_dataloader5:\n        batch = tuple(t.to(device) for t in batch)\n        cb_input_ids5, cb_input_mask5, cb_labels5 = batch\n        with torch.no_grad():\n          # Forward pass\n          cb_eval_output5 = cb_model5(cb_input_ids5, \n                              token_type_ids = None, \n                              attention_mask = cb_input_mask5)\n        logits5 = cb_eval_output5.logits.detach().cpu().numpy()\n        label_ids5 = cb_labels5.to('cpu').numpy()\n        # Calculate validation metrics\n        cb_accuracy5, cb_precision5, cb_recall5, cb_specificity5, cf1_score5 = b_metrics(logits5, label_ids5)\n        cb_val_accuracy5.append(cb_accuracy5)\n        # Update precision only when (tp + fp) !=0; ignore nan\n        if cb_precision5 != 'nan': cb_val_precision5.append(cb_precision5)\n        # Update recall only when (tp + fn) !=0; ignore nan\n        if cb_recall5 != 'nan': cb_val_recall5.append(cb_recall5)\n        # Update specificity only when (tn + fp) !=0; ignore nan\n        if cb_specificity5 != 'nan': cb_val_specificity5.append(cb_specificity5)\n        # Update f1_score only when recall and specificity != nan; ignore nan\n        if cf1_score5 != 'nan': cf1_val_score5.append(cf1_score5)\n\n    print('\\n\\t - Train loss: {:.4f}'.format(cb_tr_loss5 / nb_tr_steps5))\n    print('\\t - Validation Accuracy: {:.4f}'.format(sum(cb_val_accuracy5)/len(cb_val_accuracy5)))\n    print('\\t - Validation Precision: {:.4f}'.format(sum(cb_val_precision5)/len(cb_val_precision5)) if len(cb_val_precision5)>0 else '\\t - Validation Precision: NaN')\n    print('\\t - Validation Recall: {:.4f}'.format(sum(cb_val_recall5)/len(cb_val_recall5)) if len(cb_val_recall5)>0 else '\\t - Validation Recall: NaN')\n    print('\\t - Validation Specificity: {:.4f}'.format(sum(cb_val_specificity5)/len(cb_val_specificity5)) if len(cb_val_specificity5)>0 else '\\t - Validation Specificity: NaN')\n    print('\\t - F1 Score: {:.4f}\\n'.format(sum(cf1_val_score5)/len(cf1_val_score5)) if len(cf1_val_score5)>0 else '\\t - F1 Score: NaN')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:24:19.272105Z","iopub.execute_input":"2023-05-11T20:24:19.272492Z","iopub.status.idle":"2023-05-11T20:26:03.359213Z","shell.execute_reply.started":"2023-05-11T20:24:19.272451Z","shell.execute_reply":"2023-05-11T20:26:03.357929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### All Datasets Combined","metadata":{}},{"cell_type":"markdown","source":"#### Load Dataset, Data Preprocessing and Model Training and Evaluation","metadata":{}},{"cell_type":"code","source":"%%time\n#Dataset Importation \nfiles = ['Benchmark_Raw_Data_2.csv','CoffeeMaker_Raw_Data_2.csv','JFreeChart060_Raw_Data_2.csv',\n           'JFreeChart071_Raw_Data_2.csv','JHotDraw741_Raw_Data_2.csv']\nfolders = ['Benchmark','CoffeeMaker','JFreeChart060','JFreeChart071','JHotDraw741']\nparent_path = '/kaggle/input/sourcesniffer/SourceSniffer'\n\nfirst_path = parent_path+'/'+folders[0]+'/'+files[0]\ncodes6  = pd.read_csv(first_path, encoding='latin-1')\n\n#concatenating all other datasets\nfor i in range(1,len(files)):\n    curr_path = parent_path+'/'+folders[i]+'/'+files[i]\n    temp_df = pd.read_csv(curr_path, encoding='latin-1')\n    codes6 = pd.concat([codes6, temp_df], axis=0)\n    \ncodes6.drop(['Unnamed: 0','Index'], axis=1, inplace=True)\n#codes6.head(1)\n\n#Dataset Preprocessing (Cleaning)\ncodes6['Code and Comment'] = codes6['Code and Comment'].str.lower()\ncodes6['Code and Comment'] = codes6['Code and Comment'].apply(clean_code)\n#codes6.head(1)\n\n#Dataset Preprocessing (Assigning non-null entries to new variable)\ncoh_data6 = codes6[[\"Code and Comment\",\"Label\"]][~codes6['Label'].isnull()]\n#coh_data6.shape\n\n#Dataset Preprocessing (Label Encoding)\nle = preprocessing.LabelEncoder()\ncoh_stat6 = le.fit_transform(coh_data6['Label'])\ncode_comm6 = coh_data6['Code and Comment']\n\n#Identify which label numerical code correspond to which categories\nprint(collections.Counter(coh_stat6), collections.Counter(coh_data6['Label']))\n","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:28:45.763672Z","iopub.execute_input":"2023-05-11T20:28:45.764269Z","iopub.status.idle":"2023-05-11T20:28:46.011641Z","shell.execute_reply.started":"2023-05-11T20:28:45.764225Z","shell.execute_reply":"2023-05-11T20:28:46.010425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### BERT","metadata":{}},{"cell_type":"code","source":"%%time\n#BERT\n#Dataset Preprocessing (Code-Comment Vector-Encoding)\n\nb_token_id6 = []\nb_attention_masks6 = []\n\nfor sample in code_comm6:\n    b_encoding_dict6 = bert_preprocessing(sample, b_tokenizer)\n    b_token_id6.append(b_encoding_dict6['input_ids']) \n    b_attention_masks6.append(b_encoding_dict6['attention_mask'])\n\nb_token_id6 = torch.cat(b_token_id6, dim = 0)\nb_attention_masks6 = torch.cat(b_attention_masks6, dim = 0)\nb_labels6 = torch.tensor(coh_stat6)\n\n#BERT\n#Splitting Dataset into training and validation set, and loading into batches.\nb_val_ratio6 = 0.2\n# Recommended batch size: 16, 32. See: https://arxiv.org/pdf/1810.04805.pdf\nb_batch_size6 = 16\n\n# Indices of the train and validation splits stratified by labels\nb_train_idx6, b_val_idx6 = train_test_split(\n    np.arange(len(b_labels6)),\n    test_size = b_val_ratio6,\n    shuffle = True,\n    stratify = b_labels6)\n\n# Train and validation sets\nb_train_set6 = TensorDataset(b_token_id6[b_train_idx6], \n                          b_attention_masks6[b_train_idx6], \n                          b_labels6[b_train_idx6])\n\nb_val_set6 = TensorDataset(b_token_id6[b_val_idx6], \n                        b_attention_masks6[b_val_idx6], \n                        b_labels6[b_val_idx6])\n\n# Prepare DataLoader\nb_train_dataloader6 = DataLoader(\n            b_train_set6,\n            sampler = RandomSampler(b_train_set6),\n            batch_size = b_batch_size6)\n\nb_validation_dataloader6 = DataLoader(\n            b_val_set6,\n            sampler = SequentialSampler(b_val_set6),\n            batch_size = b_batch_size6)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:28:53.618423Z","iopub.execute_input":"2023-05-11T20:28:53.619202Z","iopub.status.idle":"2023-05-11T20:29:24.546113Z","shell.execute_reply.started":"2023-05-11T20:28:53.61916Z","shell.execute_reply":"2023-05-11T20:29:24.544924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#BERT\n#Model Object Initiation\n#Load the BertForSequenceClassification model\nb_model6 = BertForSequenceClassification.from_pretrained(\n    'bert-base-uncased',\n    num_labels = 2,\n    output_attentions = False,\n    output_hidden_states = False,)\n\n#Recommended learning rates (Adam): 5e-5, 3e-5, 2e-5. See: https://arxiv.org/pdf/1810.04805.pdf\nb_optimizer6 = torch.optim.AdamW(b_model6.parameters(), \n                              lr = 5e-5,\n                              eps = 1e-08)\n\n#Run on GPU\nb_model6.cuda()\n\n#Model Training and Evaluation\n# Recommended number of epochs: 2, 3, 4. See: https://arxiv.org/pdf/1810.04805.pdf\nb_epochs6 = 3\n\nfor _ in trange(b_epochs6, desc = 'Epoch'):\n    \n    # ========== Training ==========\n    # Set model to training mode\n    b_model6.train()\n    \n    # Tracking variables\n    b_tr_loss6 = 0\n    nb_tr_examples6, nb_tr_steps6 = 0, 0\n\n    for step, batch in enumerate(b_train_dataloader6):\n        batch = tuple(t.to(device) for t in batch)\n        b_input_ids6, b_input_mask6, b_labels6 = batch\n        b_optimizer6.zero_grad()\n        # Forward pass\n        b_train_output6 = b_model6(b_input_ids6, \n                             token_type_ids = None, \n                             attention_mask = b_input_mask6, \n                             labels = b_labels6)\n        # Backward pass\n        b_train_output6.loss.backward()\n        b_optimizer6.step()\n        # Update tracking variables\n        b_tr_loss6 += b_train_output6.loss.item()\n        nb_tr_examples6 += b_input_ids6.size(0)\n        nb_tr_steps6 += 1\n\n    # ========== Validation ==========\n    # Set model to evaluation mode\n    b_model6.eval()\n\n    # Tracking variables \n    b_val_accuracy6, b_val_precision6, b_val_recall6, b_val_specificity6, f1_val_score6 = [], [], [], [], []\n\n    for batch in b_validation_dataloader6:\n        batch = tuple(t.to(device) for t in batch)\n        b_input_ids6, b_input_mask6, b_labels6 = batch\n        with torch.no_grad():\n          # Forward pass\n          b_eval_output6 = b_model6(b_input_ids6, \n                              token_type_ids = None, \n                              attention_mask = b_input_mask6)\n        logits6 = b_eval_output6.logits.detach().cpu().numpy()\n        label_ids6 = b_labels6.to('cpu').numpy()\n        # Calculate validation metrics\n        b_accuracy6, b_precision6, b_recall6, b_specificity6, f1_score6 = b_metrics(logits6, label_ids6)\n        b_val_accuracy6.append(b_accuracy6)\n        # Update precision only when (tp + fp) !=0; ignore nan\n        if b_precision6 != 'nan': b_val_precision6.append(b_precision6)\n        # Update recall only when (tp + fn) !=0; ignore nan\n        if b_recall6 != 'nan': b_val_recall6.append(b_recall6)\n        # Update specificity only when (tn + fp) !=0; ignore nan\n        if b_specificity6 != 'nan': b_val_specificity6.append(b_specificity6)\n        # Update f1_score only when recall and specificity != nan; ignore nan\n        if f1_score6 != 'nan': f1_val_score6.append(f1_score6)\n\n    print('\\n\\t - Train loss: {:.4f}'.format(b_tr_loss6 / nb_tr_steps6))\n    print('\\t - Validation Accuracy: {:.4f}'.format(sum(b_val_accuracy6)/len(b_val_accuracy6)))\n    print('\\t - Validation Precision: {:.4f}'.format(sum(b_val_precision6)/len(b_val_precision6)) if len(b_val_precision6)>0 else '\\t - Validation Precision: NaN')\n    print('\\t - Validation Recall: {:.4f}'.format(sum(b_val_recall6)/len(b_val_recall6)) if len(b_val_recall6)>0 else '\\t - Validation Recall: NaN')\n    print('\\t - Validation Specificity: {:.4f}'.format(sum(b_val_specificity6)/len(b_val_specificity6)) if len(b_val_specificity6)>0 else '\\t - Validation Specificity: NaN')\n    print('\\t - F1 Score: {:.4f}\\n'.format(sum(f1_val_score6)/len(f1_val_score6)) if len(f1_val_score6)>0 else '\\t - F1 Score: NaN')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:29:35.533027Z","iopub.execute_input":"2023-05-11T20:29:35.533532Z","iopub.status.idle":"2023-05-11T20:34:52.447541Z","shell.execute_reply.started":"2023-05-11T20:29:35.53349Z","shell.execute_reply":"2023-05-11T20:34:52.44641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### CodeBERT","metadata":{}},{"cell_type":"code","source":"%%time\n#CodeBERT\n#Dataset Preprocessing (Code-Comment Vector-Encoding)\n\ncb_token_id6 = []\ncb_attention_masks6 = []\n\nfor sample in code_comm6:\n    cb_encoding_dict6 = codebert_preprocessing(sample, cb_tokenizer)\n    cb_token_id6.append(cb_encoding_dict6['input_ids']) \n    cb_attention_masks6.append(cb_encoding_dict6['attention_mask'])\n\ncb_token_id6 = torch.cat(cb_token_id6, dim = 0)\ncb_attention_masks6 = torch.cat(cb_attention_masks6, dim = 0)\ncb_labels6 = torch.tensor(coh_stat6)\n\n#CodeBERT\n#Splitting Dataset into training and validation set, and loading into batches.\ncb_val_ratio6 = 0.2\n#Recommended batch size: 16, 32. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_batch_size6 = 16\n\n#Indices of the train and validation splits stratified by labels\ncb_train_idx6, cb_val_idx6 = train_test_split(\n    np.arange(len(cb_labels6)),\n    test_size = cb_val_ratio6,\n    shuffle = True,\n    stratify = cb_labels6)\n\n#Train and validation sets\ncb_train_set6 = TensorDataset(cb_token_id6[cb_train_idx6], \n                          cb_attention_masks6[cb_train_idx6], \n                          cb_labels6[cb_train_idx6])\n\ncb_val_set6 = TensorDataset(cb_token_id6[cb_val_idx6], \n                        cb_attention_masks6[cb_val_idx6], \n                        cb_labels6[cb_val_idx6])\n\n#Prepare DataLoader\ncb_train_dataloader6 = DataLoader(\n            cb_train_set6,\n            sampler = RandomSampler(cb_train_set6),\n            batch_size = cb_batch_size6)\n\ncb_validation_dataloader6 = DataLoader(\n            cb_val_set6,\n            sampler = SequentialSampler(cb_val_set6),\n            batch_size = cb_batch_size6)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:34:58.464078Z","iopub.execute_input":"2023-05-11T20:34:58.464596Z","iopub.status.idle":"2023-05-11T20:35:03.116993Z","shell.execute_reply.started":"2023-05-11T20:34:58.464556Z","shell.execute_reply":"2023-05-11T20:35:03.115743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#CodeBERT\n#Model Object Initiation\n#Load the RobertaForSequenceClassification model\ncb_model6 = AutoModelForSequenceClassification.from_pretrained(\"microsoft/codebert-base\")\n\n#Recommended learning rates (Adam): 5e-5, 3e-5, 2e-5. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_optimizer6 = torch.optim.AdamW(cb_model6.parameters(), \n                              lr = 5e-5,\n                              eps = 1e-08)\n\n#Run on GPU\ncb_model6.cuda()\n\n#Model Training and Evaluation\n#Recommended number of epochs: 2, 3, 4. See: https://arxiv.org/pdf/1810.04805.pdf\ncb_epochs6 = 3\n\nfor _ in trange(cb_epochs6, desc = 'Epoch'):\n    \n    # ========== Training ==========\n    # Set model to training mode\n    cb_model6.train()\n    \n    # Tracking variables\n    cb_tr_loss6 = 0\n    nb_tr_examples6, nb_tr_steps6 = 0, 0\n\n    for step, batch in enumerate(cb_train_dataloader6):\n        batch = tuple(t.to(device) for t in batch)\n        cb_input_ids6, cb_input_mask6, cb_labels6 = batch\n        cb_optimizer6.zero_grad()\n        # Forward pass\n        cb_train_output6 = cb_model6(cb_input_ids6, \n                             token_type_ids = None, \n                             attention_mask = cb_input_mask6, \n                             labels = cb_labels6)\n        # Backward pass\n        cb_train_output6.loss.backward()\n        cb_optimizer6.step()\n        # Update tracking variables\n        cb_tr_loss6 += cb_train_output6.loss.item()\n        nb_tr_examples6 += cb_input_ids6.size(0)\n        nb_tr_steps6 += 1\n\n    # ========== Validation ==========\n    # Set model to evaluation mode\n    cb_model6.eval()\n\n    # Tracking variables \n    cb_val_accuracy6, cb_val_precision6, cb_val_recall6, cb_val_specificity6, cf1_val_score6 = [], [], [], [], []\n\n    for batch in cb_validation_dataloader6:\n        batch = tuple(t.to(device) for t in batch)\n        cb_input_ids6, cb_input_mask6, cb_labels6 = batch\n        with torch.no_grad():\n          # Forward pass\n          cb_eval_output6 = cb_model6(cb_input_ids6, \n                              token_type_ids = None, \n                              attention_mask = cb_input_mask6)\n        logits6 = cb_eval_output6.logits.detach().cpu().numpy()\n        label_ids6 = cb_labels6.to('cpu').numpy()\n        # Calculate validation metrics\n        cb_accuracy6, cb_precision6, cb_recall6, cb_specificity6, cf1_score6 = b_metrics(logits6, label_ids6)\n        cb_val_accuracy6.append(cb_accuracy6)\n        # Update precision only when (tp + fp) !=0; ignore nan\n        if cb_precision6 != 'nan': cb_val_precision6.append(cb_precision6)\n        # Update recall only when (tp + fn) !=0; ignore nan\n        if cb_recall6 != 'nan': cb_val_recall6.append(cb_recall6)\n        # Update specificity only when (tn + fp) !=0; ignore nan\n        if cb_specificity6 != 'nan': cb_val_specificity6.append(cb_specificity6)\n        # Update f1_score only when recall and specificity != nan; ignore nan\n        if cf1_score6 != 'nan': cf1_val_score6.append(cf1_score6)\n\n    print('\\n\\t - Train loss: {:.4f}'.format(cb_tr_loss6 / nb_tr_steps6))\n    print('\\t - Validation Accuracy: {:.4f}'.format(sum(cb_val_accuracy6)/len(cb_val_accuracy6)))\n    print('\\t - Validation Precision: {:.4f}'.format(sum(cb_val_precision6)/len(cb_val_precision6)) if len(cb_val_precision6)>0 else '\\t - Validation Precision: NaN')\n    print('\\t - Validation Recall: {:.4f}'.format(sum(cb_val_recall6)/len(cb_val_recall6)) if len(cb_val_recall6)>0 else '\\t - Validation Recall: NaN')\n    print('\\t - Validation Specificity: {:.4f}'.format(sum(cb_val_specificity6)/len(cb_val_specificity6)) if len(cb_val_specificity6)>0 else '\\t - Validation Specificity: NaN')\n    print('\\t - F1 Score: {:.4f}\\n'.format(sum(cf1_val_score6)/len(cf1_val_score6)) if len(cf1_val_score6)>0 else '\\t - F1 Score: NaN')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T20:35:10.024746Z","iopub.execute_input":"2023-05-11T20:35:10.025319Z","iopub.status.idle":"2023-05-11T20:40:34.788944Z","shell.execute_reply.started":"2023-05-11T20:35:10.025281Z","shell.execute_reply":"2023-05-11T20:40:34.787619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"## Baseline (Corazza et al)","metadata":{}},{"cell_type":"markdown","source":"### SVM + VSM (TD-IDF)","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.svm import SVC\n\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score","metadata":{"execution":{"iopub.status.busy":"2023-05-26T08:05:37.773629Z","iopub.execute_input":"2023-05-26T08:05:37.774098Z","iopub.status.idle":"2023-05-26T08:05:37.780763Z","shell.execute_reply.started":"2023-05-26T08:05:37.774055Z","shell.execute_reply":"2023-05-26T08:05:37.779588Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"#Defining Evaluation Metrics\ndef b_tp(preds, labels):\n    '''Returns True Positives (TP): count of correct predictions of actual class 1'''\n    return sum([preds == labels and preds == 0 for preds, labels in zip(preds, labels)])\n\ndef b_fp(preds, labels):\n    '''Returns False Positives (FP): count of wrong predictions of actual class 1'''\n    return sum([preds != labels and preds == 0 for preds, labels in zip(preds, labels)])\n\ndef b_tn(preds, labels):\n    '''Returns True Negatives (TN): count of correct predictions of actual class 0'''\n    return sum([preds == labels and preds == 1 for preds, labels in zip(preds, labels)])\n\ndef b_fn(preds, labels):\n    '''Returns False Negatives (FN): count of wrong predictions of actual class 0'''\n    return sum([preds != labels and preds == 1 for preds, labels in zip(preds, labels)])\n\ndef b_metrics(preds, labels):\n    '''\n    Returns the following metrics:\n    - accuracy    = (TP + TN) / N\n    - precision   = TP / (TP + FP)\n    - recall      = TP / (TP + FN)\n    - specificity = TN / (TN + FP)\n    '''\n    preds = preds\n    labels = labels\n    tp = b_tp(preds, labels)\n    tn = b_tn(preds, labels)\n    fp = b_fp(preds, labels)\n    fn = b_fn(preds, labels)\n    b_accuracy = (tp + tn) / len(labels)\n    b_precision = tp / (tp + fp) if (tp + fp) > 0 else 'nan'\n    b_recall = tp / (tp + fn) if (tp + fn) > 0 else 'nan'\n    b_specificity = tn / (tn + fp) if (tn + fp) > 0 else 'nan'\n    f1_score = 2*((b_precision*b_recall)/(b_precision+b_recall)) if (b_precision != 'nan' and b_recall != 'nan') else 'nan'\n    return b_accuracy, b_precision, b_recall, b_specificity, f1_score","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Benchmark","metadata":{}},{"cell_type":"code","source":"%%time\n#Dataset Importation\n\ng_codes1  = pd.read_csv(\"/kaggle/input/sourcesniffer/SourceSniffer/Benchmark/Benchmark_Raw_Data_2.csv\", encoding='latin-1')\ng_codes1.drop(['Unnamed: 0','Index'], axis=1, inplace=True)\n\n#Dataset Preprocessing (Cleaning)\n\ng_codes1['Code and Comment'] = g_codes1['Code and Comment'].str.lower()\ng_codes1['Code and Comment'] = g_codes1['Code and Comment'].apply(clean_code)\n\n#Dataset Preprocessing (Assigning non-null entries to new variable)\n\ngcoh_data1 = g_codes1[[\"Code and Comment\",\"Label\"]][~g_codes1['Label'].isnull()]\n\n#Dataset Preprocessing (Label Encoding)\n\nle = preprocessing.LabelEncoder()\ngcoh_stat1 = le.fit_transform(gcoh_data1['Label'])\n\ngcode_comm1 = gcoh_data1['Code and Comment']\ngcode_comm2 = gcode_comm1.to_list()\n\nvectorizer = TfidfVectorizer()\nvectors = vectorizer.fit_transform(gcode_comm2)\nfeature_names = vectorizer.get_feature_names_out()\ndense = vectors.todense()\ndenselist = dense.tolist()\ndf_coh = pd.DataFrame(denselist, columns=feature_names)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor i in range(3):\n    x_train, x_test, y_train, y_test = train_test_split(df_coh, gcoh_stat1, test_size=.25)\n    clf = SVC(C = 10.0, gamma = 1.0, kernel = 'rbf')\n    clf.fit(x_train, y_train)\n    print('Accuracy on Test set: %.3f' %  (clf.score(x_test, y_test)))\n    #Model Evaluation\n    y_pred = clf.predict(x_test)\n    accuracy, precision, recall, specificity, f1_score = b_metrics(y_pred, y_test)\n    print('Validation Accuracy: {:.4f}'.format(accuracy))\n    print('Validation Precision: {:.4f}'.format(precision))\n    print('Validation Recall: {:.4f}'.format(recall))\n    print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CoffeeMaker","metadata":{}},{"cell_type":"code","source":"%%time\n#Dataset Importation\n\ng_codes1  = pd.read_csv(\"/kaggle/input/sourcesniffer/SourceSniffer/CoffeeMaker/CoffeeMaker_Raw_Data_2.csv\", encoding='latin-1')\ng_codes1.drop(['Unnamed: 0','Index'], axis=1, inplace=True)\n\n#Dataset Preprocessing (Cleaning)\n\ng_codes1['Code and Comment'] = g_codes1['Code and Comment'].str.lower()\ng_codes1['Code and Comment'] = g_codes1['Code and Comment'].apply(clean_code)\n\n#Dataset Preprocessing (Assigning non-null entries to new variable)\n\ngcoh_data1 = g_codes1[[\"Code and Comment\",\"Label\"]][~g_codes1['Label'].isnull()]\n\n#Dataset Preprocessing (Label Encoding)\n\nle = preprocessing.LabelEncoder()\ngcoh_stat1 = le.fit_transform(gcoh_data1['Label'])\n\ngcode_comm1 = gcoh_data1['Code and Comment']\ngcode_comm2 = gcode_comm1.to_list()\n\nvectorizer = TfidfVectorizer()\nvectors = vectorizer.fit_transform(gcode_comm2)\nfeature_names = vectorizer.get_feature_names_out()\ndense = vectors.todense()\ndenselist = dense.tolist()\ndf_coh = pd.DataFrame(denselist, columns=feature_names)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(3):\n    x_train, x_test, y_train, y_test = train_test_split(df_coh, gcoh_stat1, test_size=.25)\n    clf = SVC(C = 10.0, gamma = 1.0, kernel = 'rbf')\n    clf.fit(x_train, y_train)\n    print('Accuracy on Test set: %.3f' %  (clf.score(x_test, y_test)))\n    #Model Evaluation\n    y_pred = clf.predict(x_test)\n    accuracy, precision, recall, specificity, f1_score = b_metrics(y_pred, y_test)\n    print('Validation Accuracy: {:.4f}'.format(accuracy))\n    print('Validation Precision: {:.4f}'.format(precision))\n    print('Validation Recall: {:.4f}'.format(recall))\n    print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### JFreeChart060","metadata":{}},{"cell_type":"code","source":"%%time\n#Dataset Importation\n\ng_codes1  = pd.read_csv(\"/kaggle/input/sourcesniffer/SourceSniffer/JFreeChart060/JFreeChart060_Raw_Data_2.csv\", encoding='latin-1')\ng_codes1.drop(['Unnamed: 0','Index'], axis=1, inplace=True)\n\n#Dataset Preprocessing (Cleaning)\n\ng_codes1['Code and Comment'] = g_codes1['Code and Comment'].str.lower()\ng_codes1['Code and Comment'] = g_codes1['Code and Comment'].apply(clean_code)\n\n#Dataset Preprocessing (Assigning non-null entries to new variable)\n\ngcoh_data1 = g_codes1[[\"Code and Comment\",\"Label\"]][~g_codes1['Label'].isnull()]\n\n#Dataset Preprocessing (Label Encoding)\n\nle = preprocessing.LabelEncoder()\ngcoh_stat1 = le.fit_transform(gcoh_data1['Label'])\n\ngcode_comm1 = gcoh_data1['Code and Comment']\ngcode_comm2 = gcode_comm1.to_list()\n\nvectorizer = TfidfVectorizer()\nvectors = vectorizer.fit_transform(gcode_comm2)\nfeature_names = vectorizer.get_feature_names_out()\ndense = vectors.todense()\ndenselist = dense.tolist()\ndf_coh = pd.DataFrame(denselist, columns=feature_names)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(3):\n    x_train, x_test, y_train, y_test = train_test_split(df_coh, gcoh_stat1, test_size=.25)\n    clf = SVC(C = 10.0, gamma = 1.0, kernel = 'rbf')\n    clf.fit(x_train, y_train)\n    print('Accuracy on Test set: %.3f' %  (clf.score(x_test, y_test)))\n    #Model Evaluation\n    y_pred = clf.predict(x_test)\n    accuracy, precision, recall, specificity, f1_score = b_metrics(y_pred, y_test)\n    print('Validation Accuracy: {:.4f}'.format(accuracy))\n    print('Validation Precision: {:.4f}'.format(precision))\n    print('Validation Recall: {:.4f}'.format(recall))\n    print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### JFreeChart071","metadata":{}},{"cell_type":"code","source":"%%time\n#Dataset Importation\n\ng_codes1  = pd.read_csv(\"/kaggle/input/sourcesniffer/SourceSniffer/JFreeChart071/JFreeChart071_Raw_Data_2.csv\", encoding='latin-1')\ng_codes1.drop(['Unnamed: 0','Index'], axis=1, inplace=True)\n\n#Dataset Preprocessing (Cleaning)\n\ng_codes1['Code and Comment'] = g_codes1['Code and Comment'].str.lower()\ng_codes1['Code and Comment'] = g_codes1['Code and Comment'].apply(clean_code)\n\n#Dataset Preprocessing (Assigning non-null entries to new variable)\n\ngcoh_data1 = g_codes1[[\"Code and Comment\",\"Label\"]][~g_codes1['Label'].isnull()]\n\n#Dataset Preprocessing (Label Encoding)\n\nle = preprocessing.LabelEncoder()\ngcoh_stat1 = le.fit_transform(gcoh_data1['Label'])\n\ngcode_comm1 = gcoh_data1['Code and Comment']\ngcode_comm2 = gcode_comm1.to_list()\n\nvectorizer = TfidfVectorizer()\nvectors = vectorizer.fit_transform(gcode_comm2)\nfeature_names = vectorizer.get_feature_names_out()\ndense = vectors.todense()\ndenselist = dense.tolist()\ndf_coh = pd.DataFrame(denselist, columns=feature_names)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(3):\n    x_train, x_test, y_train, y_test = train_test_split(df_coh, gcoh_stat1, test_size=.25)\n    clf = SVC(C = 10.0, gamma = 1.0, kernel = 'rbf')\n    clf.fit(x_train, y_train)\n    print('Accuracy on Test set: %.3f' %  (clf.score(x_test, y_test)))\n    #Model Evaluation\n    y_pred = clf.predict(x_test)\n    accuracy, precision, recall, specificity, f1_score = b_metrics(y_pred, y_test)\n    print('Validation Accuracy: {:.4f}'.format(accuracy))\n    print('Validation Precision: {:.4f}'.format(precision))\n    print('Validation Recall: {:.4f}'.format(recall))\n    print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### JHotDraw741","metadata":{}},{"cell_type":"code","source":"%%time\n#Dataset Importation\n\ng_codes1  = pd.read_csv(\"/kaggle/input/sourcesniffer/SourceSniffer/JHotDraw741/JHotDraw741_Raw_Data_2.csv\", encoding='latin-1')\ng_codes1.drop(['Unnamed: 0','Index'], axis=1, inplace=True)\n\n#Dataset Preprocessing (Cleaning)\n\ng_codes1['Code and Comment'] = g_codes1['Code and Comment'].str.lower()\ng_codes1['Code and Comment'] = g_codes1['Code and Comment'].apply(clean_code)\n\n#Dataset Preprocessing (Assigning non-null entries to new variable)\n\ngcoh_data1 = g_codes1[[\"Code and Comment\",\"Label\"]][~g_codes1['Label'].isnull()]\n\n#Dataset Preprocessing (Label Encoding)\n\nle = preprocessing.LabelEncoder()\ngcoh_stat1 = le.fit_transform(gcoh_data1['Label'])\n\ngcode_comm1 = gcoh_data1['Code and Comment']\ngcode_comm2 = gcode_comm1.to_list()\n\nvectorizer = TfidfVectorizer()\nvectors = vectorizer.fit_transform(gcode_comm2)\nfeature_names = vectorizer.get_feature_names_out()\ndense = vectors.todense()\ndenselist = dense.tolist()\ndf_coh = pd.DataFrame(denselist, columns=feature_names)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(3):\n    x_train, x_test, y_train, y_test = train_test_split(df_coh, gcoh_stat1, test_size=.25)\n    clf = SVC(C = 10.0, gamma = 1.0, kernel = 'rbf')\n    clf.fit(x_train, y_train)\n    print('Accuracy on Test set: %.3f' %  (clf.score(x_test, y_test)))\n    #Model Evaluation\n    y_pred = clf.predict(x_test)\n    accuracy, precision, recall, specificity, f1_score = b_metrics(y_pred, y_test)\n    print('Validation Accuracy: {:.4f}'.format(accuracy))\n    print('Validation Precision: {:.4f}'.format(precision))\n    print('Validation Recall: {:.4f}'.format(recall))\n    print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### All Datasets","metadata":{}},{"cell_type":"code","source":"#Dataset Importation\nfiles = ['Benchmark_Raw_Data_2.csv','CoffeeMaker_Raw_Data_2.csv','JFreeChart060_Raw_Data_2.csv',\n           'JFreeChart071_Raw_Data_2.csv','JHotDraw741_Raw_Data_2.csv']\nfolders = ['Benchmark','CoffeeMaker','JFreeChart060','JFreeChart071','JHotDraw741']\nparent_path = '/kaggle/input/sourcesniffer/SourceSniffer'\n\nfirst_path = parent_path+'/'+folders[0]+'/'+files[0]\ng_codes1  = pd.read_csv(first_path, encoding='latin-1')\n\n#concatenating all other datasets\nfor i in range(1,len(files)):\n    curr_path = parent_path+'/'+folders[i]+'/'+files[i]\n    temp_df = pd.read_csv(curr_path, encoding='latin-1')\n    g_codes1 = pd.concat([g_codes1, temp_df], axis=0)\n    \ng_codes1.drop(['Unnamed: 0','Index'], axis=1, inplace=True)\n\n#Dataset Preprocessing (Cleaning)\n\ng_codes1['Code and Comment'] = g_codes1['Code and Comment'].str.lower()\ng_codes1['Code and Comment'] = g_codes1['Code and Comment'].apply(clean_code)\n\n#Dataset Preprocessing (Assigning non-null entries to new variable)\n\ngcoh_data1 = g_codes1[[\"Code and Comment\",\"Label\"]][~g_codes1['Label'].isnull()]\n\n#Dataset Preprocessing (Label Encoding)\n\nle = preprocessing.LabelEncoder()\ngcoh_stat1 = le.fit_transform(gcoh_data1['Label'])\n\ngcode_comm1 = gcoh_data1['Code and Comment']\ngcode_comm2 = gcode_comm1.to_list()\n\nvectorizer = TfidfVectorizer()\nvectors = vectorizer.fit_transform(gcode_comm2)\nfeature_names = vectorizer.get_feature_names_out()\ndense = vectors.todense()\ndenselist = dense.tolist()\ndf_coh = pd.DataFrame(denselist, columns=feature_names)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(3):\n    x_train, x_test, y_train, y_test = train_test_split(df_coh, gcoh_stat1, test_size=.25)\n    clf = SVC(C = 10.0, gamma = 1.0, kernel = 'rbf')\n    clf.fit(x_train, y_train)\n    print('Accuracy on Test set: %.3f' %  (clf.score(x_test, y_test)))\n    #Model Evaluation\n    y_pred = clf.predict(x_test)\n    accuracy, precision, recall, specificity, f1_score = b_metrics(y_pred, y_test)\n    print('Validation Accuracy: {:.4f}'.format(accuracy))\n    print('Validation Precision: {:.4f}'.format(precision))\n    print('Validation Recall: {:.4f}'.format(recall))\n    print('F1 Score: {:.4f}\\n'.format(f1_score))","metadata":{},"execution_count":null,"outputs":[]}]}