historical feature.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1c549f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# historical three feature\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "import codecs\n",
    "from keras_preprocessing.sequence import pad_sequences\n",
    "import numpy as np\n",
    "import nltk\n",
    "from konlpy.tag import Okt\n",
    "from tqdm import tqdm\n",
    "\n",
    "# train set\n",
    "name = \"1\"\n",
    "filename1 = './new_data1/definition'+name+'_spynorth_scaling_trust.txt'\n",
    "filename2 = './new_data1/definition'+name+'_spynorth_scaling_untrust.txt'\n",
    "filename3 = './new_data1/definition'+name+'_intistranger_scaling_trust.txt'\n",
    "filename4 = './new_data1/definition'+name+'_intistranger_scaling_untrust.txt'\n",
    "filename5 = './new_data1/definition'+name+'_assassin_scaling_trust.txt'\n",
    "filename6 = './new_data1/definition'+name+'_assassin_scaling_untrust.txt'\n",
    "filename7 = './new_data1/definition'+name+'_1987_scaling_trust.txt'\n",
    "filename8 = './new_data1/definition'+name+'_1987_scaling_untrust.txt'\n",
    "filename9 = './new_data1/definition'+name+'_taxi_scaling_trust.txt'\n",
    "filename10 = './new_data1/definition'+name+'_taxi_scaling_untrust.txt'\n",
    "filename11 = './new_data1/definition'+name+'_gongjo_scaling_trust.txt'\n",
    "filename12 = './new_data1/definition'+name+'_gongjo_scaling_untrust.txt'\n",
    "\n",
    "\n",
    "with codecs.open(filename1, 'r', encoding='utf-8-sig') as f:\n",
    "    lines1 = f.readlines()\n",
    "with codecs.open(filename2, 'r', encoding='utf-8-sig') as f:\n",
    "    lines2 = f.readlines()\n",
    "with codecs.open(filename3, 'r', encoding='utf-8-sig') as f:\n",
    "    lines3 = f.readlines()\n",
    "with codecs.open(filename4, 'r', encoding='utf-8-sig') as f:\n",
    "    lines4 = f.readlines()\n",
    "with codecs.open(filename5, 'r', encoding='utf-8-sig') as f:\n",
    "    lines5 = f.readlines()\n",
    "with codecs.open(filename6, 'r', encoding='utf-8-sig') as f:\n",
    "    lines6 = f.readlines()\n",
    "with codecs.open(filename7, 'r', encoding='utf-8-sig') as f:\n",
    "    lines7 = f.readlines()\n",
    "with codecs.open(filename8, 'r', encoding='utf-8-sig') as f:\n",
    "    lines8 = f.readlines()\n",
    "with codecs.open(filename9, 'r', encoding='utf-8-sig') as f:\n",
    "    lines9 = f.readlines()\n",
    "with codecs.open(filename10, 'r', encoding='utf-8-sig') as f:\n",
    "    lines10 = f.readlines()\n",
    "with codecs.open(filename9, 'r', encoding='utf-8-sig') as f:\n",
    "    lines11 = f.readlines()\n",
    "with codecs.open(filename10, 'r', encoding='utf-8-sig') as f:\n",
    "    lines12 = f.readlines()\n",
    "\n",
    "\n",
    "\n",
    "# test set\n",
    "\n",
    "\n",
    "with codecs.open('./new_data1/definition'+name+'_spynorth_test_t.txt', 'r', 'utf-8-sig') as f:\n",
    "    test1 = f.readlines()\n",
    "with codecs.open('./new_data1/definition'+name+'_spynorth_test_ut.txt', 'r', 'utf-8-sig') as f:\n",
    "    test2 = f.readlines()\n",
    "with codecs.open('./new_data1/definition'+name+'_intistranger_test_t.txt', 'r', encoding='utf-8-sig') as f:\n",
    "    test3 = f.readlines()\n",
    "with codecs.open('./new_data1/definition'+name+'_intistranger_test_ut.txt', 'r', encoding='utf-8-sig') as f:\n",
    "    test4 = f.readlines()\n",
    "with codecs.open('./new_data1/definition'+name+'_assassin_test_t.txt', 'r', encoding='utf-8-sig') as f:\n",
    "    test5 = f.readlines()\n",
    "with codecs.open('./new_data1/definition'+name+'_assassin_test_ut.txt', 'r', encoding='utf-8-sig') as f:\n",
    "    test6 = f.readlines()\n",
    "with codecs.open('./new_data1/definition'+name+'_1987_test_t.txt', 'r', encoding='utf-8-sig') as f:\n",
    "    test7 = f.readlines()\n",
    "with codecs.open('./new_data1/definition'+name+'_1987_test_ut.txt', 'r', encoding='utf-8-sig') as f:\n",
    "    test8 = f.readlines()\n",
    "with codecs.open('./new_data1/definition'+name+'_taxi_test_t.txt', 'r', encoding='utf-8-sig') as f:\n",
    "    test9 = f.readlines()\n",
    "with codecs.open('./new_data1/definition'+name+'_taxi_test_ut.txt', 'r', encoding='utf-8-sig') as f:\n",
    "    test10 = f.readlines()\n",
    "with codecs.open('./new_data1/definition'+name+'_gongjo_test_t.txt', 'r', encoding='utf-8-sig') as f:\n",
    "    test11 = f.readlines()\n",
    "with codecs.open('./new_data1/definition'+name+'_gongjo_test_ut.txt', 'r', encoding='utf-8-sig') as f:\n",
    "    test12 = f.readlines()\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3b51974",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow_text as text\n",
    "import pandas as pd\n",
    "bert_preprocess = hub.KerasLayer(\"https://tfhub.dev/jeongukjae/distilkobert_cased_preprocess/1\")\n",
    "bert_encoder = hub.KerasLayer(\"https://tfhub.dev/jeongukjae/distilkobert_cased_L-3_H-768_A-12/1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a54ce315",
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Input, Dense\n",
    "from keras.models import Model\n",
    "lines_t = lines1[:2560]\n",
    "lines_ut = lines2[:2560]\n",
    "test_t = test1[:640]\n",
    "test_ut = test2[:640]\n",
    "\n",
    "\n",
    "lines_ = []\n",
    "test_lines_ = []\n",
    "\n",
    "lines1_ = []\n",
    "lines2_ = []\n",
    "lines3_ = []\n",
    "lines4_ = []\n",
    "# current three feature\n",
    "feature1 = [] # train\n",
    "feature2 = [] # test\n",
    "# historical three feature\n",
    "feature3 = [] # train\n",
    "feature4 = [] # test\n",
    "# current text\n",
    "current_text1 = [] # train\n",
    "current_text2= [] # test\n",
    "# historical text\n",
    "history_text1 = [] # train\n",
    "history_text2= [] # test\n",
    "\n",
    "rating = []\n",
    "sentiment = []\n",
    "correlation = []\n",
    "## current three feature\n",
    "for line in lines_t:\n",
    "    \n",
    "    try:\n",
    "        # line = line.replace(\"\\n\",\" \")\n",
    "        a = line.split(\",\")[:3]\n",
    "        a = list(map(float, a))\n",
    "        feature1.append(a) # current feature\n",
    "    except:\n",
    "        print(lines_t.index(line))\n",
    "\n",
    "for line in lines_ut:\n",
    "    line = line.replace(\"\\n\",\" \")\n",
    "    a = line.split(\",\")[:3]\n",
    "    a = list(map(float, a))\n",
    "    feature1.append(a)\n",
    "\n",
    "# historical three feature\n",
    "\n",
    "for line in lines_t:\n",
    "    line = line.replace(\"\\n\",\" \")\n",
    "    a = line.split(\",\")[3:10]\n",
    "    a = list(map(float, a))\n",
    "    ravg = a[0] + a[3]\n",
    "    savg = a[1] + a[4]\n",
    "    cavg = a[5] + a[6]\n",
    "    ll = []\n",
    "    ll.append(ravg)\n",
    "    ll.append(savg)\n",
    "    ll.append(cavg)\n",
    "    feature3.append(ll) # current feature\n",
    "\n",
    "for line in lines_ut:\n",
    "    line = line.replace(\"\\n\",\" \")\n",
    "    a = line.split(\",\")[3:10]\n",
    "    a = list(map(float, a))\n",
    "    ravg = a[0] + a[3]\n",
    "    savg = a[1] + a[4]\n",
    "    cavg = a[5] + a[6]\n",
    "    ll = []\n",
    "    ll.append(ravg)\n",
    "    ll.append(savg)\n",
    "    ll.append(cavg)\n",
    "    feature3.append(ll)\n",
    "\n",
    "# current text\n",
    "\n",
    "for line in lines_t:\n",
    "    a = line.split(\",\")[10]\n",
    "    current_text1.append(a.strip()) # current feature\n",
    "\n",
    "for line in lines_ut:\n",
    "    a = line.split(\",\")[10]\n",
    "    current_text1.append(a.strip())\n",
    "    \n",
    "# historical text\n",
    "\n",
    "for line in lines_t:\n",
    "    #print(lines_t.index(line))\n",
    "    a = line.split(\",\")[11]\n",
    "    history_text1.append(a.strip()) # current feature\n",
    "\n",
    "for line in lines_ut:\n",
    "    a = line.split(\",\")[11]\n",
    "    history_text1.append(a.strip())\n",
    "    \n",
    "    \n",
    "# test data\n",
    "# current three\n",
    "for line in test_t:\n",
    "    a = line.split(\",\")[:3]\n",
    "    a = list(map(float, a))\n",
    "    feature2.append(a)\n",
    "    \n",
    "for line in test_ut:\n",
    "    a = line.split(\",\")[:3]\n",
    "    a = list(map(float, a))\n",
    "    feature2.append(a)\n",
    "\n",
    "    \n",
    "# historical three\n",
    "for line in test_t:\n",
    "    a = line.split(\",\")[3:10]\n",
    "    a = list(map(float, a))\n",
    "    ravg = a[0] + a[3]\n",
    "    savg = a[1] + a[4]\n",
    "    cavg = a[5] + a[6]\n",
    "    ll = []\n",
    "    ll.append(ravg)\n",
    "    ll.append(savg)\n",
    "    ll.append(cavg)\n",
    "    feature4.append(ll)\n",
    "    \n",
    "for line in test_ut:\n",
    "    a = line.split(\",\")[3:10]\n",
    "    a = list(map(float, a))\n",
    "    ravg = a[0] + a[3]\n",
    "    savg = a[1] + a[4]\n",
    "    cavg = a[5] + a[6]\n",
    "    ll = []\n",
    "    ll.append(ravg)\n",
    "    ll.append(savg)\n",
    "    ll.append(cavg)\n",
    "    feature4.append(ll) \n",
    "    \n",
    "# current text\n",
    "\n",
    "for line in test_t:\n",
    "    a = line.split(\",\")[10]\n",
    "    current_text2.append(a.strip()) # current feature\n",
    "\n",
    "for line in test_ut:\n",
    "    a = line.split(\",\")[10]\n",
    "    current_text2.append(a.strip())\n",
    "\n",
    "# history text\n",
    "for line in test_t:\n",
    "    \n",
    "    a = line.split(\",\")[11]\n",
    "    history_text2.append(a.strip()) # current feature\n",
    "\n",
    "for line in test_ut:\n",
    "    a = line.split(\",\")[11]\n",
    "    history_text2.append(a.strip())\n",
    "\n",
    "train_labels = [] # train 데이터 label\n",
    "test_labels = [] # test 데이터 label\n",
    "for i in range(len(lines_t)):\n",
    "    train_labels.append(0)\n",
    "for j in range(len(lines_ut)):\n",
    "    train_labels.append(1)\n",
    "for i in range(len(test_t)):\n",
    "    test_labels.append(0)\n",
    "for j in range(len(test_ut)):\n",
    "    test_labels.append(1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e3eb51b",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = list(zip(history_text1,train_labels))\n",
    "test = list(zip(history_text2,test_labels))\n",
    "import pandas as pd\n",
    "train_df = pd.DataFrame(train, columns = ['text', 'label'])\n",
    "test_df = pd.DataFrame(test, columns = ['text', 'label'])\n",
    "\n",
    "X_train = train_df['text']\n",
    "X_test = test_df['text']\n",
    "y_train = train_df['label']\n",
    "y_test = test_df[\"label\"]\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "x_train, x_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, \n",
    "                                                 random_state=1, shuffle=False)\n",
    "\n",
    "with tf.device('/gpu:0'):\n",
    "    text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n",
    "    preprocessed_text = bert_preprocess(text_input)\n",
    "    outputs = bert_encoder(preprocessed_text)\n",
    "    l = tf.keras.layers.Dropout(0.1, name=\"dropout\")(outputs['pooled_output'])\n",
    "    #l = tf.keras.layers.Dense(1, activation='sigmoid', name=\"output\")(l)\n",
    "    #l = tf.keras.layers.Dropout(0.1, name=\"dropout\")(outputs['sequence_output'])\n",
    "    #l = tf.keras.layers.LSTM(128, name=\"lstm\")(l)\n",
    "    l = tf.keras.layers.Dense(1, activation='sigmoid', name=\"output\")(l)\n",
    "    model = tf.keras.Model(inputs=[text_input], outputs = [l])\n",
    "    METRICS = [\n",
    "      tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n",
    "      tf.keras.metrics.Precision(name='precision'),\n",
    "      tf.keras.metrics.Recall(name='recall')\n",
    "    ]\n",
    "\n",
    "    model.compile(optimizer='adam',\n",
    "     loss='binary_crossentropy',\n",
    "     metrics=METRICS)\n",
    "    model.fit(x_train, y_train, epochs=5,batch_size=128, validation_data=(x_val, y_val))\n",
    "    \n",
    "    \n",
    "model.evaluate(X_test, y_test) # definition 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e6b4c84",
   "metadata": {},
   "source": [
    "# LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fafeeccf",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = list(zip(history_text1,train_labels))\n",
    "test = list(zip(history_text2,test_labels))\n",
    "import pandas as pd\n",
    "train_df = pd.DataFrame(train, columns = ['text', 'label'])\n",
    "test_df = pd.DataFrame(test, columns = ['text', 'label'])\n",
    "\n",
    "X_train = train_df['text']\n",
    "X_test = test_df['text']\n",
    "y_train = train_df['label']\n",
    "y_test = test_df[\"label\"]\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "x_train, x_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, \n",
    "                                                 random_state=1, shuffle=False)\n",
    "\n",
    "with tf.device('/gpu:0'):\n",
    "    text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n",
    "    preprocessed_text = bert_preprocess(text_input)\n",
    "    outputs = bert_encoder(preprocessed_text)\n",
    "    l = tf.keras.layers.Dropout(0.1, name=\"dropout\")(outputs['sequence_output'])\n",
    "    #l = tf.keras.layers.Dense(1, activation='sigmoid', name=\"output\")(l)\n",
    "    #l = tf.keras.layers.Dropout(0.1, name=\"dropout\")(outputs['sequence_output'])\n",
    "    #l = tf.keras.layers.LSTM(128, name=\"lstm\")(l)\n",
    "    l = tf.keras.layers.LSTM(1, activation='sigmoid', name=\"output\")(l)\n",
    "    model = tf.keras.Model(inputs=[text_input], outputs = [l])\n",
    "    METRICS = [\n",
    "      tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n",
    "      tf.keras.metrics.Precision(name='precision'),\n",
    "      tf.keras.metrics.Recall(name='recall')\n",
    "    ]\n",
    "\n",
    "    model.compile(optimizer='adam',\n",
    "     loss='binary_crossentropy',\n",
    "     metrics=METRICS)\n",
    "    model.fit(x_train, y_train, epochs=5,batch_size=128, validation_data=(x_val, y_val))\n",
    "    \n",
    "    \n",
    "model.evaluate(X_test, y_test) # definition 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23f3e361",
   "metadata": {},
   "source": [
    "# concat\n",
    "## numerical feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb7b6537",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_feature = np.asarray(feature3) # feature를 벡터화\n",
    "test_feature = np.asarray(feature4)\n",
    "train_labels = np.asarray(train_labels)\n",
    "test_labels = np.asarray(test_labels)\n",
    "train = list(zip(history_text1,train_labels))\n",
    "test = list(zip(history_text2,test_labels))\n",
    "import pandas as pd\n",
    "train_df = pd.DataFrame(train, columns = ['text', 'label'])\n",
    "test_df = pd.DataFrame(test, columns = ['text', 'label'])\n",
    "X_test = test_df['text']\n",
    "y_test = test_df['label']\n",
    "X_train = train_df['text']\n",
    "y_train = train_df['label']\n",
    "# You can embed your sentences as follows\n",
    "listlist = X_train.tolist()\n",
    "sentences = tf.constant(listlist)\n",
    "#print(model(sentences))\n",
    "ttt = model1(sentences)\n",
    "bertdata = ttt.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff15d716",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_feature = np.asarray(feature3) # feature를 벡터화\n",
    "test_feature = np.asarray(feature4)\n",
    "train_labels = np.asarray(train_labels)\n",
    "test_labels = np.asarray(test_labels)\n",
    "train = list(zip(history_text1,train_labels))\n",
    "test = list(zip(history_text2,test_labels))\n",
    "import pandas as pd\n",
    "train_df = pd.DataFrame(train, columns = ['text', 'label'])\n",
    "test_df = pd.DataFrame(test, columns = ['text', 'label'])\n",
    "X_test = test_df['text']\n",
    "y_test = test_df['label']\n",
    "X_train = train_df['text']\n",
    "y_train = train_df['label']\n",
    "# You can embed your sentences as follows\n",
    "listlist = X_train.tolist()\n",
    "sentences = tf.constant(listlist)\n",
    "#print(model(sentences))\n",
    "ttt = model1(sentences)\n",
    "bertdata = ttt.numpy()\n",
    "# You can embed your sentences as follows\n",
    "listlist1 = X_test.tolist()\n",
    "sentences1 = tf.constant(listlist1)\n",
    "ttt1 = model1(sentences1)\n",
    "bertdata_test = ttt1.numpy()\n",
    "#print(model(sentences))\n",
    "\n",
    "train_arr = np.concatenate((train_feature, bertdata), axis=1)\n",
    "test_arr = np.concatenate((test_feature, bertdata_test), axis=1)\n",
    "train_feature = train_arr.reshape(-1,1,771)\n",
    "test_feature = test_arr.reshape(-1,1,771)\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "x_train, x_val, y_train, y_val = train_test_split(train_feature, train_labels, test_size=0.1, \n",
    "                                                 random_state=1, shuffle=False)\n",
    "\n",
    "\n",
    "#from keras.layers import LSTM\n",
    "from tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "#from keras.models import Sequential\n",
    "##from keras.layers import Embedding, SimpleRNN\n",
    "\n",
    "#from keras.layers import Dense\n",
    "\n",
    "#from keras.preprocessing import sequence\n",
    "\n",
    "model = tf.keras.models.Sequential()\n",
    "model.add(tf.keras.layers.LSTM(32, return_sequences=True))\n",
    "#model.add(LSTM(32))\n",
    "# 'binary_crossentropy\n",
    "#model.add(Dense(5))\n",
    "model.add(tf.keras.layers.LSTM(1, activation='sigmoid'))\n",
    "model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "early_stopping = EarlyStopping(monitor='val_acc', patience=10, mode='auto')\n",
    "\n",
    "history = model.fit(x_train, y_train,validation_data=(x_val, y_val),\n",
    "                    epochs=10,\n",
    "                    batch_size=128,callbacks=[early_stopping]\n",
    "                    )\n",
    "\n",
    "model.evaluate(test_feature, test_labels) # definition 1\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}