-
Notifications
You must be signed in to change notification settings - Fork 6
/
retrieval_elasticsearch_setup.py
78 lines (65 loc) · 2.7 KB
/
retrieval_elasticsearch_setup.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
from elasticsearch import Elasticsearch
from tqdm import tqdm
import utils
import argparse
import json
class ElasticSearchClient:
def __init__(self, args):
self._ES_URL = "localhost:9200"
self.es_client = Elasticsearch(self._ES_URL, timeout = 30, max_retries=10, retry_on_timeout=True)
self.index = args.index_name
print(f'Ping Elasticsearch Server : {self.es_client.ping}')
print(f'Checking elastic Search index')
self.create_index(self.index, args)
self.build_wiki(self.index, args.data_path)
def create_index(self, index, args):
if self.es_client.indices.exists(index):
print(f"Delete Existing Index : {index}")
self.es_client.indices.delete(index=args.index_name)
print(f"Creating Index : {index}")
self.es_client.indices.create(index=index, body=args.elasticsearch_config)
print(f"Finishing Creating Index : {index}")
else:
print(f"Creating Index : {index}")
self.es_client.indices.create(index=index, body=args.elasticsearch_config)
print(f"Finishing Creating Index : {index}")
def load_wiki(self, data_path):
with open(data_path, "r", encoding="utf-8") as f:
wiki = json.load(f)
# 순서대로 중복 제거
# ('key1', 'key2', 'key3', 'key1', 'key4', 'key2') -> ['key1', 'key2', 'key3', 'key4']
contexts = list(dict.fromkeys([v["text"] for v in wiki.values()])) # set 은 매번 순서가 바뀌므로
wiki_contexts = [
{"document_text": contexts[i]} for i in range(len(contexts))
]
return wiki_contexts
def build_wiki(self, index, data_path):
wiki_doc_lst = self.load_wiki(data_path)
# Inserting wiki data
for i, rec in enumerate(tqdm(wiki_doc_lst)):
try:
self.es_client.index(index=index, id=i, body=rec)
except:
print(f"Unable to load document {i}.")
n_records = self.es_client.count(index=index)["count"]
print(f"Succesfully loaded {n_records} into {index}")
return 1
def main(args):
print("Setting elasticsearch Server and Index")
if ElasticSearchClient(args):
print("Finish")
else:
print("Error Occurred")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"--config_elasticsearch",
type=str,
default="./config/elasticsearch/elasticsearch_config_1.json",
help="Elastic search configuration file",
)
args = parser.parse_args()
config = utils.read_json(args.config_elasticsearch)
parser.set_defaults(**config)
args = parser.parse_args()
main(args)