From 732f64b63a36f9587bd410431ab02634dfbbc445 Mon Sep 17 00:00:00 2001 From: ngupta10 Date: Fri, 19 Apr 2024 18:17:51 +0530 Subject: [PATCH] minor changes in engines --- querent/core/transformers/bert_ner_opensourcellm.py | 4 +--- querent/core/transformers/fixed_entities_set_opensourcellm.py | 4 +--- .../gpt_llm_bert_ner_or_fixed_entities_set_ner.py | 2 ++ querent/core/transformers/gpt_llm_gpt_ner.py | 4 ++-- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/querent/core/transformers/bert_ner_opensourcellm.py b/querent/core/transformers/bert_ner_opensourcellm.py index eab88532..2b8611f2 100644 --- a/querent/core/transformers/bert_ner_opensourcellm.py +++ b/querent/core/transformers/bert_ner_opensourcellm.py @@ -138,9 +138,7 @@ async def process_tokens(self, data: IngestedTokens): self.set_termination_event() return if data.data: - single_string = ' '.join(data.data) - # clean_text = unidecode(single_string) - clean_text = single_string + clean_text = ' '.join(data.data) else: clean_text = data.data if not data.is_token_stream : diff --git a/querent/core/transformers/fixed_entities_set_opensourcellm.py b/querent/core/transformers/fixed_entities_set_opensourcellm.py index 5a832803..57f848a4 100644 --- a/querent/core/transformers/fixed_entities_set_opensourcellm.py +++ b/querent/core/transformers/fixed_entities_set_opensourcellm.py @@ -136,9 +136,7 @@ async def process_tokens(self, data: IngestedTokens): self.set_termination_event() return if data.data: - single_string = ' '.join(data.data) - clean_text = single_string - # clean_text = unidecode(single_string) + clean_text = ' '.join(data.data) else: clean_text = data.data if not data.is_token_stream : diff --git a/querent/core/transformers/gpt_llm_bert_ner_or_fixed_entities_set_ner.py b/querent/core/transformers/gpt_llm_bert_ner_or_fixed_entities_set_ner.py index 29c5da13..fa210559 100644 --- a/querent/core/transformers/gpt_llm_bert_ner_or_fixed_entities_set_ner.py +++ b/querent/core/transformers/gpt_llm_bert_ner_or_fixed_entities_set_ner.py @@ -332,6 +332,8 @@ async def process_tokens(self, data: IngestedTokens): await self.set_state(new_state=current_state) else: return + else: + return except Exception as e: self.logger.error(f"Invalid {self.__class__.__name__} configuration. Unable to extract predicates using GPT. {e}") raise Exception(f"An error occurred while extracting predicates using GPT: {e}") diff --git a/querent/core/transformers/gpt_llm_gpt_ner.py b/querent/core/transformers/gpt_llm_gpt_ner.py index 28cf9e52..aed5a360 100644 --- a/querent/core/transformers/gpt_llm_gpt_ner.py +++ b/querent/core/transformers/gpt_llm_gpt_ner.py @@ -187,7 +187,6 @@ async def process_tokens(self, data: IngestedTokens): doc_source = data.doc_source if data.data: clean_text = ' '.join(data.data) - #clean_text = unidecode(single_string) else: clean_text = data.data if not data.is_token_stream : @@ -258,7 +257,8 @@ async def process_tokens(self, data: IngestedTokens): await self.set_state(new_state=current_state) else: return - + else: + return except Exception as e: self.logger.debug(f"Invalid {self.__class__.__name__} configuration. Unable to extract predicates using GPT NER LLM class. {e}")