minor changes in engines

Querent-ai · Apr 19, 2024 · 732f64b · 732f64b
1 parent 34e229c
commit 732f64b
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 8 deletions.
diff --git a/querent/core/transformers/bert_ner_opensourcellm.py b/querent/core/transformers/bert_ner_opensourcellm.py
@@ -138,9 +138,7 @@ async def process_tokens(self, data: IngestedTokens):
                     self.set_termination_event()                                      
                     return
             if data.data:
-                single_string = ' '.join(data.data)
-                # clean_text = unidecode(single_string)
-                clean_text = single_string
+                clean_text = ' '.join(data.data)
             else:
                 clean_text = data.data
             if not data.is_token_stream : 

diff --git a/querent/core/transformers/fixed_entities_set_opensourcellm.py b/querent/core/transformers/fixed_entities_set_opensourcellm.py
@@ -136,9 +136,7 @@ async def process_tokens(self, data: IngestedTokens):
                     self.set_termination_event()                                      
                     return
             if data.data:
-                single_string = ' '.join(data.data)
-                clean_text = single_string
-                # clean_text = unidecode(single_string)
+                clean_text = ' '.join(data.data)
             else:
                 clean_text = data.data
             if not data.is_token_stream : 

diff --git a/querent/core/transformers/gpt_llm_bert_ner_or_fixed_entities_set_ner.py b/querent/core/transformers/gpt_llm_bert_ner_or_fixed_entities_set_ner.py
@@ -332,6 +332,8 @@ async def process_tokens(self, data: IngestedTokens):
                                     await self.set_state(new_state=current_state)
                         else:
                             return
+                else:
+                    return
         except Exception as e:
             self.logger.error(f"Invalid {self.__class__.__name__} configuration. Unable to extract predicates using GPT. {e}")
             raise Exception(f"An error occurred while extracting predicates using GPT: {e}")

diff --git a/querent/core/transformers/gpt_llm_gpt_ner.py b/querent/core/transformers/gpt_llm_gpt_ner.py
@@ -187,7 +187,6 @@ async def process_tokens(self, data: IngestedTokens):
             doc_source = data.doc_source
             if data.data:
                 clean_text = ' '.join(data.data)
-                #clean_text = unidecode(single_string)
             else:
                 clean_text = data.data
             if not data.is_token_stream : 
@@ -258,7 +257,8 @@ async def process_tokens(self, data: IngestedTokens):
                                     await self.set_state(new_state=current_state)
                         else:
                             return
-
+            else:
+                return
         except Exception as e:
             self.logger.debug(f"Invalid {self.__class__.__name__} configuration. Unable to extract predicates using GPT NER LLM class. {e}")