Skip LiveQA and MedicationQA annotators if generated output is empty (s…

…tanford-crfm#2732)
AI-secure · Jun 11, 2024 · f806800 · f806800
1 parent a747a40
commit f806800
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 0 deletions.
diff --git a/src/helm/benchmark/annotation/live_qa_annotator.py b/src/helm/benchmark/annotation/live_qa_annotator.py
@@ -26,6 +26,8 @@ def annotate(self, request_state: RequestState) -> Any:
         assert len(request_state.result.completions) == 1
         model_input_text = request_state.request.prompt
         model_output_text = request_state.result.completions[0].text
+        if not model_output_text.strip():
+            return {"prompt_text": "", "reasoning": "BLOCKED_REQUEST_OR_EMPTY_RESPONSE", "score": 0.0}
         correct_responses = ""
         for i, reference in enumerate(request_state.instance.references):
             correct_responses += "Correct Response #" + str(i) + ": " + reference.output.text + "\n"

diff --git a/src/helm/benchmark/annotation/medication_qa_annotator.py b/src/helm/benchmark/annotation/medication_qa_annotator.py
@@ -26,6 +26,8 @@ def annotate(self, request_state: RequestState) -> Any:
         assert len(request_state.result.completions) == 1
         model_input_text = request_state.request.prompt
         model_output_text = request_state.result.completions[0].text
+        if not model_output_text.strip():
+            return {"prompt_text": "", "reasoning": "BLOCKED_REQUEST_OR_EMPTY_RESPONSE", "score": 0.0}
         correct_response = request_state.instance.references[0].output.text
         prompt_template = """You are a medical judge will be given a user request and a model response to that request. Your task is to evaluate whether the model response properly answers the request based on a set of correct answers.
 The user's request will be provided in these tags: