PR fixing the issue EleutherAI#1391 (wrong contexts in the mgsm task) (…

…EleutherAI#1440)

* fix the issue EleutherAI#1391, wrong contexts in mgsm tasks

* fix yaml issue for having two target_delimiter lines. For COT tasks, keep the one with a space (default)

* regenerate all task yaml files
- change naming so that file name will match with task name
- task|file follows a consistent naming way, mgsm_(mode)_(lang) for three modes, i.e., direct, en_cot, and native_cot

* English CoTs should have a space as target_delimiter

* Update utils.py

* Apply suggestions from code review

---------

Co-authored-by: Hailey Schoelkopf <[email protected]>

lm_eval/tasks/mgsm/direct/mgsm_direct_bn.yaml

@@ -1,8 +1,6 @@
 # Generated by utils.py
 dataset_name: bn
-doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{%
-  endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"প্রশ্ন:
-  "+question+"\nAnswer"}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[17:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"প্রশ্ন: "+question+"\nAnswer:"}}{% endif %}'
 include: direct_yaml
 task: mgsm_direct_bn

lm_eval/tasks/mgsm/direct/mgsm_direct_de.yaml

@@ -1,8 +1,6 @@
 # Generated by utils.py
 dataset_name: de
-doc_to_target: '{% if answer is not none %}{{answer[7+1]}}{% else %}{{answer_number|string}}{%
-  endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAntwort"}}{% else %}{{"Frage:
-  "+question+"\nAntwort"}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[29:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAntwort:"}}{% else %}{{"Frage: "+question+"\nAntwort:"}}{% endif %}'
 include: direct_yaml
 task: mgsm_direct_de

lm_eval/tasks/mgsm/direct/mgsm_direct_en.yaml

@@ -1,8 +1,6 @@
 # Generated by utils.py
 dataset_name: en
-doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{%
-  endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Question:
-  "+question+"\nAnswer"}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
 include: direct_yaml
 task: mgsm_direct_en

lm_eval/tasks/mgsm/direct/mgsm_direct_es.yaml

@@ -1,8 +1,6 @@
 # Generated by utils.py
 dataset_name: es
-doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{%
-  endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Pregunta:
-  "+question+"\nAnswer"}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[23:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nRespuesta:"}}{% else %}{{"Pregunta: "+question+"\nRespuesta:"}}{% endif %}'
 include: direct_yaml
 task: mgsm_direct_es

lm_eval/tasks/mgsm/direct/mgsm_direct_fr.yaml

@@ -1,8 +1,6 @@
 # Generated by utils.py
 dataset_name: fr
-doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{%
-  endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Question
-  : "+question+"\nAnswer"}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[26:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nRéponse :"}}{% else %}{{"Question : "+question+"\nRéponse :"}}{% endif %}'
 include: direct_yaml
 task: mgsm_direct_fr

lm_eval/tasks/mgsm/direct/mgsm_direct_ja.yaml

@@ -1,8 +1,6 @@
 # Generated by utils.py
 dataset_name: ja
-doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{%
-  endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"問題: "+question+"\nAnswer"}}{%
-  endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"問題: "+question+"\nAnswer:"}}{% endif %}'
 include: direct_yaml
 task: mgsm_direct_ja

lm_eval/tasks/mgsm/direct/mgsm_direct_ru.yaml

@@ -1,8 +1,6 @@
 # Generated by utils.py
 dataset_name: ru
-doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{%
-  endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Задача:
-  "+question+"\nAnswer"}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Задача: "+question+"\nAnswer:"}}{% endif %}'
 include: direct_yaml
 task: mgsm_direct_ru

lm_eval/tasks/mgsm/direct/mgsm_direct_sw.yaml

@@ -1,8 +1,6 @@
 # Generated by utils.py
 dataset_name: sw
-doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{%
-  endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Swali:
-  "+question+"\nAnswer"}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[25:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Swali: "+question+"\nAnswer:"}}{% endif %}'
 include: direct_yaml
 task: mgsm_direct_sw

lm_eval/tasks/mgsm/direct/mgsm_direct_te.yaml

@@ -1,8 +1,6 @@
 # Generated by utils.py
 dataset_name: te
-doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{%
-  endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"ప్రశ్న:
-  "+question+"\nAnswer"}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[19:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"ప్రశ్న: "+question+"\nAnswer:"}}{% endif %}'
 include: direct_yaml
 task: mgsm_direct_te

lm_eval/tasks/mgsm/direct/mgsm_direct_th.yaml

@@ -1,8 +1,6 @@
 # Generated by utils.py
 dataset_name: th
-doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{%
-  endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"โจทย์:
-  "+question+"\nAnswer"}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"โจทย์: "+question+"\nAnswer:"}}{% endif %}'
 include: direct_yaml
 task: mgsm_direct_th

lm_eval/tasks/mgsm/direct/mgsm_direct_zh.yaml

@@ -1,8 +1,6 @@
 # Generated by utils.py
 dataset_name: zh
-doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{%
-  endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"问题: "+question+"\nAnswer"}}{%
-  endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[6:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"问题: "+question+"\nAnswer:"}}{% endif %}'
 include: direct_yaml
 task: mgsm_direct_zh

lm_eval/tasks/mgsm/en_cot/cot_yaml

@@ -7,7 +7,6 @@ dataset_name: null  # Overridden by language-specific config.
 output_type: generate_until
 training_split: train
 test_split: test
-target_delimiter: ""
 generation_kwargs:
   until:
     - "\n\n"

lm_eval/tasks/mgsm/en_cot/mgsm_bn_en-cot.yaml → lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_bn.yaml

@@ -1,6 +1,6 @@
 # Generated by utils.py
 dataset_name: bn
-doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[17:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"প্রশ্ন: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_bn_direct
+task: mgsm_en_cot_bn

lm_eval/tasks/mgsm/en_cot/mgsm_de_en-cot.yaml → lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_de.yaml

@@ -1,6 +1,6 @@
 # Generated by utils.py
 dataset_name: de
-doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[29:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Frage: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_de_direct
+task: mgsm_en_cot_de

lm_eval/tasks/mgsm/en_cot/mgsm_en_en-cot.yaml → lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_en.yaml

@@ -1,6 +1,6 @@
 # Generated by utils.py
 dataset_name: en
-doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_en_direct
+task: mgsm_en_cot_en

lm_eval/tasks/mgsm/en_cot/mgsm_es_en-cot.yaml → lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_es.yaml

@@ -1,6 +1,6 @@
 # Generated by utils.py
 dataset_name: es
-doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[23:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Pregunta: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_es_direct
+task: mgsm_en_cot_es

lm_eval/tasks/mgsm/en_cot/mgsm_fr_en-cot.yaml → lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_fr.yaml

@@ -1,6 +1,6 @@
 # Generated by utils.py
 dataset_name: fr
-doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[26:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question : "+question+"\nStep-by-Step Answer:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_fr_direct
+task: mgsm_en_cot_fr

lm_eval/tasks/mgsm/en_cot/mgsm_ja_en-cot.yaml → lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ja.yaml

@@ -1,6 +1,6 @@
 # Generated by utils.py
 dataset_name: ja
-doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"問題: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_ja_direct
+task: mgsm_en_cot_ja

lm_eval/tasks/mgsm/en_cot/mgsm_ru_en-cot.yaml → lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ru.yaml

@@ -1,6 +1,6 @@
 # Generated by utils.py
 dataset_name: ru
-doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Задача: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_ru_direct
+task: mgsm_en_cot_ru

lm_eval/tasks/mgsm/en_cot/mgsm_sw_en-cot.yaml → lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_sw.yaml

@@ -1,6 +1,6 @@
 # Generated by utils.py
 dataset_name: sw
-doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[25:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Swali: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_sw_direct
+task: mgsm_en_cot_sw

lm_eval/tasks/mgsm/en_cot/mgsm_te_en-cot.yaml → lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_te.yaml

@@ -1,6 +1,6 @@
 # Generated by utils.py
 dataset_name: te
-doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[19:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"ప్రశ్న: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_te_direct
+task: mgsm_en_cot_te

lm_eval/tasks/mgsm/en_cot/mgsm_th_en-cot.yaml → lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_th.yaml

@@ -1,6 +1,6 @@
 # Generated by utils.py
 dataset_name: th
-doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"โจทย์: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_th_direct
+task: mgsm_en_cot_th

lm_eval/tasks/mgsm/en_cot/mgsm_zh_en-cot.yaml → lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_zh.yaml

@@ -1,6 +1,6 @@
 # Generated by utils.py
 dataset_name: zh
-doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_target: '{% if answer is not none %}{{answer[6:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"问题: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_zh_direct
+task: mgsm_en_cot_zh

lm_eval/tasks/mgsm/gen_yaml.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+python utils.py --overwrite --output-dir direct --mode direct
+python utils.py --overwrite --output-dir en_cot --mode en-cot
+python utils.py --overwrite --output-dir native_cot --mode native-cot

lm_eval/tasks/mgsm/native_cot/cot_yaml

@@ -7,7 +7,7 @@ dataset_name: null  # Overridden by language-specific config.
 output_type: generate_until
 training_split: train
 test_split: test
-target_delimiter: ""
+# target_delimiter: ""
 generation_kwargs:
   until:
     - "\n\n"

lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_bn.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: bn
+doc_to_target: '{% if answer is not none %}{{answer[17:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nধাপে ধাপে উত্তর:"}}{% else %}{{"প্রশ্ন: "+question+"\nধাপে ধাপে উত্তর:"}}{% endif %}'
+filter_list:
+- filter:
+  - function: regex
+    regex_pattern: The answer is (\-?[0-9\.\,]+)
+  - function: take_first
+  name: get-answer
+include: cot_yaml
+task: mgsm_native_cot_bn

lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_de.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: de
+doc_to_target: '{% if answer is not none %}{{answer[29:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nSchritt-für-Schritt-Antwort:"}}{% else %}{{"Frage: "+question+"\nSchritt-für-Schritt-Antwort:"}}{% endif %}'
+filter_list:
+- filter:
+  - function: regex
+    regex_pattern: Die Antwort lautet (\-?[0-9\.\,]+)
+  - function: take_first
+  name: get-answer
+include: cot_yaml
+task: mgsm_native_cot_de

lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_en.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: en
+doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+filter_list:
+- filter:
+  - function: regex
+    regex_pattern: The answer is (\-?[0-9\.\,]+)
+  - function: take_first
+  name: get-answer
+include: cot_yaml
+task: mgsm_native_cot_en

lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_es.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: es
+doc_to_target: '{% if answer is not none %}{{answer[23:]}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nRespuesta paso a paso:"}}{% else %}{{"Pregunta: "+question+"\nRespuesta paso a paso:"}}{% endif %}'
+filter_list:
+- filter:
+  - function: regex
+    regex_pattern: La respuesta es (\-?[0-9\.\,]+)
+  - function: take_first
+  name: get-answer
+include: cot_yaml
+task: mgsm_native_cot_es