diff --git a/lm_eval/tasks/mgsm/direct/mgsm_direct_bn.yaml b/lm_eval/tasks/mgsm/direct/mgsm_direct_bn.yaml index 7c43724ed4..d836bfb22a 100644 --- a/lm_eval/tasks/mgsm/direct/mgsm_direct_bn.yaml +++ b/lm_eval/tasks/mgsm/direct/mgsm_direct_bn.yaml @@ -1,8 +1,6 @@ # Generated by utils.py dataset_name: bn -doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"প্রশ্ন: - "+question+"\nAnswer"}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[17:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"প্রশ্ন: "+question+"\nAnswer:"}}{% endif %}' include: direct_yaml task: mgsm_direct_bn diff --git a/lm_eval/tasks/mgsm/direct/mgsm_direct_de.yaml b/lm_eval/tasks/mgsm/direct/mgsm_direct_de.yaml index a523c0ef88..dd78df2d33 100644 --- a/lm_eval/tasks/mgsm/direct/mgsm_direct_de.yaml +++ b/lm_eval/tasks/mgsm/direct/mgsm_direct_de.yaml @@ -1,8 +1,6 @@ # Generated by utils.py dataset_name: de -doc_to_target: '{% if answer is not none %}{{answer[7+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAntwort"}}{% else %}{{"Frage: - "+question+"\nAntwort"}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[29:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAntwort:"}}{% else %}{{"Frage: "+question+"\nAntwort:"}}{% endif %}' include: direct_yaml task: mgsm_direct_de diff --git a/lm_eval/tasks/mgsm/direct/mgsm_direct_en.yaml b/lm_eval/tasks/mgsm/direct/mgsm_direct_en.yaml index 0570267f14..6d1522bb97 100644 --- a/lm_eval/tasks/mgsm/direct/mgsm_direct_en.yaml +++ b/lm_eval/tasks/mgsm/direct/mgsm_direct_en.yaml @@ -1,8 +1,6 @@ # Generated by utils.py dataset_name: en -doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Question: - "+question+"\nAnswer"}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' include: direct_yaml task: mgsm_direct_en diff --git a/lm_eval/tasks/mgsm/direct/mgsm_direct_es.yaml b/lm_eval/tasks/mgsm/direct/mgsm_direct_es.yaml index 9d2f1fa38d..0d550ffa5f 100644 --- a/lm_eval/tasks/mgsm/direct/mgsm_direct_es.yaml +++ b/lm_eval/tasks/mgsm/direct/mgsm_direct_es.yaml @@ -1,8 +1,6 @@ # Generated by utils.py dataset_name: es -doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Pregunta: - "+question+"\nAnswer"}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[23:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nRespuesta:"}}{% else %}{{"Pregunta: "+question+"\nRespuesta:"}}{% endif %}' include: direct_yaml task: mgsm_direct_es diff --git a/lm_eval/tasks/mgsm/direct/mgsm_direct_fr.yaml b/lm_eval/tasks/mgsm/direct/mgsm_direct_fr.yaml index b39d97cec5..fd036698ca 100644 --- a/lm_eval/tasks/mgsm/direct/mgsm_direct_fr.yaml +++ b/lm_eval/tasks/mgsm/direct/mgsm_direct_fr.yaml @@ -1,8 +1,6 @@ # Generated by utils.py dataset_name: fr -doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Question - : "+question+"\nAnswer"}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[26:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nRéponse :"}}{% else %}{{"Question : "+question+"\nRéponse :"}}{% endif %}' include: direct_yaml task: mgsm_direct_fr diff --git a/lm_eval/tasks/mgsm/direct/mgsm_direct_ja.yaml b/lm_eval/tasks/mgsm/direct/mgsm_direct_ja.yaml index 8f81ff4314..c34ba9d0ce 100644 --- a/lm_eval/tasks/mgsm/direct/mgsm_direct_ja.yaml +++ b/lm_eval/tasks/mgsm/direct/mgsm_direct_ja.yaml @@ -1,8 +1,6 @@ # Generated by utils.py dataset_name: ja -doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"問題: "+question+"\nAnswer"}}{% - endif %}' +doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"問題: "+question+"\nAnswer:"}}{% endif %}' include: direct_yaml task: mgsm_direct_ja diff --git a/lm_eval/tasks/mgsm/direct/mgsm_direct_ru.yaml b/lm_eval/tasks/mgsm/direct/mgsm_direct_ru.yaml index 01e7987f79..847774e8fe 100644 --- a/lm_eval/tasks/mgsm/direct/mgsm_direct_ru.yaml +++ b/lm_eval/tasks/mgsm/direct/mgsm_direct_ru.yaml @@ -1,8 +1,6 @@ # Generated by utils.py dataset_name: ru -doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Задача: - "+question+"\nAnswer"}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Задача: "+question+"\nAnswer:"}}{% endif %}' include: direct_yaml task: mgsm_direct_ru diff --git a/lm_eval/tasks/mgsm/direct/mgsm_direct_sw.yaml b/lm_eval/tasks/mgsm/direct/mgsm_direct_sw.yaml index 7f04a7c439..03cc8637de 100644 --- a/lm_eval/tasks/mgsm/direct/mgsm_direct_sw.yaml +++ b/lm_eval/tasks/mgsm/direct/mgsm_direct_sw.yaml @@ -1,8 +1,6 @@ # Generated by utils.py dataset_name: sw -doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Swali: - "+question+"\nAnswer"}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[25:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Swali: "+question+"\nAnswer:"}}{% endif %}' include: direct_yaml task: mgsm_direct_sw diff --git a/lm_eval/tasks/mgsm/direct/mgsm_direct_te.yaml b/lm_eval/tasks/mgsm/direct/mgsm_direct_te.yaml index ce0a146115..5aa1533442 100644 --- a/lm_eval/tasks/mgsm/direct/mgsm_direct_te.yaml +++ b/lm_eval/tasks/mgsm/direct/mgsm_direct_te.yaml @@ -1,8 +1,6 @@ # Generated by utils.py dataset_name: te -doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"ప్రశ్న: - "+question+"\nAnswer"}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[19:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"ప్రశ్న: "+question+"\nAnswer:"}}{% endif %}' include: direct_yaml task: mgsm_direct_te diff --git a/lm_eval/tasks/mgsm/direct/mgsm_direct_th.yaml b/lm_eval/tasks/mgsm/direct/mgsm_direct_th.yaml index 1db7cd33c9..f7e3ce0be3 100644 --- a/lm_eval/tasks/mgsm/direct/mgsm_direct_th.yaml +++ b/lm_eval/tasks/mgsm/direct/mgsm_direct_th.yaml @@ -1,8 +1,6 @@ # Generated by utils.py dataset_name: th -doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"โจทย์: - "+question+"\nAnswer"}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"โจทย์: "+question+"\nAnswer:"}}{% endif %}' include: direct_yaml task: mgsm_direct_th diff --git a/lm_eval/tasks/mgsm/direct/mgsm_direct_zh.yaml b/lm_eval/tasks/mgsm/direct/mgsm_direct_zh.yaml index 1b873be46a..f5daa1ab42 100644 --- a/lm_eval/tasks/mgsm/direct/mgsm_direct_zh.yaml +++ b/lm_eval/tasks/mgsm/direct/mgsm_direct_zh.yaml @@ -1,8 +1,6 @@ # Generated by utils.py dataset_name: zh -doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"问题: "+question+"\nAnswer"}}{% - endif %}' +doc_to_target: '{% if answer is not none %}{{answer[6:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"问题: "+question+"\nAnswer:"}}{% endif %}' include: direct_yaml task: mgsm_direct_zh diff --git a/lm_eval/tasks/mgsm/en_cot/cot_yaml b/lm_eval/tasks/mgsm/en_cot/cot_yaml index e6f96160aa..b2217ad3b7 100644 --- a/lm_eval/tasks/mgsm/en_cot/cot_yaml +++ b/lm_eval/tasks/mgsm/en_cot/cot_yaml @@ -7,7 +7,6 @@ dataset_name: null # Overridden by language-specific config. output_type: generate_until training_split: train test_split: test -target_delimiter: "" generation_kwargs: until: - "\n\n" diff --git a/lm_eval/tasks/mgsm/en_cot/mgsm_bn_en-cot.yaml b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_bn.yaml similarity index 63% rename from lm_eval/tasks/mgsm/en_cot/mgsm_bn_en-cot.yaml rename to lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_bn.yaml index 8cdcf4a47f..27b0e51883 100644 --- a/lm_eval/tasks/mgsm/en_cot/mgsm_bn_en-cot.yaml +++ b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_bn.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: bn -doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[17:]}}{% else %}{{answer_number|string}}{% endif %}' doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"প্রশ্ন: "+question+"\nStep-by-Step Answer:"}}{% endif %}' include: cot_yaml -task: mgsm_bn_direct +task: mgsm_en_cot_bn diff --git a/lm_eval/tasks/mgsm/en_cot/mgsm_de_en-cot.yaml b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_de.yaml similarity index 61% rename from lm_eval/tasks/mgsm/en_cot/mgsm_de_en-cot.yaml rename to lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_de.yaml index 01bc74fc80..75c2aaad37 100644 --- a/lm_eval/tasks/mgsm/en_cot/mgsm_de_en-cot.yaml +++ b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_de.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: de -doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[29:]}}{% else %}{{answer_number|string}}{% endif %}' doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Frage: "+question+"\nStep-by-Step Answer:"}}{% endif %}' include: cot_yaml -task: mgsm_de_direct +task: mgsm_en_cot_de diff --git a/lm_eval/tasks/mgsm/en_cot/mgsm_en_en-cot.yaml b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_en.yaml similarity index 62% rename from lm_eval/tasks/mgsm/en_cot/mgsm_en_en-cot.yaml rename to lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_en.yaml index 110336f1d0..c704619453 100644 --- a/lm_eval/tasks/mgsm/en_cot/mgsm_en_en-cot.yaml +++ b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_en.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: en -doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' include: cot_yaml -task: mgsm_en_direct +task: mgsm_en_cot_en diff --git a/lm_eval/tasks/mgsm/en_cot/mgsm_es_en-cot.yaml b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_es.yaml similarity index 62% rename from lm_eval/tasks/mgsm/en_cot/mgsm_es_en-cot.yaml rename to lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_es.yaml index ed63485119..9f5291bb19 100644 --- a/lm_eval/tasks/mgsm/en_cot/mgsm_es_en-cot.yaml +++ b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_es.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: es -doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[23:]}}{% else %}{{answer_number|string}}{% endif %}' doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Pregunta: "+question+"\nStep-by-Step Answer:"}}{% endif %}' include: cot_yaml -task: mgsm_es_direct +task: mgsm_en_cot_es diff --git a/lm_eval/tasks/mgsm/en_cot/mgsm_fr_en-cot.yaml b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_fr.yaml similarity index 62% rename from lm_eval/tasks/mgsm/en_cot/mgsm_fr_en-cot.yaml rename to lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_fr.yaml index c9241eef28..c8744a617f 100644 --- a/lm_eval/tasks/mgsm/en_cot/mgsm_fr_en-cot.yaml +++ b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_fr.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: fr -doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[26:]}}{% else %}{{answer_number|string}}{% endif %}' doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question : "+question+"\nStep-by-Step Answer:"}}{% endif %}' include: cot_yaml -task: mgsm_fr_direct +task: mgsm_en_cot_fr diff --git a/lm_eval/tasks/mgsm/en_cot/mgsm_ja_en-cot.yaml b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ja.yaml similarity index 62% rename from lm_eval/tasks/mgsm/en_cot/mgsm_ja_en-cot.yaml rename to lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ja.yaml index 184ecfc5d9..fcf6d293b6 100644 --- a/lm_eval/tasks/mgsm/en_cot/mgsm_ja_en-cot.yaml +++ b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ja.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: ja -doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}' doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"問題: "+question+"\nStep-by-Step Answer:"}}{% endif %}' include: cot_yaml -task: mgsm_ja_direct +task: mgsm_en_cot_ja diff --git a/lm_eval/tasks/mgsm/en_cot/mgsm_ru_en-cot.yaml b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ru.yaml similarity index 62% rename from lm_eval/tasks/mgsm/en_cot/mgsm_ru_en-cot.yaml rename to lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ru.yaml index 244d1e1ecc..be4667f742 100644 --- a/lm_eval/tasks/mgsm/en_cot/mgsm_ru_en-cot.yaml +++ b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ru.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: ru -doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}' doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Задача: "+question+"\nStep-by-Step Answer:"}}{% endif %}' include: cot_yaml -task: mgsm_ru_direct +task: mgsm_en_cot_ru diff --git a/lm_eval/tasks/mgsm/en_cot/mgsm_sw_en-cot.yaml b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_sw.yaml similarity index 61% rename from lm_eval/tasks/mgsm/en_cot/mgsm_sw_en-cot.yaml rename to lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_sw.yaml index 4ec09e7f7b..91643455e0 100644 --- a/lm_eval/tasks/mgsm/en_cot/mgsm_sw_en-cot.yaml +++ b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_sw.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: sw -doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[25:]}}{% else %}{{answer_number|string}}{% endif %}' doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Swali: "+question+"\nStep-by-Step Answer:"}}{% endif %}' include: cot_yaml -task: mgsm_sw_direct +task: mgsm_en_cot_sw diff --git a/lm_eval/tasks/mgsm/en_cot/mgsm_te_en-cot.yaml b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_te.yaml similarity index 63% rename from lm_eval/tasks/mgsm/en_cot/mgsm_te_en-cot.yaml rename to lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_te.yaml index 97d2d47779..b87d736154 100644 --- a/lm_eval/tasks/mgsm/en_cot/mgsm_te_en-cot.yaml +++ b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_te.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: te -doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[19:]}}{% else %}{{answer_number|string}}{% endif %}' doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"ప్రశ్న: "+question+"\nStep-by-Step Answer:"}}{% endif %}' include: cot_yaml -task: mgsm_te_direct +task: mgsm_en_cot_te diff --git a/lm_eval/tasks/mgsm/en_cot/mgsm_th_en-cot.yaml b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_th.yaml similarity index 63% rename from lm_eval/tasks/mgsm/en_cot/mgsm_th_en-cot.yaml rename to lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_th.yaml index 2990ff8e7b..33b8be0a0f 100644 --- a/lm_eval/tasks/mgsm/en_cot/mgsm_th_en-cot.yaml +++ b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_th.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: th -doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}' doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"โจทย์: "+question+"\nStep-by-Step Answer:"}}{% endif %}' include: cot_yaml -task: mgsm_th_direct +task: mgsm_en_cot_th diff --git a/lm_eval/tasks/mgsm/en_cot/mgsm_zh_en-cot.yaml b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_zh.yaml similarity index 62% rename from lm_eval/tasks/mgsm/en_cot/mgsm_zh_en-cot.yaml rename to lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_zh.yaml index 8a3b241d7f..516d009abc 100644 --- a/lm_eval/tasks/mgsm/en_cot/mgsm_zh_en-cot.yaml +++ b/lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_zh.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: zh -doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_target: '{% if answer is not none %}{{answer[6:]}}{% else %}{{answer_number|string}}{% endif %}' doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"问题: "+question+"\nStep-by-Step Answer:"}}{% endif %}' include: cot_yaml -task: mgsm_zh_direct +task: mgsm_en_cot_zh diff --git a/lm_eval/tasks/mgsm/gen_yaml.sh b/lm_eval/tasks/mgsm/gen_yaml.sh new file mode 100755 index 0000000000..27cbbcfdc7 --- /dev/null +++ b/lm_eval/tasks/mgsm/gen_yaml.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +python utils.py --overwrite --output-dir direct --mode direct +python utils.py --overwrite --output-dir en_cot --mode en-cot +python utils.py --overwrite --output-dir native_cot --mode native-cot diff --git a/lm_eval/tasks/mgsm/native_cot/cot_yaml b/lm_eval/tasks/mgsm/native_cot/cot_yaml index b95e36af0d..4d79f8301b 100644 --- a/lm_eval/tasks/mgsm/native_cot/cot_yaml +++ b/lm_eval/tasks/mgsm/native_cot/cot_yaml @@ -7,7 +7,7 @@ dataset_name: null # Overridden by language-specific config. output_type: generate_until training_split: train test_split: test -target_delimiter: "" +# target_delimiter: "" generation_kwargs: until: - "\n\n" diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_bn.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_bn.yaml deleted file mode 100644 index 76e52a7ca3..0000000000 --- a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_bn.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Generated by utils.py -dataset_name: bn -doc_to_target: '{% if answer is not none %}{{answer[16+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nধাপে ধাপে উত্তর:"}}{% else - %}{{"প্রশ্ন: "+question+"\nধাপে ধাপে উত্তর:"}}{% endif %}' -include: cot_yaml -task: mgsm_bn_native_cot diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_de.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_de.yaml deleted file mode 100644 index ada27858a7..0000000000 --- a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_de.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Generated by utils.py -dataset_name: de -doc_to_target: '{% if answer is not none %}{{answer[28+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nSchritt-für-Schritt-Antwort:"}}{% - else %}{{"Frage: "+question+"\nSchritt-für-Schritt-Antwort:"}}{% endif %}' -include: cot_yaml -task: mgsm_de_native_cot diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_en.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_en.yaml deleted file mode 100644 index c1fa07f5f0..0000000000 --- a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_en.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Generated by utils.py -dataset_name: en -doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else - %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -include: cot_yaml -task: mgsm_en_native_cot diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_es.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_es.yaml deleted file mode 100644 index 5a0ba219c3..0000000000 --- a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_es.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Generated by utils.py -dataset_name: es -doc_to_target: '{% if answer is not none %}{{answer[22+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nRespuesta paso a paso:"}}{% - else %}{{"Pregunta: "+question+"\nRespuesta paso a paso:"}}{% endif %}' -include: cot_yaml -task: mgsm_es_native_cot diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_fr.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_fr.yaml deleted file mode 100644 index c1ce3ad139..0000000000 --- a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_fr.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Generated by utils.py -dataset_name: fr -doc_to_target: '{% if answer is not none %}{{answer[25+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nRéponse étape par étape :"}}{% - else %}{{"Question : "+question+"\nRéponse étape par étape :"}}{% endif %}' -include: cot_yaml -task: mgsm_fr_native_cot diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_ja.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_ja.yaml deleted file mode 100644 index f2866f5469..0000000000 --- a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_ja.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Generated by utils.py -dataset_name: ja -doc_to_target: '{% if answer is not none %}{{answer[10+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nステップごとの答え:"}}{% else %}{{"問題: - "+question+"\nステップごとの答え:"}}{% endif %}' -include: cot_yaml -task: mgsm_ja_native_cot diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_ru.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_ru.yaml deleted file mode 100644 index 00b00b0973..0000000000 --- a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_ru.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Generated by utils.py -dataset_name: ru -doc_to_target: '{% if answer is not none %}{{answer[17+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nПошаговоерешение:"}}{% else - %}{{"Задача: "+question+"\nПошаговоерешение:"}}{% endif %}' -include: cot_yaml -task: mgsm_ru_native_cot diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_sw.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_sw.yaml deleted file mode 100644 index 693eea2415..0000000000 --- a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_sw.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Generated by utils.py -dataset_name: sw -doc_to_target: '{% if answer is not none %}{{answer[24+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nJibu la Hatua kwa Hatua:"}}{% - else %}{{"Swali: "+question+"\nJibu la Hatua kwa Hatua:"}}{% endif %}' -include: cot_yaml -task: mgsm_sw_native_cot diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_te.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_te.yaml deleted file mode 100644 index 30cab55042..0000000000 --- a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_te.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Generated by utils.py -dataset_name: te -doc_to_target: '{% if answer is not none %}{{answer[18+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nదశలవారీగా సమాధానం:"}}{% else - %}{{"ప్రశ్న: "+question+"\nదశలవారీగా సమాధానం:"}}{% endif %}' -include: cot_yaml -task: mgsm_te_native_cot diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_th.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_th.yaml deleted file mode 100644 index 1e57000099..0000000000 --- a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_th.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Generated by utils.py -dataset_name: th -doc_to_target: '{% if answer is not none %}{{answer[17+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nคำตอบทีละขั้นตอน:"}}{% else - %}{{"โจทย์: "+question+"\nคำตอบทีละขั้นตอน:"}}{% endif %}' -include: cot_yaml -task: mgsm_th_native_cot diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_zh.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_zh.yaml deleted file mode 100644 index 3e1be0b3c3..0000000000 --- a/lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_zh.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Generated by utils.py -dataset_name: zh -doc_to_target: '{% if answer is not none %}{{answer[5+1]}}{% else %}{{answer_number|string}}{% - endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\n逐步解答:"}}{% else %}{{"问题: "+question+"\n逐步解答:"}}{% - endif %}' -include: cot_yaml -task: mgsm_zh_native_cot diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_bn.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_bn.yaml new file mode 100644 index 0000000000..470370287b --- /dev/null +++ b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_bn.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: bn +doc_to_target: '{% if answer is not none %}{{answer[17:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nধাপে ধাপে উত্তর:"}}{% else %}{{"প্রশ্ন: "+question+"\nধাপে ধাপে উত্তর:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: The answer is (\-?[0-9\.\,]+) + - function: take_first + name: get-answer +include: cot_yaml +task: mgsm_native_cot_bn diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_de.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_de.yaml new file mode 100644 index 0000000000..84d774767b --- /dev/null +++ b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_de.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: de +doc_to_target: '{% if answer is not none %}{{answer[29:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nSchritt-für-Schritt-Antwort:"}}{% else %}{{"Frage: "+question+"\nSchritt-für-Schritt-Antwort:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: Die Antwort lautet (\-?[0-9\.\,]+) + - function: take_first + name: get-answer +include: cot_yaml +task: mgsm_native_cot_de diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_en.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_en.yaml new file mode 100644 index 0000000000..4b91e3ebea --- /dev/null +++ b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_en.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: en +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: The answer is (\-?[0-9\.\,]+) + - function: take_first + name: get-answer +include: cot_yaml +task: mgsm_native_cot_en diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_es.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_es.yaml new file mode 100644 index 0000000000..cec340dfd4 --- /dev/null +++ b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_es.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: es +doc_to_target: '{% if answer is not none %}{{answer[23:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nRespuesta paso a paso:"}}{% else %}{{"Pregunta: "+question+"\nRespuesta paso a paso:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: La respuesta es (\-?[0-9\.\,]+) + - function: take_first + name: get-answer +include: cot_yaml +task: mgsm_native_cot_es diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_fr.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_fr.yaml new file mode 100644 index 0000000000..dfefc22c42 --- /dev/null +++ b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_fr.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: fr +doc_to_target: '{% if answer is not none %}{{answer[26:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nRéponse étape par étape :"}}{% else %}{{"Question : "+question+"\nRéponse étape par étape :"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: La réponse est (\-?[0-9\.\,]+) + - function: take_first + name: get-answer +include: cot_yaml +task: mgsm_native_cot_fr diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_ja.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_ja.yaml new file mode 100644 index 0000000000..190cb50047 --- /dev/null +++ b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_ja.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ja +doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nステップごとの答え:"}}{% else %}{{"問題: "+question+"\nステップごとの答え:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: 答えは(\-?[0-9\.\,]+)です。 + - function: take_first + name: get-answer +include: cot_yaml +target_delimiter: "" +task: mgsm_native_cot_ja diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_ru.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_ru.yaml new file mode 100644 index 0000000000..bade2b3da0 --- /dev/null +++ b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_ru.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ru +doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nПошаговоерешение:"}}{% else %}{{"Задача: "+question+"\nПошаговоерешение:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: Ответ — (\-?[0-9\.\,]+) + - function: take_first + name: get-answer +include: cot_yaml +task: mgsm_native_cot_ru diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_sw.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_sw.yaml new file mode 100644 index 0000000000..2299ee610e --- /dev/null +++ b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_sw.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: sw +doc_to_target: '{% if answer is not none %}{{answer[25:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nJibu la Hatua kwa Hatua:"}}{% else %}{{"Swali: "+question+"\nJibu la Hatua kwa Hatua:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: Jibu ni (\-?[0-9\.\,]+) + - function: take_first + name: get-answer +include: cot_yaml +task: mgsm_native_cot_sw diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_te.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_te.yaml new file mode 100644 index 0000000000..53d7dd4893 --- /dev/null +++ b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_te.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: te +doc_to_target: '{% if answer is not none %}{{answer[19:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nదశలవారీగా సమాధానం:"}}{% else %}{{"ప్రశ్న: "+question+"\nదశలవారీగా సమాధానం:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: సమాధానం (\-?[0-9\.\,]+) + - function: take_first + name: get-answer +include: cot_yaml +task: mgsm_native_cot_te diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_th.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_th.yaml new file mode 100644 index 0000000000..75060b5f36 --- /dev/null +++ b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_th.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: th +doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nคำตอบทีละขั้นตอน:"}}{% else %}{{"โจทย์: "+question+"\nคำตอบทีละขั้นตอน:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: คำตอบคือ (\-?[0-9\.\,]+) + - function: take_first + name: get-answer +include: cot_yaml +task: mgsm_native_cot_th diff --git a/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_zh.yaml b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_zh.yaml new file mode 100644 index 0000000000..0ba2d7f71c --- /dev/null +++ b/lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_zh.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: zh +doc_to_target: '{% if answer is not none %}{{answer[6:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\n逐步解答:"}}{% else %}{{"问题: "+question+"\n逐步解答:"}}{% endif %}' +filter_list: +- filter: + - function: regex + regex_pattern: 答案是 (\-?[0-9\.\,]+)。 + - function: take_first + name: get-answer +include: cot_yaml +target_delimiter: "" +task: mgsm_native_cot_zh diff --git a/lm_eval/tasks/mgsm/utils.py b/lm_eval/tasks/mgsm/utils.py index 3edc78ab28..46af2b5591 100644 --- a/lm_eval/tasks/mgsm/utils.py +++ b/lm_eval/tasks/mgsm/utils.py @@ -128,23 +128,25 @@ def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: yaml_template = "cot_yaml" filter_list = {} + DELIMITER = None if mode == "direct": ANSWER = LANGUAGES[lang]["DIRECT"] REGEX = None - task_name = f"mgsm_{lang}_direct" + task_name = f"mgsm_direct_{lang}" yaml_template = "direct_yaml" elif mode == "native-cot": ANSWER = LANGUAGES[lang]["ANSWER"] REGEX = LANGUAGES[lang]["REGEX"] - task_name = f"mgsm_{lang}_native-cot" + task_name = f"mgsm_native_cot_{lang}" filter_list = add_regex_pattern(REGEX) + DELIMITER = "" if lang in ["zh", "ja"] elif mode == "en-cot": ANSWER = LANGUAGES["en"]["ANSWER"] REGEX = LANGUAGES["en"]["REGEX"] - task_name = f"mgsm_{lang}_en-cot" + task_name = f"mgsm_en_cot_{lang}" file_name = f"{task_name}.yaml" - + ANSWER_TO_SKIP = len(LANGUAGES[lang]["ANSWER"])+1 with open( f"{output_dir}/{file_name}", "w" if overwrite else "x", encoding="utf8" ) as f: @@ -153,18 +155,19 @@ def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: { "include": yaml_template, "dataset_name": lang, - "task": f"mgsm_{lang}_direct", + "task": f"{task_name}", "doc_to_text": f"""{{% if answer is not none %}}""" f"""{{{{question+"\\n{ANSWER}"}}}}""" f"""{{% else %}}""" f"""{{{{"{QUESTION} "+question+"\\n{ANSWER}"}}}}""" f"""{{% endif %}}""", "doc_to_target": f"""{{% if answer is not none %}}""" - f"""{{{{answer[{len(ANSWER)}+1]}}}}""" + f"""{{{{answer[{ANSWER_TO_SKIP}:]}}}}""" f"""{{% else %}}""" f"""{{{{answer_number|string}}}}""" f"""{{% endif %}}""", **filter_list, + **({"target_delimiter": DELIMITER} if DELIMITER else {}), }, f, allow_unicode=True,