From ca2dd8b19ac9748875a078df828e6ea4f470a410 Mon Sep 17 00:00:00 2001 From: Giulio Lovisotto Date: Tue, 17 Sep 2024 16:18:32 +0200 Subject: [PATCH] First commit. --- .../tasks/openai_mmlu/_default_template.yaml | 15 ++ .../tasks/openai_mmlu/_generate_configs.py | 164 ++++++++++++++++++ .../openai_mmlu/ar_xy/_openai_mmlu_ar_xy.yaml | 9 + .../ar_xy/openai_mmlu_abstract_algebra.yaml | 8 + .../ar_xy/openai_mmlu_anatomy.yaml | 8 + .../ar_xy/openai_mmlu_astronomy.yaml | 8 + .../ar_xy/openai_mmlu_business_ethics.yaml | 8 + .../ar_xy/openai_mmlu_clinical_knowledge.yaml | 8 + .../ar_xy/openai_mmlu_college_biology.yaml | 8 + .../ar_xy/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../ar_xy/openai_mmlu_college_medicine.yaml | 8 + .../ar_xy/openai_mmlu_college_physics.yaml | 8 + .../ar_xy/openai_mmlu_computer_security.yaml | 8 + .../ar_xy/openai_mmlu_conceptual_physics.yaml | 8 + .../ar_xy/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../ar_xy/openai_mmlu_formal_logic.yaml | 8 + .../ar_xy/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../ar_xy/openai_mmlu_human_aging.yaml | 8 + .../ar_xy/openai_mmlu_human_sexuality.yaml | 8 + .../ar_xy/openai_mmlu_international_law.yaml | 8 + .../ar_xy/openai_mmlu_jurisprudence.yaml | 8 + .../ar_xy/openai_mmlu_logical_fallacies.yaml | 8 + .../ar_xy/openai_mmlu_machine_learning.yaml | 8 + .../ar_xy/openai_mmlu_management.yaml | 8 + .../ar_xy/openai_mmlu_marketing.yaml | 8 + .../ar_xy/openai_mmlu_medical_genetics.yaml | 8 + .../ar_xy/openai_mmlu_miscellaneous.yaml | 8 + .../ar_xy/openai_mmlu_moral_disputes.yaml | 8 + .../ar_xy/openai_mmlu_moral_scenarios.yaml | 8 + .../ar_xy/openai_mmlu_nutrition.yaml | 8 + .../ar_xy/openai_mmlu_philosophy.yaml | 8 + .../ar_xy/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../ar_xy/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../ar_xy/openai_mmlu_public_relations.yaml | 8 + .../ar_xy/openai_mmlu_security_studies.yaml | 8 + .../ar_xy/openai_mmlu_sociology.yaml | 8 + .../ar_xy/openai_mmlu_us_foreign_policy.yaml | 8 + .../ar_xy/openai_mmlu_virology.yaml | 8 + .../ar_xy/openai_mmlu_world_religions.yaml | 8 + .../openai_mmlu/bn_bd/_openai_mmlu_bn_bd.yaml | 9 + .../bn_bd/openai_mmlu_abstract_algebra.yaml | 8 + .../bn_bd/openai_mmlu_anatomy.yaml | 8 + .../bn_bd/openai_mmlu_astronomy.yaml | 8 + .../bn_bd/openai_mmlu_business_ethics.yaml | 8 + .../bn_bd/openai_mmlu_clinical_knowledge.yaml | 8 + .../bn_bd/openai_mmlu_college_biology.yaml | 8 + .../bn_bd/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../bn_bd/openai_mmlu_college_medicine.yaml | 8 + .../bn_bd/openai_mmlu_college_physics.yaml | 8 + .../bn_bd/openai_mmlu_computer_security.yaml | 8 + .../bn_bd/openai_mmlu_conceptual_physics.yaml | 8 + .../bn_bd/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../bn_bd/openai_mmlu_formal_logic.yaml | 8 + .../bn_bd/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../bn_bd/openai_mmlu_human_aging.yaml | 8 + .../bn_bd/openai_mmlu_human_sexuality.yaml | 8 + .../bn_bd/openai_mmlu_international_law.yaml | 8 + .../bn_bd/openai_mmlu_jurisprudence.yaml | 8 + .../bn_bd/openai_mmlu_logical_fallacies.yaml | 8 + .../bn_bd/openai_mmlu_machine_learning.yaml | 8 + .../bn_bd/openai_mmlu_management.yaml | 8 + .../bn_bd/openai_mmlu_marketing.yaml | 8 + .../bn_bd/openai_mmlu_medical_genetics.yaml | 8 + .../bn_bd/openai_mmlu_miscellaneous.yaml | 8 + .../bn_bd/openai_mmlu_moral_disputes.yaml | 8 + .../bn_bd/openai_mmlu_moral_scenarios.yaml | 8 + .../bn_bd/openai_mmlu_nutrition.yaml | 8 + .../bn_bd/openai_mmlu_philosophy.yaml | 8 + .../bn_bd/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../bn_bd/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../bn_bd/openai_mmlu_public_relations.yaml | 8 + .../bn_bd/openai_mmlu_security_studies.yaml | 8 + .../bn_bd/openai_mmlu_sociology.yaml | 8 + .../bn_bd/openai_mmlu_us_foreign_policy.yaml | 8 + .../bn_bd/openai_mmlu_virology.yaml | 8 + .../bn_bd/openai_mmlu_world_religions.yaml | 8 + .../openai_mmlu/de_de/_openai_mmlu_de_de.yaml | 9 + .../de_de/openai_mmlu_abstract_algebra.yaml | 8 + .../de_de/openai_mmlu_anatomy.yaml | 8 + .../de_de/openai_mmlu_astronomy.yaml | 8 + .../de_de/openai_mmlu_business_ethics.yaml | 8 + .../de_de/openai_mmlu_clinical_knowledge.yaml | 8 + .../de_de/openai_mmlu_college_biology.yaml | 8 + .../de_de/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../de_de/openai_mmlu_college_medicine.yaml | 8 + .../de_de/openai_mmlu_college_physics.yaml | 8 + .../de_de/openai_mmlu_computer_security.yaml | 8 + .../de_de/openai_mmlu_conceptual_physics.yaml | 8 + .../de_de/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../de_de/openai_mmlu_formal_logic.yaml | 8 + .../de_de/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../de_de/openai_mmlu_human_aging.yaml | 8 + .../de_de/openai_mmlu_human_sexuality.yaml | 8 + .../de_de/openai_mmlu_international_law.yaml | 8 + .../de_de/openai_mmlu_jurisprudence.yaml | 8 + .../de_de/openai_mmlu_logical_fallacies.yaml | 8 + .../de_de/openai_mmlu_machine_learning.yaml | 8 + .../de_de/openai_mmlu_management.yaml | 8 + .../de_de/openai_mmlu_marketing.yaml | 8 + .../de_de/openai_mmlu_medical_genetics.yaml | 8 + .../de_de/openai_mmlu_miscellaneous.yaml | 8 + .../de_de/openai_mmlu_moral_disputes.yaml | 8 + .../de_de/openai_mmlu_moral_scenarios.yaml | 8 + .../de_de/openai_mmlu_nutrition.yaml | 8 + .../de_de/openai_mmlu_philosophy.yaml | 8 + .../de_de/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../de_de/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../de_de/openai_mmlu_public_relations.yaml | 8 + .../de_de/openai_mmlu_security_studies.yaml | 8 + .../de_de/openai_mmlu_sociology.yaml | 8 + .../de_de/openai_mmlu_us_foreign_policy.yaml | 8 + .../de_de/openai_mmlu_virology.yaml | 8 + .../de_de/openai_mmlu_world_religions.yaml | 8 + .../openai_mmlu/en_us/_openai_mmlu_en_us.yaml | 9 + .../en_us/openai_mmlu_abstract_algebra.yaml | 8 + .../en_us/openai_mmlu_anatomy.yaml | 8 + .../en_us/openai_mmlu_astronomy.yaml | 8 + .../en_us/openai_mmlu_business_ethics.yaml | 8 + .../en_us/openai_mmlu_clinical_knowledge.yaml | 8 + .../en_us/openai_mmlu_college_biology.yaml | 8 + .../en_us/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../en_us/openai_mmlu_college_medicine.yaml | 8 + .../en_us/openai_mmlu_college_physics.yaml | 8 + .../en_us/openai_mmlu_computer_security.yaml | 8 + .../en_us/openai_mmlu_conceptual_physics.yaml | 8 + .../en_us/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../en_us/openai_mmlu_formal_logic.yaml | 8 + .../en_us/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../en_us/openai_mmlu_human_aging.yaml | 8 + .../en_us/openai_mmlu_human_sexuality.yaml | 8 + .../en_us/openai_mmlu_international_law.yaml | 8 + .../en_us/openai_mmlu_jurisprudence.yaml | 8 + .../en_us/openai_mmlu_logical_fallacies.yaml | 8 + .../en_us/openai_mmlu_machine_learning.yaml | 8 + .../en_us/openai_mmlu_management.yaml | 8 + .../en_us/openai_mmlu_marketing.yaml | 8 + .../en_us/openai_mmlu_medical_genetics.yaml | 8 + .../en_us/openai_mmlu_miscellaneous.yaml | 8 + .../en_us/openai_mmlu_moral_disputes.yaml | 8 + .../en_us/openai_mmlu_moral_scenarios.yaml | 8 + .../en_us/openai_mmlu_nutrition.yaml | 8 + .../en_us/openai_mmlu_philosophy.yaml | 8 + .../en_us/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../en_us/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../en_us/openai_mmlu_public_relations.yaml | 8 + .../en_us/openai_mmlu_security_studies.yaml | 8 + .../en_us/openai_mmlu_sociology.yaml | 8 + .../en_us/openai_mmlu_us_foreign_policy.yaml | 8 + .../en_us/openai_mmlu_virology.yaml | 8 + .../en_us/openai_mmlu_world_religions.yaml | 8 + .../openai_mmlu/es_la/_openai_mmlu_es_la.yaml | 9 + .../es_la/openai_mmlu_abstract_algebra.yaml | 8 + .../es_la/openai_mmlu_anatomy.yaml | 8 + .../es_la/openai_mmlu_astronomy.yaml | 8 + .../es_la/openai_mmlu_business_ethics.yaml | 8 + .../es_la/openai_mmlu_clinical_knowledge.yaml | 8 + .../es_la/openai_mmlu_college_biology.yaml | 8 + .../es_la/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../es_la/openai_mmlu_college_medicine.yaml | 8 + .../es_la/openai_mmlu_college_physics.yaml | 8 + .../es_la/openai_mmlu_computer_security.yaml | 8 + .../es_la/openai_mmlu_conceptual_physics.yaml | 8 + .../es_la/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../es_la/openai_mmlu_formal_logic.yaml | 8 + .../es_la/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../es_la/openai_mmlu_human_aging.yaml | 8 + .../es_la/openai_mmlu_human_sexuality.yaml | 8 + .../es_la/openai_mmlu_international_law.yaml | 8 + .../es_la/openai_mmlu_jurisprudence.yaml | 8 + .../es_la/openai_mmlu_logical_fallacies.yaml | 8 + .../es_la/openai_mmlu_machine_learning.yaml | 8 + .../es_la/openai_mmlu_management.yaml | 8 + .../es_la/openai_mmlu_marketing.yaml | 8 + .../es_la/openai_mmlu_medical_genetics.yaml | 8 + .../es_la/openai_mmlu_miscellaneous.yaml | 8 + .../es_la/openai_mmlu_moral_disputes.yaml | 8 + .../es_la/openai_mmlu_moral_scenarios.yaml | 8 + .../es_la/openai_mmlu_nutrition.yaml | 8 + .../es_la/openai_mmlu_philosophy.yaml | 8 + .../es_la/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../es_la/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../es_la/openai_mmlu_public_relations.yaml | 8 + .../es_la/openai_mmlu_security_studies.yaml | 8 + .../es_la/openai_mmlu_sociology.yaml | 8 + .../es_la/openai_mmlu_us_foreign_policy.yaml | 8 + .../es_la/openai_mmlu_virology.yaml | 8 + .../es_la/openai_mmlu_world_religions.yaml | 8 + .../openai_mmlu/fr_fr/_openai_mmlu_fr_fr.yaml | 9 + .../fr_fr/openai_mmlu_abstract_algebra.yaml | 8 + .../fr_fr/openai_mmlu_anatomy.yaml | 8 + .../fr_fr/openai_mmlu_astronomy.yaml | 8 + .../fr_fr/openai_mmlu_business_ethics.yaml | 8 + .../fr_fr/openai_mmlu_clinical_knowledge.yaml | 8 + .../fr_fr/openai_mmlu_college_biology.yaml | 8 + .../fr_fr/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../fr_fr/openai_mmlu_college_medicine.yaml | 8 + .../fr_fr/openai_mmlu_college_physics.yaml | 8 + .../fr_fr/openai_mmlu_computer_security.yaml | 8 + .../fr_fr/openai_mmlu_conceptual_physics.yaml | 8 + .../fr_fr/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../fr_fr/openai_mmlu_formal_logic.yaml | 8 + .../fr_fr/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../fr_fr/openai_mmlu_human_aging.yaml | 8 + .../fr_fr/openai_mmlu_human_sexuality.yaml | 8 + .../fr_fr/openai_mmlu_international_law.yaml | 8 + .../fr_fr/openai_mmlu_jurisprudence.yaml | 8 + .../fr_fr/openai_mmlu_logical_fallacies.yaml | 8 + .../fr_fr/openai_mmlu_machine_learning.yaml | 8 + .../fr_fr/openai_mmlu_management.yaml | 8 + .../fr_fr/openai_mmlu_marketing.yaml | 8 + .../fr_fr/openai_mmlu_medical_genetics.yaml | 8 + .../fr_fr/openai_mmlu_miscellaneous.yaml | 8 + .../fr_fr/openai_mmlu_moral_disputes.yaml | 8 + .../fr_fr/openai_mmlu_moral_scenarios.yaml | 8 + .../fr_fr/openai_mmlu_nutrition.yaml | 8 + .../fr_fr/openai_mmlu_philosophy.yaml | 8 + .../fr_fr/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../fr_fr/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../fr_fr/openai_mmlu_public_relations.yaml | 8 + .../fr_fr/openai_mmlu_security_studies.yaml | 8 + .../fr_fr/openai_mmlu_sociology.yaml | 8 + .../fr_fr/openai_mmlu_us_foreign_policy.yaml | 8 + .../fr_fr/openai_mmlu_virology.yaml | 8 + .../fr_fr/openai_mmlu_world_religions.yaml | 8 + .../openai_mmlu/hi_in/_openai_mmlu_hi_in.yaml | 9 + .../hi_in/openai_mmlu_abstract_algebra.yaml | 8 + .../hi_in/openai_mmlu_anatomy.yaml | 8 + .../hi_in/openai_mmlu_astronomy.yaml | 8 + .../hi_in/openai_mmlu_business_ethics.yaml | 8 + .../hi_in/openai_mmlu_clinical_knowledge.yaml | 8 + .../hi_in/openai_mmlu_college_biology.yaml | 8 + .../hi_in/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../hi_in/openai_mmlu_college_medicine.yaml | 8 + .../hi_in/openai_mmlu_college_physics.yaml | 8 + .../hi_in/openai_mmlu_computer_security.yaml | 8 + .../hi_in/openai_mmlu_conceptual_physics.yaml | 8 + .../hi_in/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../hi_in/openai_mmlu_formal_logic.yaml | 8 + .../hi_in/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../hi_in/openai_mmlu_human_aging.yaml | 8 + .../hi_in/openai_mmlu_human_sexuality.yaml | 8 + .../hi_in/openai_mmlu_international_law.yaml | 8 + .../hi_in/openai_mmlu_jurisprudence.yaml | 8 + .../hi_in/openai_mmlu_logical_fallacies.yaml | 8 + .../hi_in/openai_mmlu_machine_learning.yaml | 8 + .../hi_in/openai_mmlu_management.yaml | 8 + .../hi_in/openai_mmlu_marketing.yaml | 8 + .../hi_in/openai_mmlu_medical_genetics.yaml | 8 + .../hi_in/openai_mmlu_miscellaneous.yaml | 8 + .../hi_in/openai_mmlu_moral_disputes.yaml | 8 + .../hi_in/openai_mmlu_moral_scenarios.yaml | 8 + .../hi_in/openai_mmlu_nutrition.yaml | 8 + .../hi_in/openai_mmlu_philosophy.yaml | 8 + .../hi_in/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../hi_in/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../hi_in/openai_mmlu_public_relations.yaml | 8 + .../hi_in/openai_mmlu_security_studies.yaml | 8 + .../hi_in/openai_mmlu_sociology.yaml | 8 + .../hi_in/openai_mmlu_us_foreign_policy.yaml | 8 + .../hi_in/openai_mmlu_virology.yaml | 8 + .../hi_in/openai_mmlu_world_religions.yaml | 8 + .../openai_mmlu/id_id/_openai_mmlu_id_id.yaml | 9 + .../id_id/openai_mmlu_abstract_algebra.yaml | 8 + .../id_id/openai_mmlu_anatomy.yaml | 8 + .../id_id/openai_mmlu_astronomy.yaml | 8 + .../id_id/openai_mmlu_business_ethics.yaml | 8 + .../id_id/openai_mmlu_clinical_knowledge.yaml | 8 + .../id_id/openai_mmlu_college_biology.yaml | 8 + .../id_id/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../id_id/openai_mmlu_college_medicine.yaml | 8 + .../id_id/openai_mmlu_college_physics.yaml | 8 + .../id_id/openai_mmlu_computer_security.yaml | 8 + .../id_id/openai_mmlu_conceptual_physics.yaml | 8 + .../id_id/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../id_id/openai_mmlu_formal_logic.yaml | 8 + .../id_id/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../id_id/openai_mmlu_human_aging.yaml | 8 + .../id_id/openai_mmlu_human_sexuality.yaml | 8 + .../id_id/openai_mmlu_international_law.yaml | 8 + .../id_id/openai_mmlu_jurisprudence.yaml | 8 + .../id_id/openai_mmlu_logical_fallacies.yaml | 8 + .../id_id/openai_mmlu_machine_learning.yaml | 8 + .../id_id/openai_mmlu_management.yaml | 8 + .../id_id/openai_mmlu_marketing.yaml | 8 + .../id_id/openai_mmlu_medical_genetics.yaml | 8 + .../id_id/openai_mmlu_miscellaneous.yaml | 8 + .../id_id/openai_mmlu_moral_disputes.yaml | 8 + .../id_id/openai_mmlu_moral_scenarios.yaml | 8 + .../id_id/openai_mmlu_nutrition.yaml | 8 + .../id_id/openai_mmlu_philosophy.yaml | 8 + .../id_id/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../id_id/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../id_id/openai_mmlu_public_relations.yaml | 8 + .../id_id/openai_mmlu_security_studies.yaml | 8 + .../id_id/openai_mmlu_sociology.yaml | 8 + .../id_id/openai_mmlu_us_foreign_policy.yaml | 8 + .../id_id/openai_mmlu_virology.yaml | 8 + .../id_id/openai_mmlu_world_religions.yaml | 8 + .../openai_mmlu/it_it/_openai_mmlu_it_it.yaml | 9 + .../it_it/openai_mmlu_abstract_algebra.yaml | 8 + .../it_it/openai_mmlu_anatomy.yaml | 8 + .../it_it/openai_mmlu_astronomy.yaml | 8 + .../it_it/openai_mmlu_business_ethics.yaml | 8 + .../it_it/openai_mmlu_clinical_knowledge.yaml | 8 + .../it_it/openai_mmlu_college_biology.yaml | 8 + .../it_it/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../it_it/openai_mmlu_college_medicine.yaml | 8 + .../it_it/openai_mmlu_college_physics.yaml | 8 + .../it_it/openai_mmlu_computer_security.yaml | 8 + .../it_it/openai_mmlu_conceptual_physics.yaml | 8 + .../it_it/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../it_it/openai_mmlu_formal_logic.yaml | 8 + .../it_it/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../it_it/openai_mmlu_human_aging.yaml | 8 + .../it_it/openai_mmlu_human_sexuality.yaml | 8 + .../it_it/openai_mmlu_international_law.yaml | 8 + .../it_it/openai_mmlu_jurisprudence.yaml | 8 + .../it_it/openai_mmlu_logical_fallacies.yaml | 8 + .../it_it/openai_mmlu_machine_learning.yaml | 8 + .../it_it/openai_mmlu_management.yaml | 8 + .../it_it/openai_mmlu_marketing.yaml | 8 + .../it_it/openai_mmlu_medical_genetics.yaml | 8 + .../it_it/openai_mmlu_miscellaneous.yaml | 8 + .../it_it/openai_mmlu_moral_disputes.yaml | 8 + .../it_it/openai_mmlu_moral_scenarios.yaml | 8 + .../it_it/openai_mmlu_nutrition.yaml | 8 + .../it_it/openai_mmlu_philosophy.yaml | 8 + .../it_it/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../it_it/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../it_it/openai_mmlu_public_relations.yaml | 8 + .../it_it/openai_mmlu_security_studies.yaml | 8 + .../it_it/openai_mmlu_sociology.yaml | 8 + .../it_it/openai_mmlu_us_foreign_policy.yaml | 8 + .../it_it/openai_mmlu_virology.yaml | 8 + .../it_it/openai_mmlu_world_religions.yaml | 8 + .../openai_mmlu/ja_jp/_openai_mmlu_ja_jp.yaml | 9 + .../ja_jp/openai_mmlu_abstract_algebra.yaml | 8 + .../ja_jp/openai_mmlu_anatomy.yaml | 8 + .../ja_jp/openai_mmlu_astronomy.yaml | 8 + .../ja_jp/openai_mmlu_business_ethics.yaml | 8 + .../ja_jp/openai_mmlu_clinical_knowledge.yaml | 8 + .../ja_jp/openai_mmlu_college_biology.yaml | 8 + .../ja_jp/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../ja_jp/openai_mmlu_college_medicine.yaml | 8 + .../ja_jp/openai_mmlu_college_physics.yaml | 8 + .../ja_jp/openai_mmlu_computer_security.yaml | 8 + .../ja_jp/openai_mmlu_conceptual_physics.yaml | 8 + .../ja_jp/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../ja_jp/openai_mmlu_formal_logic.yaml | 8 + .../ja_jp/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../ja_jp/openai_mmlu_human_aging.yaml | 8 + .../ja_jp/openai_mmlu_human_sexuality.yaml | 8 + .../ja_jp/openai_mmlu_international_law.yaml | 8 + .../ja_jp/openai_mmlu_jurisprudence.yaml | 8 + .../ja_jp/openai_mmlu_logical_fallacies.yaml | 8 + .../ja_jp/openai_mmlu_machine_learning.yaml | 8 + .../ja_jp/openai_mmlu_management.yaml | 8 + .../ja_jp/openai_mmlu_marketing.yaml | 8 + .../ja_jp/openai_mmlu_medical_genetics.yaml | 8 + .../ja_jp/openai_mmlu_miscellaneous.yaml | 8 + .../ja_jp/openai_mmlu_moral_disputes.yaml | 8 + .../ja_jp/openai_mmlu_moral_scenarios.yaml | 8 + .../ja_jp/openai_mmlu_nutrition.yaml | 8 + .../ja_jp/openai_mmlu_philosophy.yaml | 8 + .../ja_jp/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../ja_jp/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../ja_jp/openai_mmlu_public_relations.yaml | 8 + .../ja_jp/openai_mmlu_security_studies.yaml | 8 + .../ja_jp/openai_mmlu_sociology.yaml | 8 + .../ja_jp/openai_mmlu_us_foreign_policy.yaml | 8 + .../ja_jp/openai_mmlu_virology.yaml | 8 + .../ja_jp/openai_mmlu_world_religions.yaml | 8 + .../openai_mmlu/ko_kr/_openai_mmlu_ko_kr.yaml | 9 + .../ko_kr/openai_mmlu_abstract_algebra.yaml | 8 + .../ko_kr/openai_mmlu_anatomy.yaml | 8 + .../ko_kr/openai_mmlu_astronomy.yaml | 8 + .../ko_kr/openai_mmlu_business_ethics.yaml | 8 + .../ko_kr/openai_mmlu_clinical_knowledge.yaml | 8 + .../ko_kr/openai_mmlu_college_biology.yaml | 8 + .../ko_kr/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../ko_kr/openai_mmlu_college_medicine.yaml | 8 + .../ko_kr/openai_mmlu_college_physics.yaml | 8 + .../ko_kr/openai_mmlu_computer_security.yaml | 8 + .../ko_kr/openai_mmlu_conceptual_physics.yaml | 8 + .../ko_kr/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../ko_kr/openai_mmlu_formal_logic.yaml | 8 + .../ko_kr/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../ko_kr/openai_mmlu_human_aging.yaml | 8 + .../ko_kr/openai_mmlu_human_sexuality.yaml | 8 + .../ko_kr/openai_mmlu_international_law.yaml | 8 + .../ko_kr/openai_mmlu_jurisprudence.yaml | 8 + .../ko_kr/openai_mmlu_logical_fallacies.yaml | 8 + .../ko_kr/openai_mmlu_machine_learning.yaml | 8 + .../ko_kr/openai_mmlu_management.yaml | 8 + .../ko_kr/openai_mmlu_marketing.yaml | 8 + .../ko_kr/openai_mmlu_medical_genetics.yaml | 8 + .../ko_kr/openai_mmlu_miscellaneous.yaml | 8 + .../ko_kr/openai_mmlu_moral_disputes.yaml | 8 + .../ko_kr/openai_mmlu_moral_scenarios.yaml | 8 + .../ko_kr/openai_mmlu_nutrition.yaml | 8 + .../ko_kr/openai_mmlu_philosophy.yaml | 8 + .../ko_kr/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../ko_kr/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../ko_kr/openai_mmlu_public_relations.yaml | 8 + .../ko_kr/openai_mmlu_security_studies.yaml | 8 + .../ko_kr/openai_mmlu_sociology.yaml | 8 + .../ko_kr/openai_mmlu_us_foreign_policy.yaml | 8 + .../ko_kr/openai_mmlu_virology.yaml | 8 + .../ko_kr/openai_mmlu_world_religions.yaml | 8 + .../openai_mmlu/pt_br/_openai_mmlu_pt_br.yaml | 9 + .../pt_br/openai_mmlu_abstract_algebra.yaml | 8 + .../pt_br/openai_mmlu_anatomy.yaml | 8 + .../pt_br/openai_mmlu_astronomy.yaml | 8 + .../pt_br/openai_mmlu_business_ethics.yaml | 8 + .../pt_br/openai_mmlu_clinical_knowledge.yaml | 8 + .../pt_br/openai_mmlu_college_biology.yaml | 8 + .../pt_br/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../pt_br/openai_mmlu_college_medicine.yaml | 8 + .../pt_br/openai_mmlu_college_physics.yaml | 8 + .../pt_br/openai_mmlu_computer_security.yaml | 8 + .../pt_br/openai_mmlu_conceptual_physics.yaml | 8 + .../pt_br/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../pt_br/openai_mmlu_formal_logic.yaml | 8 + .../pt_br/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../pt_br/openai_mmlu_human_aging.yaml | 8 + .../pt_br/openai_mmlu_human_sexuality.yaml | 8 + .../pt_br/openai_mmlu_international_law.yaml | 8 + .../pt_br/openai_mmlu_jurisprudence.yaml | 8 + .../pt_br/openai_mmlu_logical_fallacies.yaml | 8 + .../pt_br/openai_mmlu_machine_learning.yaml | 8 + .../pt_br/openai_mmlu_management.yaml | 8 + .../pt_br/openai_mmlu_marketing.yaml | 8 + .../pt_br/openai_mmlu_medical_genetics.yaml | 8 + .../pt_br/openai_mmlu_miscellaneous.yaml | 8 + .../pt_br/openai_mmlu_moral_disputes.yaml | 8 + .../pt_br/openai_mmlu_moral_scenarios.yaml | 8 + .../pt_br/openai_mmlu_nutrition.yaml | 8 + .../pt_br/openai_mmlu_philosophy.yaml | 8 + .../pt_br/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../pt_br/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../pt_br/openai_mmlu_public_relations.yaml | 8 + .../pt_br/openai_mmlu_security_studies.yaml | 8 + .../pt_br/openai_mmlu_sociology.yaml | 8 + .../pt_br/openai_mmlu_us_foreign_policy.yaml | 8 + .../pt_br/openai_mmlu_virology.yaml | 8 + .../pt_br/openai_mmlu_world_religions.yaml | 8 + .../openai_mmlu/sw_ke/_openai_mmlu_sw_ke.yaml | 9 + .../sw_ke/openai_mmlu_abstract_algebra.yaml | 8 + .../sw_ke/openai_mmlu_anatomy.yaml | 8 + .../sw_ke/openai_mmlu_astronomy.yaml | 8 + .../sw_ke/openai_mmlu_business_ethics.yaml | 8 + .../sw_ke/openai_mmlu_clinical_knowledge.yaml | 8 + .../sw_ke/openai_mmlu_college_biology.yaml | 8 + .../sw_ke/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../sw_ke/openai_mmlu_college_medicine.yaml | 8 + .../sw_ke/openai_mmlu_college_physics.yaml | 8 + .../sw_ke/openai_mmlu_computer_security.yaml | 8 + .../sw_ke/openai_mmlu_conceptual_physics.yaml | 8 + .../sw_ke/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../sw_ke/openai_mmlu_formal_logic.yaml | 8 + .../sw_ke/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../sw_ke/openai_mmlu_human_aging.yaml | 8 + .../sw_ke/openai_mmlu_human_sexuality.yaml | 8 + .../sw_ke/openai_mmlu_international_law.yaml | 8 + .../sw_ke/openai_mmlu_jurisprudence.yaml | 8 + .../sw_ke/openai_mmlu_logical_fallacies.yaml | 8 + .../sw_ke/openai_mmlu_machine_learning.yaml | 8 + .../sw_ke/openai_mmlu_management.yaml | 8 + .../sw_ke/openai_mmlu_marketing.yaml | 8 + .../sw_ke/openai_mmlu_medical_genetics.yaml | 8 + .../sw_ke/openai_mmlu_miscellaneous.yaml | 8 + .../sw_ke/openai_mmlu_moral_disputes.yaml | 8 + .../sw_ke/openai_mmlu_moral_scenarios.yaml | 8 + .../sw_ke/openai_mmlu_nutrition.yaml | 8 + .../sw_ke/openai_mmlu_philosophy.yaml | 8 + .../sw_ke/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../sw_ke/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../sw_ke/openai_mmlu_public_relations.yaml | 8 + .../sw_ke/openai_mmlu_security_studies.yaml | 8 + .../sw_ke/openai_mmlu_sociology.yaml | 8 + .../sw_ke/openai_mmlu_us_foreign_policy.yaml | 8 + .../sw_ke/openai_mmlu_virology.yaml | 8 + .../sw_ke/openai_mmlu_world_religions.yaml | 8 + .../openai_mmlu/yo_ng/_openai_mmlu_yo_ng.yaml | 9 + .../yo_ng/openai_mmlu_abstract_algebra.yaml | 8 + .../yo_ng/openai_mmlu_anatomy.yaml | 8 + .../yo_ng/openai_mmlu_astronomy.yaml | 8 + .../yo_ng/openai_mmlu_business_ethics.yaml | 8 + .../yo_ng/openai_mmlu_clinical_knowledge.yaml | 8 + .../yo_ng/openai_mmlu_college_biology.yaml | 8 + .../yo_ng/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../yo_ng/openai_mmlu_college_medicine.yaml | 8 + .../yo_ng/openai_mmlu_college_physics.yaml | 8 + .../yo_ng/openai_mmlu_computer_security.yaml | 8 + .../yo_ng/openai_mmlu_conceptual_physics.yaml | 8 + .../yo_ng/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../yo_ng/openai_mmlu_formal_logic.yaml | 8 + .../yo_ng/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../yo_ng/openai_mmlu_human_aging.yaml | 8 + .../yo_ng/openai_mmlu_human_sexuality.yaml | 8 + .../yo_ng/openai_mmlu_international_law.yaml | 8 + .../yo_ng/openai_mmlu_jurisprudence.yaml | 8 + .../yo_ng/openai_mmlu_logical_fallacies.yaml | 8 + .../yo_ng/openai_mmlu_machine_learning.yaml | 8 + .../yo_ng/openai_mmlu_management.yaml | 8 + .../yo_ng/openai_mmlu_marketing.yaml | 8 + .../yo_ng/openai_mmlu_medical_genetics.yaml | 8 + .../yo_ng/openai_mmlu_miscellaneous.yaml | 8 + .../yo_ng/openai_mmlu_moral_disputes.yaml | 8 + .../yo_ng/openai_mmlu_moral_scenarios.yaml | 8 + .../yo_ng/openai_mmlu_nutrition.yaml | 8 + .../yo_ng/openai_mmlu_philosophy.yaml | 8 + .../yo_ng/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../yo_ng/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../yo_ng/openai_mmlu_public_relations.yaml | 8 + .../yo_ng/openai_mmlu_security_studies.yaml | 8 + .../yo_ng/openai_mmlu_sociology.yaml | 8 + .../yo_ng/openai_mmlu_us_foreign_policy.yaml | 8 + .../yo_ng/openai_mmlu_virology.yaml | 8 + .../yo_ng/openai_mmlu_world_religions.yaml | 8 + .../openai_mmlu/zh_cn/_openai_mmlu_zh_cn.yaml | 9 + .../zh_cn/openai_mmlu_abstract_algebra.yaml | 8 + .../zh_cn/openai_mmlu_anatomy.yaml | 8 + .../zh_cn/openai_mmlu_astronomy.yaml | 8 + .../zh_cn/openai_mmlu_business_ethics.yaml | 8 + .../zh_cn/openai_mmlu_clinical_knowledge.yaml | 8 + .../zh_cn/openai_mmlu_college_biology.yaml | 8 + .../zh_cn/openai_mmlu_college_chemistry.yaml | 8 + .../openai_mmlu_college_computer_science.yaml | 8 + .../openai_mmlu_college_mathematics.yaml | 8 + .../zh_cn/openai_mmlu_college_medicine.yaml | 8 + .../zh_cn/openai_mmlu_college_physics.yaml | 8 + .../zh_cn/openai_mmlu_computer_security.yaml | 8 + .../zh_cn/openai_mmlu_conceptual_physics.yaml | 8 + .../zh_cn/openai_mmlu_econometrics.yaml | 8 + .../openai_mmlu_electrical_engineering.yaml | 8 + .../openai_mmlu_elementary_mathematics.yaml | 8 + .../zh_cn/openai_mmlu_formal_logic.yaml | 8 + .../zh_cn/openai_mmlu_global_facts.yaml | 8 + .../openai_mmlu_high_school_biology.yaml | 8 + .../openai_mmlu_high_school_chemistry.yaml | 8 + ...nai_mmlu_high_school_computer_science.yaml | 8 + ...nai_mmlu_high_school_european_history.yaml | 8 + .../openai_mmlu_high_school_geography.yaml | 8 + ...u_high_school_government_and_politics.yaml | 8 + ...penai_mmlu_high_school_macroeconomics.yaml | 8 + .../openai_mmlu_high_school_mathematics.yaml | 8 + ...penai_mmlu_high_school_microeconomics.yaml | 8 + .../openai_mmlu_high_school_physics.yaml | 8 + .../openai_mmlu_high_school_psychology.yaml | 8 + .../openai_mmlu_high_school_statistics.yaml | 8 + .../openai_mmlu_high_school_us_history.yaml | 8 + ...openai_mmlu_high_school_world_history.yaml | 8 + .../zh_cn/openai_mmlu_human_aging.yaml | 8 + .../zh_cn/openai_mmlu_human_sexuality.yaml | 8 + .../zh_cn/openai_mmlu_international_law.yaml | 8 + .../zh_cn/openai_mmlu_jurisprudence.yaml | 8 + .../zh_cn/openai_mmlu_logical_fallacies.yaml | 8 + .../zh_cn/openai_mmlu_machine_learning.yaml | 8 + .../zh_cn/openai_mmlu_management.yaml | 8 + .../zh_cn/openai_mmlu_marketing.yaml | 8 + .../zh_cn/openai_mmlu_medical_genetics.yaml | 8 + .../zh_cn/openai_mmlu_miscellaneous.yaml | 8 + .../zh_cn/openai_mmlu_moral_disputes.yaml | 8 + .../zh_cn/openai_mmlu_moral_scenarios.yaml | 8 + .../zh_cn/openai_mmlu_nutrition.yaml | 8 + .../zh_cn/openai_mmlu_philosophy.yaml | 8 + .../zh_cn/openai_mmlu_prehistory.yaml | 8 + .../openai_mmlu_professional_accounting.yaml | 8 + .../zh_cn/openai_mmlu_professional_law.yaml | 8 + .../openai_mmlu_professional_medicine.yaml | 8 + .../openai_mmlu_professional_psychology.yaml | 8 + .../zh_cn/openai_mmlu_public_relations.yaml | 8 + .../zh_cn/openai_mmlu_security_studies.yaml | 8 + .../zh_cn/openai_mmlu_sociology.yaml | 8 + .../zh_cn/openai_mmlu_us_foreign_policy.yaml | 8 + .../zh_cn/openai_mmlu_virology.yaml | 8 + .../zh_cn/openai_mmlu_world_religions.yaml | 8 + 872 files changed, 7154 insertions(+) create mode 100644 lm_eval/tasks/openai_mmlu/_default_template.yaml create mode 100644 lm_eval/tasks/openai_mmlu/_generate_configs.py create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/_openai_mmlu_ar_xy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/_openai_mmlu_bn_bd.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/_openai_mmlu_de_de.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/_openai_mmlu_en_us.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/_openai_mmlu_es_la.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/_openai_mmlu_fr_fr.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/_openai_mmlu_hi_in.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/_openai_mmlu_id_id.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/_openai_mmlu_it_it.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/_openai_mmlu_ja_jp.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/_openai_mmlu_ko_kr.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/_openai_mmlu_pt_br.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/_openai_mmlu_sw_ke.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/_openai_mmlu_yo_ng.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/_openai_mmlu_zh_cn.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_management.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_virology.yaml create mode 100644 lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_world_religions.yaml diff --git a/lm_eval/tasks/openai_mmlu/_default_template.yaml b/lm_eval/tasks/openai_mmlu/_default_template.yaml new file mode 100644 index 0000000000..2cd29a52e4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/_default_template.yaml @@ -0,0 +1,15 @@ +dataset_path: baber/multilingual_mmlu # a copy of `cais/mmlu` with no auxiliary_train split +fewshot_split: null +fewshot_config: null +output_type: multiple_choice +doc_to_text: "{{Question.strip()}}\nA. {{A.strip()}}\nB. {{B.strip()}}\nC. {{C.strip()}}\nD. {{D.strip()}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: "{{Answer.strip()}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm_eval/tasks/openai_mmlu/_generate_configs.py b/lm_eval/tasks/openai_mmlu/_generate_configs.py new file mode 100644 index 0000000000..e72b3322ef --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/_generate_configs.py @@ -0,0 +1,164 @@ +""" +Take in a YAML, and output all "other" splits with this YAML +""" + +import argparse +import logging +import os +from pathlib import Path + +import yaml +from tqdm import tqdm + + +eval_logger = logging.getLogger("lm-eval") + + +SUBJECTS = { + "abstract_algebra": "stem", + "anatomy": "stem", + "astronomy": "stem", + "business_ethics": "other", + "clinical_knowledge": "other", + "college_biology": "stem", + "college_chemistry": "stem", + "college_computer_science": "stem", + "college_mathematics": "stem", + "college_medicine": "other", + "college_physics": "stem", + "computer_security": "stem", + "conceptual_physics": "stem", + "econometrics": "social_sciences", + "electrical_engineering": "stem", + "elementary_mathematics": "stem", + "formal_logic": "humanities", + "global_facts": "other", + "high_school_biology": "stem", + "high_school_chemistry": "stem", + "high_school_computer_science": "stem", + "high_school_european_history": "humanities", + "high_school_geography": "social_sciences", + "high_school_government_and_politics": "social_sciences", + "high_school_macroeconomics": "social_sciences", + "high_school_mathematics": "stem", + "high_school_microeconomics": "social_sciences", + "high_school_physics": "stem", + "high_school_psychology": "social_sciences", + "high_school_statistics": "stem", + "high_school_us_history": "humanities", + "high_school_world_history": "humanities", + "human_aging": "other", + "human_sexuality": "social_sciences", + "international_law": "humanities", + "jurisprudence": "humanities", + "logical_fallacies": "humanities", + "machine_learning": "stem", + "management": "other", + "marketing": "other", + "medical_genetics": "other", + "miscellaneous": "other", + "moral_disputes": "humanities", + "moral_scenarios": "humanities", + "nutrition": "other", + "philosophy": "humanities", + "prehistory": "humanities", + "professional_accounting": "other", + "professional_law": "humanities", + "professional_medicine": "other", + "professional_psychology": "social_sciences", + "public_relations": "social_sciences", + "security_studies": "social_sciences", + "sociology": "social_sciences", + "us_foreign_policy": "social_sciences", + "virology": "other", + "world_religions": "humanities", +} + +LANGUAGES = { + "AR_XY": "Arabic (Generic)", + "BN_BD": "Bengali (Bangladesh)", + "DE_DE": "German (Germany)", + "ES_LA": "Spanish (Latin America)", + "FR_FR": "French (France)", + "HI_IN": "Hindi (India)", + "ID_ID": "Indonesian (Indonesia)", + "IT_IT": "Italian (Italy)", + "JA_JP": "Japanese (Japan)", + "KO_KR": "Korean (South Korea)", + "PT_BR": "Portuguese (Brazil)", + "ZH_CN": "Chinese (China)", + "SW_KE": "Swahili (Kenya)", + "YO_NG": "Yoruba (Nigeria)", + "EN_US": "English (United States)", +} + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--base_yaml_path", required=True) + parser.add_argument("--save_prefix_path", default="openai_mmlu") + parser.add_argument("--group_prefix", default="") + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + + # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs. + base_yaml_name = os.path.split(args.base_yaml_path)[-1] + with open(args.base_yaml_path, encoding="utf-8") as f: + base_yaml = yaml.full_load(f) + + ALL_CATEGORIES = [] + for langgode, language_full_name in tqdm(LANGUAGES.items()): + _langgode = langgode.lower() + out_folder = Path(_langgode) + out_folder.mkdir(exist_ok=True) + for subject, category in SUBJECTS.items(): + if category not in ALL_CATEGORIES: + ALL_CATEGORIES.append(category) + + description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))} in the {language_full_name} language.\n\n" + + yaml_dict = { + "include": f"../{base_yaml_name}", + "tag": f"mmlu_{_langgode}_{category}", + "task": f"mmlu_{_langgode}_{subject}", + "task_alias": f'{_langgode} {subject.replace("_", " ")}', + "dataset_name": subject, + "test_split": langgode, + "description": description, + } + + file_save_path = out_folder / (args.save_prefix_path + f"_{subject}.yaml") + eval_logger.info( + f"Saving yaml for subset {_langgode},{subject} to {file_save_path}" + ) + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump( + yaml_dict, + yaml_file, + allow_unicode=True, + default_style='"', + ) + + # shutil.copy("_default_template.yaml", out_folder/"_default_template.yaml") + + file_save_path = out_folder / ( + "_" + args.save_prefix_path + f"_{_langgode}.yaml" + ) + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + dct = { + "group": f"openai_mmlu_{_langgode}", + "group_alias": _langgode, + "task": [f"mmlu_{_langgode}_tasks"], + "aggregate_metric_list": [{"metric": "acc", "weight_by_size": True}], + "metadata": {"version": "1.0.0"}, + } + + yaml.dump( + dct, + yaml_file, + indent=4, + default_flow_style=False, + ) diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/_openai_mmlu_ar_xy.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/_openai_mmlu_ar_xy.yaml new file mode 100644 index 0000000000..732281f7ee --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/_openai_mmlu_ar_xy.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_ar_xy +group_alias: ar_xy +metadata: + version: 1.0.0 +task: +- mmlu_ar_xy_tasks diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..0056be8476 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_abstract_algebra" +"task_alias": "ar_xy abstract algebra" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..bf6eab4928 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_anatomy" +"task_alias": "ar_xy anatomy" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..613a81fae9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_astronomy" +"task_alias": "ar_xy astronomy" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..1059f21f38 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_other" +"task": "mmlu_ar_xy_business_ethics" +"task_alias": "ar_xy business ethics" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..9d0c80059c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_other" +"task": "mmlu_ar_xy_clinical_knowledge" +"task_alias": "ar_xy clinical knowledge" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..d577a073af --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_college_biology" +"task_alias": "ar_xy college biology" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..d1281588da --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_college_chemistry" +"task_alias": "ar_xy college chemistry" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..e0b4d12933 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_college_computer_science" +"task_alias": "ar_xy college computer science" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..b799c8285d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_college_mathematics" +"task_alias": "ar_xy college mathematics" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..bb699b4cbd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_other" +"task": "mmlu_ar_xy_college_medicine" +"task_alias": "ar_xy college medicine" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..17820af82b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_college_physics" +"task_alias": "ar_xy college physics" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..69678dbdfe --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_computer_security" +"task_alias": "ar_xy computer security" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..c62e2aea39 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_conceptual_physics" +"task_alias": "ar_xy conceptual physics" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..d93e75cf03 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_social_sciences" +"task": "mmlu_ar_xy_econometrics" +"task_alias": "ar_xy econometrics" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..5dc39f17fa --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_electrical_engineering" +"task_alias": "ar_xy electrical engineering" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..f55b89d88d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_elementary_mathematics" +"task_alias": "ar_xy elementary mathematics" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..d5f8cd4bda --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_humanities" +"task": "mmlu_ar_xy_formal_logic" +"task_alias": "ar_xy formal logic" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..a41fd6e1d2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_other" +"task": "mmlu_ar_xy_global_facts" +"task_alias": "ar_xy global facts" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..737b90e0fd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_high_school_biology" +"task_alias": "ar_xy high school biology" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..56c375e266 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_high_school_chemistry" +"task_alias": "ar_xy high school chemistry" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..8f46e2cd47 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_high_school_computer_science" +"task_alias": "ar_xy high school computer science" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..f80a651f06 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_humanities" +"task": "mmlu_ar_xy_high_school_european_history" +"task_alias": "ar_xy high school european history" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..edca3ef22f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_social_sciences" +"task": "mmlu_ar_xy_high_school_geography" +"task_alias": "ar_xy high school geography" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..c42737654b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_social_sciences" +"task": "mmlu_ar_xy_high_school_government_and_politics" +"task_alias": "ar_xy high school government and politics" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..a29031283d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_social_sciences" +"task": "mmlu_ar_xy_high_school_macroeconomics" +"task_alias": "ar_xy high school macroeconomics" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..5b0ba0fd0d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_high_school_mathematics" +"task_alias": "ar_xy high school mathematics" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..5560ae79d6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_social_sciences" +"task": "mmlu_ar_xy_high_school_microeconomics" +"task_alias": "ar_xy high school microeconomics" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..f187ea6b57 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_high_school_physics" +"task_alias": "ar_xy high school physics" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..d1ae7c67bb --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_social_sciences" +"task": "mmlu_ar_xy_high_school_psychology" +"task_alias": "ar_xy high school psychology" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..505eea00a0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_high_school_statistics" +"task_alias": "ar_xy high school statistics" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..cf960e4a4e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_humanities" +"task": "mmlu_ar_xy_high_school_us_history" +"task_alias": "ar_xy high school us history" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..3b9b6c259f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_humanities" +"task": "mmlu_ar_xy_high_school_world_history" +"task_alias": "ar_xy high school world history" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..43180e6081 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_other" +"task": "mmlu_ar_xy_human_aging" +"task_alias": "ar_xy human aging" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..55f57942d4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_social_sciences" +"task": "mmlu_ar_xy_human_sexuality" +"task_alias": "ar_xy human sexuality" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..eff83768dc --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_humanities" +"task": "mmlu_ar_xy_international_law" +"task_alias": "ar_xy international law" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..f304e764b3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_humanities" +"task": "mmlu_ar_xy_jurisprudence" +"task_alias": "ar_xy jurisprudence" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..67a5a6b67f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_humanities" +"task": "mmlu_ar_xy_logical_fallacies" +"task_alias": "ar_xy logical fallacies" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..69db7be85e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_stem" +"task": "mmlu_ar_xy_machine_learning" +"task_alias": "ar_xy machine learning" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_management.yaml new file mode 100644 index 0000000000..7e66d2049f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_other" +"task": "mmlu_ar_xy_management" +"task_alias": "ar_xy management" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..4d51495062 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_other" +"task": "mmlu_ar_xy_marketing" +"task_alias": "ar_xy marketing" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..9ce5d2783d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_other" +"task": "mmlu_ar_xy_medical_genetics" +"task_alias": "ar_xy medical genetics" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..49e996faf5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_other" +"task": "mmlu_ar_xy_miscellaneous" +"task_alias": "ar_xy miscellaneous" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..4cf14eb2a3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_humanities" +"task": "mmlu_ar_xy_moral_disputes" +"task_alias": "ar_xy moral disputes" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..5c64a50f6a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_humanities" +"task": "mmlu_ar_xy_moral_scenarios" +"task_alias": "ar_xy moral scenarios" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..ecba03ebad --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_other" +"task": "mmlu_ar_xy_nutrition" +"task_alias": "ar_xy nutrition" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..783b7675ce --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_humanities" +"task": "mmlu_ar_xy_philosophy" +"task_alias": "ar_xy philosophy" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..560d3ff43b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_humanities" +"task": "mmlu_ar_xy_prehistory" +"task_alias": "ar_xy prehistory" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..523834b35b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_other" +"task": "mmlu_ar_xy_professional_accounting" +"task_alias": "ar_xy professional accounting" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..52c702443b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_humanities" +"task": "mmlu_ar_xy_professional_law" +"task_alias": "ar_xy professional law" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..4358a851b6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_other" +"task": "mmlu_ar_xy_professional_medicine" +"task_alias": "ar_xy professional medicine" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..1c6757cd59 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_social_sciences" +"task": "mmlu_ar_xy_professional_psychology" +"task_alias": "ar_xy professional psychology" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..af660d809c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_social_sciences" +"task": "mmlu_ar_xy_public_relations" +"task_alias": "ar_xy public relations" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..acbe8ea14c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_social_sciences" +"task": "mmlu_ar_xy_security_studies" +"task_alias": "ar_xy security studies" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..e291024f17 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_social_sciences" +"task": "mmlu_ar_xy_sociology" +"task_alias": "ar_xy sociology" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..079bf5ed8d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_social_sciences" +"task": "mmlu_ar_xy_us_foreign_policy" +"task_alias": "ar_xy us foreign policy" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..264f44791f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_other" +"task": "mmlu_ar_xy_virology" +"task_alias": "ar_xy virology" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..ac0ebb0063 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ar_xy/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the Arabic (Generic) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ar_xy_humanities" +"task": "mmlu_ar_xy_world_religions" +"task_alias": "ar_xy world religions" +"test_split": "AR_XY" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/_openai_mmlu_bn_bd.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/_openai_mmlu_bn_bd.yaml new file mode 100644 index 0000000000..3c74ab823b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/_openai_mmlu_bn_bd.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_bn_bd +group_alias: bn_bd +metadata: + version: 1.0.0 +task: +- mmlu_bn_bd_tasks diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..4f14a344f6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_abstract_algebra" +"task_alias": "bn_bd abstract algebra" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..8b949e2707 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_anatomy" +"task_alias": "bn_bd anatomy" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..6922ffda79 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_astronomy" +"task_alias": "bn_bd astronomy" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..ed152d9e3c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_other" +"task": "mmlu_bn_bd_business_ethics" +"task_alias": "bn_bd business ethics" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..95462c09aa --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_other" +"task": "mmlu_bn_bd_clinical_knowledge" +"task_alias": "bn_bd clinical knowledge" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..7b3a38e822 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_college_biology" +"task_alias": "bn_bd college biology" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..c42b587c81 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_college_chemistry" +"task_alias": "bn_bd college chemistry" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..ca80ad31ea --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_college_computer_science" +"task_alias": "bn_bd college computer science" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..d58d0ebda5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_college_mathematics" +"task_alias": "bn_bd college mathematics" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..1fe6ffe989 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_other" +"task": "mmlu_bn_bd_college_medicine" +"task_alias": "bn_bd college medicine" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..8767095548 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_college_physics" +"task_alias": "bn_bd college physics" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..2526439ec8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_computer_security" +"task_alias": "bn_bd computer security" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..4ba06e406d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_conceptual_physics" +"task_alias": "bn_bd conceptual physics" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..a61fc5d0bf --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_social_sciences" +"task": "mmlu_bn_bd_econometrics" +"task_alias": "bn_bd econometrics" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..d719ab719d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_electrical_engineering" +"task_alias": "bn_bd electrical engineering" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..de2a01390a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_elementary_mathematics" +"task_alias": "bn_bd elementary mathematics" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..878c45e3cf --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_humanities" +"task": "mmlu_bn_bd_formal_logic" +"task_alias": "bn_bd formal logic" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..e361221beb --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_other" +"task": "mmlu_bn_bd_global_facts" +"task_alias": "bn_bd global facts" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..518d59ab10 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_high_school_biology" +"task_alias": "bn_bd high school biology" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..cd4d092661 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_high_school_chemistry" +"task_alias": "bn_bd high school chemistry" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..3177927c4b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_high_school_computer_science" +"task_alias": "bn_bd high school computer science" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..c22b2ab1fb --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_humanities" +"task": "mmlu_bn_bd_high_school_european_history" +"task_alias": "bn_bd high school european history" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..15150a281e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_social_sciences" +"task": "mmlu_bn_bd_high_school_geography" +"task_alias": "bn_bd high school geography" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..aacc0a5ac8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_social_sciences" +"task": "mmlu_bn_bd_high_school_government_and_politics" +"task_alias": "bn_bd high school government and politics" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..5d29f9cac1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_social_sciences" +"task": "mmlu_bn_bd_high_school_macroeconomics" +"task_alias": "bn_bd high school macroeconomics" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..de0a7bfb08 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_high_school_mathematics" +"task_alias": "bn_bd high school mathematics" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..82e213596e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_social_sciences" +"task": "mmlu_bn_bd_high_school_microeconomics" +"task_alias": "bn_bd high school microeconomics" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..ccbd1d29dd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_high_school_physics" +"task_alias": "bn_bd high school physics" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..765048ffa4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_social_sciences" +"task": "mmlu_bn_bd_high_school_psychology" +"task_alias": "bn_bd high school psychology" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..14973060d9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_high_school_statistics" +"task_alias": "bn_bd high school statistics" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..0671bf49e0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_humanities" +"task": "mmlu_bn_bd_high_school_us_history" +"task_alias": "bn_bd high school us history" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..1cc3547ffb --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_humanities" +"task": "mmlu_bn_bd_high_school_world_history" +"task_alias": "bn_bd high school world history" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..b850d9e65a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_other" +"task": "mmlu_bn_bd_human_aging" +"task_alias": "bn_bd human aging" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..da63d11cd5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_social_sciences" +"task": "mmlu_bn_bd_human_sexuality" +"task_alias": "bn_bd human sexuality" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..287da1b9d1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_humanities" +"task": "mmlu_bn_bd_international_law" +"task_alias": "bn_bd international law" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..719b06624d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_humanities" +"task": "mmlu_bn_bd_jurisprudence" +"task_alias": "bn_bd jurisprudence" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..7423a8c4a7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_humanities" +"task": "mmlu_bn_bd_logical_fallacies" +"task_alias": "bn_bd logical fallacies" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..78fa8fa9e1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_stem" +"task": "mmlu_bn_bd_machine_learning" +"task_alias": "bn_bd machine learning" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_management.yaml new file mode 100644 index 0000000000..a51699e4ce --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_other" +"task": "mmlu_bn_bd_management" +"task_alias": "bn_bd management" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..d5743c5a11 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_other" +"task": "mmlu_bn_bd_marketing" +"task_alias": "bn_bd marketing" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..14d7bd48e2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_other" +"task": "mmlu_bn_bd_medical_genetics" +"task_alias": "bn_bd medical genetics" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..7b99029539 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_other" +"task": "mmlu_bn_bd_miscellaneous" +"task_alias": "bn_bd miscellaneous" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..563e89c9e8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_humanities" +"task": "mmlu_bn_bd_moral_disputes" +"task_alias": "bn_bd moral disputes" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..67efa659ad --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_humanities" +"task": "mmlu_bn_bd_moral_scenarios" +"task_alias": "bn_bd moral scenarios" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..836ef14908 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_other" +"task": "mmlu_bn_bd_nutrition" +"task_alias": "bn_bd nutrition" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..7da78f8a4c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_humanities" +"task": "mmlu_bn_bd_philosophy" +"task_alias": "bn_bd philosophy" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..e553a458aa --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_humanities" +"task": "mmlu_bn_bd_prehistory" +"task_alias": "bn_bd prehistory" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..2be33749c4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_other" +"task": "mmlu_bn_bd_professional_accounting" +"task_alias": "bn_bd professional accounting" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..d177286000 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_humanities" +"task": "mmlu_bn_bd_professional_law" +"task_alias": "bn_bd professional law" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..450ee0cd57 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_other" +"task": "mmlu_bn_bd_professional_medicine" +"task_alias": "bn_bd professional medicine" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..186d3d463d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_social_sciences" +"task": "mmlu_bn_bd_professional_psychology" +"task_alias": "bn_bd professional psychology" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..c1639e0879 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_social_sciences" +"task": "mmlu_bn_bd_public_relations" +"task_alias": "bn_bd public relations" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..19f44c9283 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_social_sciences" +"task": "mmlu_bn_bd_security_studies" +"task_alias": "bn_bd security studies" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..044689cc33 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_social_sciences" +"task": "mmlu_bn_bd_sociology" +"task_alias": "bn_bd sociology" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..bb47a3d838 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_social_sciences" +"task": "mmlu_bn_bd_us_foreign_policy" +"task_alias": "bn_bd us foreign policy" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..e50250d4e4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_other" +"task": "mmlu_bn_bd_virology" +"task_alias": "bn_bd virology" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..36803b302e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/bn_bd/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the Bengali (Bangladesh) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_bn_bd_humanities" +"task": "mmlu_bn_bd_world_religions" +"task_alias": "bn_bd world religions" +"test_split": "BN_BD" diff --git a/lm_eval/tasks/openai_mmlu/de_de/_openai_mmlu_de_de.yaml b/lm_eval/tasks/openai_mmlu/de_de/_openai_mmlu_de_de.yaml new file mode 100644 index 0000000000..e723513152 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/_openai_mmlu_de_de.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_de_de +group_alias: de_de +metadata: + version: 1.0.0 +task: +- mmlu_de_de_tasks diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..3f9015fd82 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_abstract_algebra" +"task_alias": "de_de abstract algebra" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..5c90bd8001 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_anatomy" +"task_alias": "de_de anatomy" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..ec9fb4c2c7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_astronomy" +"task_alias": "de_de astronomy" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..e1f6c9675d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_other" +"task": "mmlu_de_de_business_ethics" +"task_alias": "de_de business ethics" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..e58d7d93fd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_other" +"task": "mmlu_de_de_clinical_knowledge" +"task_alias": "de_de clinical knowledge" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..3190ab564b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_college_biology" +"task_alias": "de_de college biology" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..16dd5709a6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_college_chemistry" +"task_alias": "de_de college chemistry" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..c38c20ff84 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_college_computer_science" +"task_alias": "de_de college computer science" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..e4622023b4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_college_mathematics" +"task_alias": "de_de college mathematics" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..39238fda1d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_other" +"task": "mmlu_de_de_college_medicine" +"task_alias": "de_de college medicine" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..cd4960a8f3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_college_physics" +"task_alias": "de_de college physics" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..2a137f228e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_computer_security" +"task_alias": "de_de computer security" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..d53d4dd428 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_conceptual_physics" +"task_alias": "de_de conceptual physics" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..824068ee1f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_social_sciences" +"task": "mmlu_de_de_econometrics" +"task_alias": "de_de econometrics" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..c7edb2f8d1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_electrical_engineering" +"task_alias": "de_de electrical engineering" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..6e38ff75c5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_elementary_mathematics" +"task_alias": "de_de elementary mathematics" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..b6a89d6d6a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_humanities" +"task": "mmlu_de_de_formal_logic" +"task_alias": "de_de formal logic" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..8638e7ac6e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_other" +"task": "mmlu_de_de_global_facts" +"task_alias": "de_de global facts" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..f45174a7f4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_high_school_biology" +"task_alias": "de_de high school biology" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..8944286cd7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_high_school_chemistry" +"task_alias": "de_de high school chemistry" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..9ddb527aa4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_high_school_computer_science" +"task_alias": "de_de high school computer science" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..514aaa8b18 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_humanities" +"task": "mmlu_de_de_high_school_european_history" +"task_alias": "de_de high school european history" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..2fdacd56b3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_social_sciences" +"task": "mmlu_de_de_high_school_geography" +"task_alias": "de_de high school geography" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..925bfaf750 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_social_sciences" +"task": "mmlu_de_de_high_school_government_and_politics" +"task_alias": "de_de high school government and politics" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..bc981e04ad --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_social_sciences" +"task": "mmlu_de_de_high_school_macroeconomics" +"task_alias": "de_de high school macroeconomics" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..7ab331f0b6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_high_school_mathematics" +"task_alias": "de_de high school mathematics" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..55d771ff38 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_social_sciences" +"task": "mmlu_de_de_high_school_microeconomics" +"task_alias": "de_de high school microeconomics" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..7bc45e8a93 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_high_school_physics" +"task_alias": "de_de high school physics" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..a87ab76464 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_social_sciences" +"task": "mmlu_de_de_high_school_psychology" +"task_alias": "de_de high school psychology" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..8dd3418152 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_high_school_statistics" +"task_alias": "de_de high school statistics" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..4b16798b56 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_humanities" +"task": "mmlu_de_de_high_school_us_history" +"task_alias": "de_de high school us history" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..ee6a22b25b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_humanities" +"task": "mmlu_de_de_high_school_world_history" +"task_alias": "de_de high school world history" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..0a4a9cef74 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_other" +"task": "mmlu_de_de_human_aging" +"task_alias": "de_de human aging" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..74718e7303 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_social_sciences" +"task": "mmlu_de_de_human_sexuality" +"task_alias": "de_de human sexuality" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..7eb22186e2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_humanities" +"task": "mmlu_de_de_international_law" +"task_alias": "de_de international law" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..ac5d35f956 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_humanities" +"task": "mmlu_de_de_jurisprudence" +"task_alias": "de_de jurisprudence" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..b5f786a638 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_humanities" +"task": "mmlu_de_de_logical_fallacies" +"task_alias": "de_de logical fallacies" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..dd4c60f163 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_stem" +"task": "mmlu_de_de_machine_learning" +"task_alias": "de_de machine learning" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_management.yaml new file mode 100644 index 0000000000..0fbb7b9c02 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_other" +"task": "mmlu_de_de_management" +"task_alias": "de_de management" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..be20683536 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_other" +"task": "mmlu_de_de_marketing" +"task_alias": "de_de marketing" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..34fd455c36 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_other" +"task": "mmlu_de_de_medical_genetics" +"task_alias": "de_de medical genetics" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..3f216e733b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_other" +"task": "mmlu_de_de_miscellaneous" +"task_alias": "de_de miscellaneous" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..fa410b67fb --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_humanities" +"task": "mmlu_de_de_moral_disputes" +"task_alias": "de_de moral disputes" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..863c04a952 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_humanities" +"task": "mmlu_de_de_moral_scenarios" +"task_alias": "de_de moral scenarios" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..7f90346156 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_other" +"task": "mmlu_de_de_nutrition" +"task_alias": "de_de nutrition" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..6e33bf9b67 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_humanities" +"task": "mmlu_de_de_philosophy" +"task_alias": "de_de philosophy" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..f4694b49b4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_humanities" +"task": "mmlu_de_de_prehistory" +"task_alias": "de_de prehistory" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..9771dde9e5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_other" +"task": "mmlu_de_de_professional_accounting" +"task_alias": "de_de professional accounting" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..580fe35a0a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_humanities" +"task": "mmlu_de_de_professional_law" +"task_alias": "de_de professional law" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..c995455692 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_other" +"task": "mmlu_de_de_professional_medicine" +"task_alias": "de_de professional medicine" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..17d5c73a43 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_social_sciences" +"task": "mmlu_de_de_professional_psychology" +"task_alias": "de_de professional psychology" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..3e88a960ee --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_social_sciences" +"task": "mmlu_de_de_public_relations" +"task_alias": "de_de public relations" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..914ae4dd19 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_social_sciences" +"task": "mmlu_de_de_security_studies" +"task_alias": "de_de security studies" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..c0fba17d16 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_social_sciences" +"task": "mmlu_de_de_sociology" +"task_alias": "de_de sociology" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..1252d982ef --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_social_sciences" +"task": "mmlu_de_de_us_foreign_policy" +"task_alias": "de_de us foreign policy" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..8c4787eb28 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_other" +"task": "mmlu_de_de_virology" +"task_alias": "de_de virology" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..59170f2262 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/de_de/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the German (Germany) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_de_de_humanities" +"task": "mmlu_de_de_world_religions" +"task_alias": "de_de world religions" +"test_split": "DE_DE" diff --git a/lm_eval/tasks/openai_mmlu/en_us/_openai_mmlu_en_us.yaml b/lm_eval/tasks/openai_mmlu/en_us/_openai_mmlu_en_us.yaml new file mode 100644 index 0000000000..c2b0015712 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/_openai_mmlu_en_us.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_en_us +group_alias: en_us +metadata: + version: 1.0.0 +task: +- mmlu_en_us_tasks diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..c2a2426ea8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_abstract_algebra" +"task_alias": "en_us abstract algebra" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..570baaff09 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_anatomy" +"task_alias": "en_us anatomy" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..de50120228 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_astronomy" +"task_alias": "en_us astronomy" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..a5a5af2317 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_other" +"task": "mmlu_en_us_business_ethics" +"task_alias": "en_us business ethics" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..e7f7649344 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_other" +"task": "mmlu_en_us_clinical_knowledge" +"task_alias": "en_us clinical knowledge" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..cc9e579538 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_college_biology" +"task_alias": "en_us college biology" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..ccffbb6da9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_college_chemistry" +"task_alias": "en_us college chemistry" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..00c3bf1290 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_college_computer_science" +"task_alias": "en_us college computer science" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..68f950044a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_college_mathematics" +"task_alias": "en_us college mathematics" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..0de26b6ed6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_other" +"task": "mmlu_en_us_college_medicine" +"task_alias": "en_us college medicine" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..a4509fd633 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_college_physics" +"task_alias": "en_us college physics" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..6d0295bfd7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_computer_security" +"task_alias": "en_us computer security" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..091e0f9d96 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_conceptual_physics" +"task_alias": "en_us conceptual physics" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..0dc37d98b1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_social_sciences" +"task": "mmlu_en_us_econometrics" +"task_alias": "en_us econometrics" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..ba80308ebb --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_electrical_engineering" +"task_alias": "en_us electrical engineering" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..7c502bc39c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_elementary_mathematics" +"task_alias": "en_us elementary mathematics" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..3c5718a184 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_humanities" +"task": "mmlu_en_us_formal_logic" +"task_alias": "en_us formal logic" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..de87d715e0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_other" +"task": "mmlu_en_us_global_facts" +"task_alias": "en_us global facts" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..7e2997a224 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_high_school_biology" +"task_alias": "en_us high school biology" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..1aa69e4970 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_high_school_chemistry" +"task_alias": "en_us high school chemistry" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..d8f97c7ee8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_high_school_computer_science" +"task_alias": "en_us high school computer science" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..94d67b803f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_humanities" +"task": "mmlu_en_us_high_school_european_history" +"task_alias": "en_us high school european history" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..710899e68c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_social_sciences" +"task": "mmlu_en_us_high_school_geography" +"task_alias": "en_us high school geography" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..67526b4b18 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_social_sciences" +"task": "mmlu_en_us_high_school_government_and_politics" +"task_alias": "en_us high school government and politics" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..386f266787 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_social_sciences" +"task": "mmlu_en_us_high_school_macroeconomics" +"task_alias": "en_us high school macroeconomics" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..902b87a5dc --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_high_school_mathematics" +"task_alias": "en_us high school mathematics" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..b4dc850761 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_social_sciences" +"task": "mmlu_en_us_high_school_microeconomics" +"task_alias": "en_us high school microeconomics" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..35b08f4675 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_high_school_physics" +"task_alias": "en_us high school physics" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..df1331bf6f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_social_sciences" +"task": "mmlu_en_us_high_school_psychology" +"task_alias": "en_us high school psychology" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..10335c582f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_high_school_statistics" +"task_alias": "en_us high school statistics" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..e3e3061937 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_humanities" +"task": "mmlu_en_us_high_school_us_history" +"task_alias": "en_us high school us history" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..287bd89909 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_humanities" +"task": "mmlu_en_us_high_school_world_history" +"task_alias": "en_us high school world history" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..7c2c86443b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_other" +"task": "mmlu_en_us_human_aging" +"task_alias": "en_us human aging" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..d44877f50b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_social_sciences" +"task": "mmlu_en_us_human_sexuality" +"task_alias": "en_us human sexuality" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..63b4a9a2e6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_humanities" +"task": "mmlu_en_us_international_law" +"task_alias": "en_us international law" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..6311f8f0e7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_humanities" +"task": "mmlu_en_us_jurisprudence" +"task_alias": "en_us jurisprudence" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..561ee31c3e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_humanities" +"task": "mmlu_en_us_logical_fallacies" +"task_alias": "en_us logical fallacies" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..bc1f511dac --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_stem" +"task": "mmlu_en_us_machine_learning" +"task_alias": "en_us machine learning" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_management.yaml new file mode 100644 index 0000000000..f60816059f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_other" +"task": "mmlu_en_us_management" +"task_alias": "en_us management" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..bb822d06c7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_other" +"task": "mmlu_en_us_marketing" +"task_alias": "en_us marketing" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..771ad892ea --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_other" +"task": "mmlu_en_us_medical_genetics" +"task_alias": "en_us medical genetics" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..a5dcf436b3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_other" +"task": "mmlu_en_us_miscellaneous" +"task_alias": "en_us miscellaneous" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..123d7a57f9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_humanities" +"task": "mmlu_en_us_moral_disputes" +"task_alias": "en_us moral disputes" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..4dbff84e9e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_humanities" +"task": "mmlu_en_us_moral_scenarios" +"task_alias": "en_us moral scenarios" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..58da549a43 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_other" +"task": "mmlu_en_us_nutrition" +"task_alias": "en_us nutrition" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..c06f083035 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_humanities" +"task": "mmlu_en_us_philosophy" +"task_alias": "en_us philosophy" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..08b75cd5c8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_humanities" +"task": "mmlu_en_us_prehistory" +"task_alias": "en_us prehistory" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..3e76ee7e9a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_other" +"task": "mmlu_en_us_professional_accounting" +"task_alias": "en_us professional accounting" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..b68ac66384 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_humanities" +"task": "mmlu_en_us_professional_law" +"task_alias": "en_us professional law" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..912315ec2c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_other" +"task": "mmlu_en_us_professional_medicine" +"task_alias": "en_us professional medicine" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..c0b7bdda26 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_social_sciences" +"task": "mmlu_en_us_professional_psychology" +"task_alias": "en_us professional psychology" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..1bb118fb04 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_social_sciences" +"task": "mmlu_en_us_public_relations" +"task_alias": "en_us public relations" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..4a7dfa8f79 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_social_sciences" +"task": "mmlu_en_us_security_studies" +"task_alias": "en_us security studies" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..e8bdc7b47e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_social_sciences" +"task": "mmlu_en_us_sociology" +"task_alias": "en_us sociology" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..9007d1deda --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_social_sciences" +"task": "mmlu_en_us_us_foreign_policy" +"task_alias": "en_us us foreign policy" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..2082acef78 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_other" +"task": "mmlu_en_us_virology" +"task_alias": "en_us virology" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..12d9ec141e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/en_us/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the English (United States) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_en_us_humanities" +"task": "mmlu_en_us_world_religions" +"task_alias": "en_us world religions" +"test_split": "EN_US" diff --git a/lm_eval/tasks/openai_mmlu/es_la/_openai_mmlu_es_la.yaml b/lm_eval/tasks/openai_mmlu/es_la/_openai_mmlu_es_la.yaml new file mode 100644 index 0000000000..74738d0a4c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/_openai_mmlu_es_la.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_es_la +group_alias: es_la +metadata: + version: 1.0.0 +task: +- mmlu_es_la_tasks diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..310b6bab4e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_abstract_algebra" +"task_alias": "es_la abstract algebra" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..e5a1513c53 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_anatomy" +"task_alias": "es_la anatomy" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..09ed4b21cb --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_astronomy" +"task_alias": "es_la astronomy" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..93a04e08c9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_other" +"task": "mmlu_es_la_business_ethics" +"task_alias": "es_la business ethics" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..ab76bae6c3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_other" +"task": "mmlu_es_la_clinical_knowledge" +"task_alias": "es_la clinical knowledge" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..7c330a5c78 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_college_biology" +"task_alias": "es_la college biology" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..d047078992 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_college_chemistry" +"task_alias": "es_la college chemistry" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..433072429a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_college_computer_science" +"task_alias": "es_la college computer science" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..e4ff304053 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_college_mathematics" +"task_alias": "es_la college mathematics" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..42c7c7e3e1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_other" +"task": "mmlu_es_la_college_medicine" +"task_alias": "es_la college medicine" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..2c1d90e185 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_college_physics" +"task_alias": "es_la college physics" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..845aeabe41 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_computer_security" +"task_alias": "es_la computer security" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..c41a70d426 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_conceptual_physics" +"task_alias": "es_la conceptual physics" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..d28efcb0f9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_social_sciences" +"task": "mmlu_es_la_econometrics" +"task_alias": "es_la econometrics" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..6689712e22 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_electrical_engineering" +"task_alias": "es_la electrical engineering" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..60b582937e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_elementary_mathematics" +"task_alias": "es_la elementary mathematics" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..2061d108e7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_humanities" +"task": "mmlu_es_la_formal_logic" +"task_alias": "es_la formal logic" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..6734a0d6ba --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_other" +"task": "mmlu_es_la_global_facts" +"task_alias": "es_la global facts" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..2f02d8e4ce --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_high_school_biology" +"task_alias": "es_la high school biology" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..2c84a4392b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_high_school_chemistry" +"task_alias": "es_la high school chemistry" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..346f0192de --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_high_school_computer_science" +"task_alias": "es_la high school computer science" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..e7205c3dea --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_humanities" +"task": "mmlu_es_la_high_school_european_history" +"task_alias": "es_la high school european history" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..46e24bebfa --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_social_sciences" +"task": "mmlu_es_la_high_school_geography" +"task_alias": "es_la high school geography" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..302fa5423d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_social_sciences" +"task": "mmlu_es_la_high_school_government_and_politics" +"task_alias": "es_la high school government and politics" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..77d5b0ba91 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_social_sciences" +"task": "mmlu_es_la_high_school_macroeconomics" +"task_alias": "es_la high school macroeconomics" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..d302b92753 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_high_school_mathematics" +"task_alias": "es_la high school mathematics" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..31e7fd5765 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_social_sciences" +"task": "mmlu_es_la_high_school_microeconomics" +"task_alias": "es_la high school microeconomics" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..cfc00811cd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_high_school_physics" +"task_alias": "es_la high school physics" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..07221b473a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_social_sciences" +"task": "mmlu_es_la_high_school_psychology" +"task_alias": "es_la high school psychology" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..133e2f314d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_high_school_statistics" +"task_alias": "es_la high school statistics" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..8b862a102c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_humanities" +"task": "mmlu_es_la_high_school_us_history" +"task_alias": "es_la high school us history" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..1c455c8548 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_humanities" +"task": "mmlu_es_la_high_school_world_history" +"task_alias": "es_la high school world history" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..4751778c6b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_other" +"task": "mmlu_es_la_human_aging" +"task_alias": "es_la human aging" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..f2373eea70 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_social_sciences" +"task": "mmlu_es_la_human_sexuality" +"task_alias": "es_la human sexuality" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..09cad206e7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_humanities" +"task": "mmlu_es_la_international_law" +"task_alias": "es_la international law" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..3bd7373570 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_humanities" +"task": "mmlu_es_la_jurisprudence" +"task_alias": "es_la jurisprudence" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..6439b4d671 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_humanities" +"task": "mmlu_es_la_logical_fallacies" +"task_alias": "es_la logical fallacies" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..14953f7df4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_stem" +"task": "mmlu_es_la_machine_learning" +"task_alias": "es_la machine learning" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_management.yaml new file mode 100644 index 0000000000..dd7830ec18 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_other" +"task": "mmlu_es_la_management" +"task_alias": "es_la management" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..0e6cf48b9e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_other" +"task": "mmlu_es_la_marketing" +"task_alias": "es_la marketing" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..5580e09224 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_other" +"task": "mmlu_es_la_medical_genetics" +"task_alias": "es_la medical genetics" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..254d11defe --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_other" +"task": "mmlu_es_la_miscellaneous" +"task_alias": "es_la miscellaneous" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..42dd008fe5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_humanities" +"task": "mmlu_es_la_moral_disputes" +"task_alias": "es_la moral disputes" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..239738a9d4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_humanities" +"task": "mmlu_es_la_moral_scenarios" +"task_alias": "es_la moral scenarios" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..ff0605d5a1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_other" +"task": "mmlu_es_la_nutrition" +"task_alias": "es_la nutrition" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..a2e5841380 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_humanities" +"task": "mmlu_es_la_philosophy" +"task_alias": "es_la philosophy" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..6720515c9b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_humanities" +"task": "mmlu_es_la_prehistory" +"task_alias": "es_la prehistory" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..7b6c11ee71 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_other" +"task": "mmlu_es_la_professional_accounting" +"task_alias": "es_la professional accounting" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..50327e5b10 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_humanities" +"task": "mmlu_es_la_professional_law" +"task_alias": "es_la professional law" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..9d4a1a22fa --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_other" +"task": "mmlu_es_la_professional_medicine" +"task_alias": "es_la professional medicine" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..9b5630a329 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_social_sciences" +"task": "mmlu_es_la_professional_psychology" +"task_alias": "es_la professional psychology" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..4b1b5bc448 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_social_sciences" +"task": "mmlu_es_la_public_relations" +"task_alias": "es_la public relations" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..94e650b952 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_social_sciences" +"task": "mmlu_es_la_security_studies" +"task_alias": "es_la security studies" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..06d3edbbe7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_social_sciences" +"task": "mmlu_es_la_sociology" +"task_alias": "es_la sociology" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..48afff0428 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_social_sciences" +"task": "mmlu_es_la_us_foreign_policy" +"task_alias": "es_la us foreign policy" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..d666bc8179 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_other" +"task": "mmlu_es_la_virology" +"task_alias": "es_la virology" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..447f46a9bb --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/es_la/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the Spanish (Latin America) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_es_la_humanities" +"task": "mmlu_es_la_world_religions" +"task_alias": "es_la world religions" +"test_split": "ES_LA" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/_openai_mmlu_fr_fr.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/_openai_mmlu_fr_fr.yaml new file mode 100644 index 0000000000..5319dd8b2d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/_openai_mmlu_fr_fr.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_fr_fr +group_alias: fr_fr +metadata: + version: 1.0.0 +task: +- mmlu_fr_fr_tasks diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..4ed3cc916a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_abstract_algebra" +"task_alias": "fr_fr abstract algebra" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..1df124b015 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_anatomy" +"task_alias": "fr_fr anatomy" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..9cd5051b3f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_astronomy" +"task_alias": "fr_fr astronomy" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..0aac622dc3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_other" +"task": "mmlu_fr_fr_business_ethics" +"task_alias": "fr_fr business ethics" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..8db5b8e3b2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_other" +"task": "mmlu_fr_fr_clinical_knowledge" +"task_alias": "fr_fr clinical knowledge" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..1b497345ff --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_college_biology" +"task_alias": "fr_fr college biology" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..b09967a96f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_college_chemistry" +"task_alias": "fr_fr college chemistry" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..743fd68a82 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_college_computer_science" +"task_alias": "fr_fr college computer science" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..f9d1c66422 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_college_mathematics" +"task_alias": "fr_fr college mathematics" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..3f2c71b1cd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_other" +"task": "mmlu_fr_fr_college_medicine" +"task_alias": "fr_fr college medicine" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..d5d5b48422 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_college_physics" +"task_alias": "fr_fr college physics" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..0c76eef696 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_computer_security" +"task_alias": "fr_fr computer security" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..5a6c2d86a7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_conceptual_physics" +"task_alias": "fr_fr conceptual physics" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..5edc2e2afa --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_social_sciences" +"task": "mmlu_fr_fr_econometrics" +"task_alias": "fr_fr econometrics" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..a6abaf6029 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_electrical_engineering" +"task_alias": "fr_fr electrical engineering" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..c684f83c6a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_elementary_mathematics" +"task_alias": "fr_fr elementary mathematics" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..992a79fd8e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_humanities" +"task": "mmlu_fr_fr_formal_logic" +"task_alias": "fr_fr formal logic" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..85dc4afdd6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_other" +"task": "mmlu_fr_fr_global_facts" +"task_alias": "fr_fr global facts" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..721809b99b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_high_school_biology" +"task_alias": "fr_fr high school biology" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..1a3dc35f55 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_high_school_chemistry" +"task_alias": "fr_fr high school chemistry" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..8a4ac08fd6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_high_school_computer_science" +"task_alias": "fr_fr high school computer science" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..71eb10cece --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_humanities" +"task": "mmlu_fr_fr_high_school_european_history" +"task_alias": "fr_fr high school european history" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..171ed42f8f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_social_sciences" +"task": "mmlu_fr_fr_high_school_geography" +"task_alias": "fr_fr high school geography" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..c54ed30a24 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_social_sciences" +"task": "mmlu_fr_fr_high_school_government_and_politics" +"task_alias": "fr_fr high school government and politics" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..3ae8e1734f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_social_sciences" +"task": "mmlu_fr_fr_high_school_macroeconomics" +"task_alias": "fr_fr high school macroeconomics" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..493d04057a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_high_school_mathematics" +"task_alias": "fr_fr high school mathematics" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..846fac0fb9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_social_sciences" +"task": "mmlu_fr_fr_high_school_microeconomics" +"task_alias": "fr_fr high school microeconomics" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..ec47295103 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_high_school_physics" +"task_alias": "fr_fr high school physics" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..e5d712bb7c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_social_sciences" +"task": "mmlu_fr_fr_high_school_psychology" +"task_alias": "fr_fr high school psychology" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..7b9001e844 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_high_school_statistics" +"task_alias": "fr_fr high school statistics" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..d262785429 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_humanities" +"task": "mmlu_fr_fr_high_school_us_history" +"task_alias": "fr_fr high school us history" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..5179088b7b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_humanities" +"task": "mmlu_fr_fr_high_school_world_history" +"task_alias": "fr_fr high school world history" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..92b8d8b223 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_other" +"task": "mmlu_fr_fr_human_aging" +"task_alias": "fr_fr human aging" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..28903049dc --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_social_sciences" +"task": "mmlu_fr_fr_human_sexuality" +"task_alias": "fr_fr human sexuality" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..0d6e26629b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_humanities" +"task": "mmlu_fr_fr_international_law" +"task_alias": "fr_fr international law" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..50e4d2febb --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_humanities" +"task": "mmlu_fr_fr_jurisprudence" +"task_alias": "fr_fr jurisprudence" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..a8c8e0af7b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_humanities" +"task": "mmlu_fr_fr_logical_fallacies" +"task_alias": "fr_fr logical fallacies" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..d890afeb8c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_stem" +"task": "mmlu_fr_fr_machine_learning" +"task_alias": "fr_fr machine learning" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_management.yaml new file mode 100644 index 0000000000..461c3c5234 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_other" +"task": "mmlu_fr_fr_management" +"task_alias": "fr_fr management" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..2fb0b364a7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_other" +"task": "mmlu_fr_fr_marketing" +"task_alias": "fr_fr marketing" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..d6ad89e9a1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_other" +"task": "mmlu_fr_fr_medical_genetics" +"task_alias": "fr_fr medical genetics" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..1e960191be --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_other" +"task": "mmlu_fr_fr_miscellaneous" +"task_alias": "fr_fr miscellaneous" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..cd41652498 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_humanities" +"task": "mmlu_fr_fr_moral_disputes" +"task_alias": "fr_fr moral disputes" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..752f1750d0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_humanities" +"task": "mmlu_fr_fr_moral_scenarios" +"task_alias": "fr_fr moral scenarios" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..fc118addba --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_other" +"task": "mmlu_fr_fr_nutrition" +"task_alias": "fr_fr nutrition" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..5653ee1cfd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_humanities" +"task": "mmlu_fr_fr_philosophy" +"task_alias": "fr_fr philosophy" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..4de1d8526e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_humanities" +"task": "mmlu_fr_fr_prehistory" +"task_alias": "fr_fr prehistory" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..ddaa1ed175 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_other" +"task": "mmlu_fr_fr_professional_accounting" +"task_alias": "fr_fr professional accounting" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..f82caf9de6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_humanities" +"task": "mmlu_fr_fr_professional_law" +"task_alias": "fr_fr professional law" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..5167a799d3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_other" +"task": "mmlu_fr_fr_professional_medicine" +"task_alias": "fr_fr professional medicine" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..cf47361542 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_social_sciences" +"task": "mmlu_fr_fr_professional_psychology" +"task_alias": "fr_fr professional psychology" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..53e3e06ee3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_social_sciences" +"task": "mmlu_fr_fr_public_relations" +"task_alias": "fr_fr public relations" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..910b149b48 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_social_sciences" +"task": "mmlu_fr_fr_security_studies" +"task_alias": "fr_fr security studies" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..f32dc6d6bc --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_social_sciences" +"task": "mmlu_fr_fr_sociology" +"task_alias": "fr_fr sociology" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..99b0146950 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_social_sciences" +"task": "mmlu_fr_fr_us_foreign_policy" +"task_alias": "fr_fr us foreign policy" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..e59d9ffb85 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_other" +"task": "mmlu_fr_fr_virology" +"task_alias": "fr_fr virology" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..09d511a749 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/fr_fr/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the French (France) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_fr_fr_humanities" +"task": "mmlu_fr_fr_world_religions" +"task_alias": "fr_fr world religions" +"test_split": "FR_FR" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/_openai_mmlu_hi_in.yaml b/lm_eval/tasks/openai_mmlu/hi_in/_openai_mmlu_hi_in.yaml new file mode 100644 index 0000000000..54e137a479 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/_openai_mmlu_hi_in.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_hi_in +group_alias: hi_in +metadata: + version: 1.0.0 +task: +- mmlu_hi_in_tasks diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..adfafdb0ec --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_abstract_algebra" +"task_alias": "hi_in abstract algebra" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..30f60cdb68 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_anatomy" +"task_alias": "hi_in anatomy" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..2c7bf64b80 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_astronomy" +"task_alias": "hi_in astronomy" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..9e9c5cc177 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_other" +"task": "mmlu_hi_in_business_ethics" +"task_alias": "hi_in business ethics" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..103c4c91ac --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_other" +"task": "mmlu_hi_in_clinical_knowledge" +"task_alias": "hi_in clinical knowledge" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..f49087d9b0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_college_biology" +"task_alias": "hi_in college biology" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..0ed33e4197 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_college_chemistry" +"task_alias": "hi_in college chemistry" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..1e6586c94d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_college_computer_science" +"task_alias": "hi_in college computer science" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..2a8b45bab7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_college_mathematics" +"task_alias": "hi_in college mathematics" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..e03a0e964c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_other" +"task": "mmlu_hi_in_college_medicine" +"task_alias": "hi_in college medicine" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..31a8fa444f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_college_physics" +"task_alias": "hi_in college physics" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..7b2624e489 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_computer_security" +"task_alias": "hi_in computer security" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..b171a74f7c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_conceptual_physics" +"task_alias": "hi_in conceptual physics" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..f7e03d3d5a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_social_sciences" +"task": "mmlu_hi_in_econometrics" +"task_alias": "hi_in econometrics" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..428e51f360 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_electrical_engineering" +"task_alias": "hi_in electrical engineering" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..bd8cc4a505 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_elementary_mathematics" +"task_alias": "hi_in elementary mathematics" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..d8bcaa92f8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_humanities" +"task": "mmlu_hi_in_formal_logic" +"task_alias": "hi_in formal logic" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..f5e56b8e6e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_other" +"task": "mmlu_hi_in_global_facts" +"task_alias": "hi_in global facts" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..b8726ff998 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_high_school_biology" +"task_alias": "hi_in high school biology" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..4c65ccb515 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_high_school_chemistry" +"task_alias": "hi_in high school chemistry" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..dffdaa1dc3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_high_school_computer_science" +"task_alias": "hi_in high school computer science" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..92cc0a49d9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_humanities" +"task": "mmlu_hi_in_high_school_european_history" +"task_alias": "hi_in high school european history" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..4d35b03390 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_social_sciences" +"task": "mmlu_hi_in_high_school_geography" +"task_alias": "hi_in high school geography" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..90fdb23237 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_social_sciences" +"task": "mmlu_hi_in_high_school_government_and_politics" +"task_alias": "hi_in high school government and politics" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..ef7b94a695 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_social_sciences" +"task": "mmlu_hi_in_high_school_macroeconomics" +"task_alias": "hi_in high school macroeconomics" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..60ae5e169c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_high_school_mathematics" +"task_alias": "hi_in high school mathematics" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..18c775b224 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_social_sciences" +"task": "mmlu_hi_in_high_school_microeconomics" +"task_alias": "hi_in high school microeconomics" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..d0a4d02bb7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_high_school_physics" +"task_alias": "hi_in high school physics" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..5daaf7d585 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_social_sciences" +"task": "mmlu_hi_in_high_school_psychology" +"task_alias": "hi_in high school psychology" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..aa79f5fa28 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_high_school_statistics" +"task_alias": "hi_in high school statistics" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..3f8bcccb96 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_humanities" +"task": "mmlu_hi_in_high_school_us_history" +"task_alias": "hi_in high school us history" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..11fd2de572 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_humanities" +"task": "mmlu_hi_in_high_school_world_history" +"task_alias": "hi_in high school world history" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..9738b66aa2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_other" +"task": "mmlu_hi_in_human_aging" +"task_alias": "hi_in human aging" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..80824a3643 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_social_sciences" +"task": "mmlu_hi_in_human_sexuality" +"task_alias": "hi_in human sexuality" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..004f075e05 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_humanities" +"task": "mmlu_hi_in_international_law" +"task_alias": "hi_in international law" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..424674782c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_humanities" +"task": "mmlu_hi_in_jurisprudence" +"task_alias": "hi_in jurisprudence" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..db84a3df3b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_humanities" +"task": "mmlu_hi_in_logical_fallacies" +"task_alias": "hi_in logical fallacies" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..1541f64290 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_stem" +"task": "mmlu_hi_in_machine_learning" +"task_alias": "hi_in machine learning" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_management.yaml new file mode 100644 index 0000000000..ba560b10af --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_other" +"task": "mmlu_hi_in_management" +"task_alias": "hi_in management" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..4a96691b19 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_other" +"task": "mmlu_hi_in_marketing" +"task_alias": "hi_in marketing" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..48015ba213 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_other" +"task": "mmlu_hi_in_medical_genetics" +"task_alias": "hi_in medical genetics" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..e902241331 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_other" +"task": "mmlu_hi_in_miscellaneous" +"task_alias": "hi_in miscellaneous" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..faa8a3fc98 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_humanities" +"task": "mmlu_hi_in_moral_disputes" +"task_alias": "hi_in moral disputes" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..e303c54cbe --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_humanities" +"task": "mmlu_hi_in_moral_scenarios" +"task_alias": "hi_in moral scenarios" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..65ad5cf8e9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_other" +"task": "mmlu_hi_in_nutrition" +"task_alias": "hi_in nutrition" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..145b17e013 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_humanities" +"task": "mmlu_hi_in_philosophy" +"task_alias": "hi_in philosophy" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..f128000a34 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_humanities" +"task": "mmlu_hi_in_prehistory" +"task_alias": "hi_in prehistory" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..37609ed776 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_other" +"task": "mmlu_hi_in_professional_accounting" +"task_alias": "hi_in professional accounting" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..8a1724a9c1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_humanities" +"task": "mmlu_hi_in_professional_law" +"task_alias": "hi_in professional law" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..a29a60695d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_other" +"task": "mmlu_hi_in_professional_medicine" +"task_alias": "hi_in professional medicine" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..9c7ad0077b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_social_sciences" +"task": "mmlu_hi_in_professional_psychology" +"task_alias": "hi_in professional psychology" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..4f3bb0cef3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_social_sciences" +"task": "mmlu_hi_in_public_relations" +"task_alias": "hi_in public relations" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..5140b93264 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_social_sciences" +"task": "mmlu_hi_in_security_studies" +"task_alias": "hi_in security studies" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..3d128589be --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_social_sciences" +"task": "mmlu_hi_in_sociology" +"task_alias": "hi_in sociology" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..288e8d0b42 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_social_sciences" +"task": "mmlu_hi_in_us_foreign_policy" +"task_alias": "hi_in us foreign policy" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..1307130898 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_other" +"task": "mmlu_hi_in_virology" +"task_alias": "hi_in virology" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..d33a6ab8da --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/hi_in/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the Hindi (India) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_hi_in_humanities" +"task": "mmlu_hi_in_world_religions" +"task_alias": "hi_in world religions" +"test_split": "HI_IN" diff --git a/lm_eval/tasks/openai_mmlu/id_id/_openai_mmlu_id_id.yaml b/lm_eval/tasks/openai_mmlu/id_id/_openai_mmlu_id_id.yaml new file mode 100644 index 0000000000..3765cab217 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/_openai_mmlu_id_id.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_id_id +group_alias: id_id +metadata: + version: 1.0.0 +task: +- mmlu_id_id_tasks diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..27df796d42 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_abstract_algebra" +"task_alias": "id_id abstract algebra" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..c11e44be06 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_anatomy" +"task_alias": "id_id anatomy" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..4a7e2728d0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_astronomy" +"task_alias": "id_id astronomy" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..f5ec8c4be1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_other" +"task": "mmlu_id_id_business_ethics" +"task_alias": "id_id business ethics" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..d5c68f356b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_other" +"task": "mmlu_id_id_clinical_knowledge" +"task_alias": "id_id clinical knowledge" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..f411825a1b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_college_biology" +"task_alias": "id_id college biology" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..5996dce3c7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_college_chemistry" +"task_alias": "id_id college chemistry" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..26a0b05d35 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_college_computer_science" +"task_alias": "id_id college computer science" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..4e16441554 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_college_mathematics" +"task_alias": "id_id college mathematics" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..60f4b2ff98 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_other" +"task": "mmlu_id_id_college_medicine" +"task_alias": "id_id college medicine" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..757d393a4c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_college_physics" +"task_alias": "id_id college physics" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..38a6eb652c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_computer_security" +"task_alias": "id_id computer security" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..0ce554db07 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_conceptual_physics" +"task_alias": "id_id conceptual physics" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..0f1b98029f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_social_sciences" +"task": "mmlu_id_id_econometrics" +"task_alias": "id_id econometrics" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..57d970f516 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_electrical_engineering" +"task_alias": "id_id electrical engineering" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..792667b9d8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_elementary_mathematics" +"task_alias": "id_id elementary mathematics" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..f7281a0f7f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_humanities" +"task": "mmlu_id_id_formal_logic" +"task_alias": "id_id formal logic" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..1eb97c2da0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_other" +"task": "mmlu_id_id_global_facts" +"task_alias": "id_id global facts" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..876d384b1b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_high_school_biology" +"task_alias": "id_id high school biology" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..124a97d43e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_high_school_chemistry" +"task_alias": "id_id high school chemistry" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..389c64463c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_high_school_computer_science" +"task_alias": "id_id high school computer science" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..2a0bde337a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_humanities" +"task": "mmlu_id_id_high_school_european_history" +"task_alias": "id_id high school european history" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..38b923bd3c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_social_sciences" +"task": "mmlu_id_id_high_school_geography" +"task_alias": "id_id high school geography" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..35f718e7e7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_social_sciences" +"task": "mmlu_id_id_high_school_government_and_politics" +"task_alias": "id_id high school government and politics" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..ebdfd45de7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_social_sciences" +"task": "mmlu_id_id_high_school_macroeconomics" +"task_alias": "id_id high school macroeconomics" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..ab4cd41667 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_high_school_mathematics" +"task_alias": "id_id high school mathematics" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..76bd920b6e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_social_sciences" +"task": "mmlu_id_id_high_school_microeconomics" +"task_alias": "id_id high school microeconomics" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..d33d9b5037 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_high_school_physics" +"task_alias": "id_id high school physics" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..d93f718edf --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_social_sciences" +"task": "mmlu_id_id_high_school_psychology" +"task_alias": "id_id high school psychology" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..c14f5c3fec --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_high_school_statistics" +"task_alias": "id_id high school statistics" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..78a06e8470 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_humanities" +"task": "mmlu_id_id_high_school_us_history" +"task_alias": "id_id high school us history" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..b36d36cf1c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_humanities" +"task": "mmlu_id_id_high_school_world_history" +"task_alias": "id_id high school world history" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..5a56d50a3b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_other" +"task": "mmlu_id_id_human_aging" +"task_alias": "id_id human aging" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..16073926c7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_social_sciences" +"task": "mmlu_id_id_human_sexuality" +"task_alias": "id_id human sexuality" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..6d5652b79c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_humanities" +"task": "mmlu_id_id_international_law" +"task_alias": "id_id international law" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..f98a86d3c2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_humanities" +"task": "mmlu_id_id_jurisprudence" +"task_alias": "id_id jurisprudence" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..d9062a9f46 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_humanities" +"task": "mmlu_id_id_logical_fallacies" +"task_alias": "id_id logical fallacies" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..594cb25c2e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_stem" +"task": "mmlu_id_id_machine_learning" +"task_alias": "id_id machine learning" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_management.yaml new file mode 100644 index 0000000000..2c3e01ec47 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_other" +"task": "mmlu_id_id_management" +"task_alias": "id_id management" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..85703c6baa --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_other" +"task": "mmlu_id_id_marketing" +"task_alias": "id_id marketing" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..98e085382e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_other" +"task": "mmlu_id_id_medical_genetics" +"task_alias": "id_id medical genetics" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..44d1c100d0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_other" +"task": "mmlu_id_id_miscellaneous" +"task_alias": "id_id miscellaneous" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..5f25159aec --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_humanities" +"task": "mmlu_id_id_moral_disputes" +"task_alias": "id_id moral disputes" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..74cfc96dff --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_humanities" +"task": "mmlu_id_id_moral_scenarios" +"task_alias": "id_id moral scenarios" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..59e5a13466 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_other" +"task": "mmlu_id_id_nutrition" +"task_alias": "id_id nutrition" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..a2e7cea67a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_humanities" +"task": "mmlu_id_id_philosophy" +"task_alias": "id_id philosophy" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..691c2b18d0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_humanities" +"task": "mmlu_id_id_prehistory" +"task_alias": "id_id prehistory" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..a965a2fef7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_other" +"task": "mmlu_id_id_professional_accounting" +"task_alias": "id_id professional accounting" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..3b7faffe1c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_humanities" +"task": "mmlu_id_id_professional_law" +"task_alias": "id_id professional law" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..f216f7dafd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_other" +"task": "mmlu_id_id_professional_medicine" +"task_alias": "id_id professional medicine" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..f65b310ada --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_social_sciences" +"task": "mmlu_id_id_professional_psychology" +"task_alias": "id_id professional psychology" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..adf66e43aa --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_social_sciences" +"task": "mmlu_id_id_public_relations" +"task_alias": "id_id public relations" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..75418e6a3d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_social_sciences" +"task": "mmlu_id_id_security_studies" +"task_alias": "id_id security studies" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..3bdd48d486 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_social_sciences" +"task": "mmlu_id_id_sociology" +"task_alias": "id_id sociology" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..6653ad1c1d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_social_sciences" +"task": "mmlu_id_id_us_foreign_policy" +"task_alias": "id_id us foreign policy" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..d02cad8a7c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_other" +"task": "mmlu_id_id_virology" +"task_alias": "id_id virology" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..f9f4320b4d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/id_id/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the Indonesian (Indonesia) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_id_id_humanities" +"task": "mmlu_id_id_world_religions" +"task_alias": "id_id world religions" +"test_split": "ID_ID" diff --git a/lm_eval/tasks/openai_mmlu/it_it/_openai_mmlu_it_it.yaml b/lm_eval/tasks/openai_mmlu/it_it/_openai_mmlu_it_it.yaml new file mode 100644 index 0000000000..a09a4553cc --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/_openai_mmlu_it_it.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_it_it +group_alias: it_it +metadata: + version: 1.0.0 +task: +- mmlu_it_it_tasks diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..33fd81b200 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_abstract_algebra" +"task_alias": "it_it abstract algebra" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..ddec81e219 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_anatomy" +"task_alias": "it_it anatomy" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..dfa3db24d2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_astronomy" +"task_alias": "it_it astronomy" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..f7b3ba2071 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_other" +"task": "mmlu_it_it_business_ethics" +"task_alias": "it_it business ethics" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..49a500d18c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_other" +"task": "mmlu_it_it_clinical_knowledge" +"task_alias": "it_it clinical knowledge" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..8da82519d5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_college_biology" +"task_alias": "it_it college biology" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..50699b15d7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_college_chemistry" +"task_alias": "it_it college chemistry" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..d0b529d59c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_college_computer_science" +"task_alias": "it_it college computer science" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..16582c229d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_college_mathematics" +"task_alias": "it_it college mathematics" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..894aa63acd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_other" +"task": "mmlu_it_it_college_medicine" +"task_alias": "it_it college medicine" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..fec253adce --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_college_physics" +"task_alias": "it_it college physics" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..e9191d6794 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_computer_security" +"task_alias": "it_it computer security" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..61992b26da --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_conceptual_physics" +"task_alias": "it_it conceptual physics" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..e9edb1253a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_social_sciences" +"task": "mmlu_it_it_econometrics" +"task_alias": "it_it econometrics" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..6b36b89ea4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_electrical_engineering" +"task_alias": "it_it electrical engineering" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..d8fad67d37 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_elementary_mathematics" +"task_alias": "it_it elementary mathematics" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..c1a1edb3ef --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_humanities" +"task": "mmlu_it_it_formal_logic" +"task_alias": "it_it formal logic" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..aa04009c7f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_other" +"task": "mmlu_it_it_global_facts" +"task_alias": "it_it global facts" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..a8ff5d6535 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_high_school_biology" +"task_alias": "it_it high school biology" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..2776bb5686 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_high_school_chemistry" +"task_alias": "it_it high school chemistry" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..67f9423f38 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_high_school_computer_science" +"task_alias": "it_it high school computer science" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..f448b2eb70 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_humanities" +"task": "mmlu_it_it_high_school_european_history" +"task_alias": "it_it high school european history" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..850447bc5b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_social_sciences" +"task": "mmlu_it_it_high_school_geography" +"task_alias": "it_it high school geography" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..4613f5126a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_social_sciences" +"task": "mmlu_it_it_high_school_government_and_politics" +"task_alias": "it_it high school government and politics" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..bc0f79b2fe --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_social_sciences" +"task": "mmlu_it_it_high_school_macroeconomics" +"task_alias": "it_it high school macroeconomics" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..f89e37cb80 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_high_school_mathematics" +"task_alias": "it_it high school mathematics" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..ad5d6907e5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_social_sciences" +"task": "mmlu_it_it_high_school_microeconomics" +"task_alias": "it_it high school microeconomics" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..6dd091a3be --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_high_school_physics" +"task_alias": "it_it high school physics" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..03df0f1e9c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_social_sciences" +"task": "mmlu_it_it_high_school_psychology" +"task_alias": "it_it high school psychology" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..f05e648eb4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_high_school_statistics" +"task_alias": "it_it high school statistics" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..3a940880da --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_humanities" +"task": "mmlu_it_it_high_school_us_history" +"task_alias": "it_it high school us history" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..2c0846b899 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_humanities" +"task": "mmlu_it_it_high_school_world_history" +"task_alias": "it_it high school world history" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..5e8c7b6233 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_other" +"task": "mmlu_it_it_human_aging" +"task_alias": "it_it human aging" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..98b3751711 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_social_sciences" +"task": "mmlu_it_it_human_sexuality" +"task_alias": "it_it human sexuality" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..b55fe02160 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_humanities" +"task": "mmlu_it_it_international_law" +"task_alias": "it_it international law" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..1a45cf93a6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_humanities" +"task": "mmlu_it_it_jurisprudence" +"task_alias": "it_it jurisprudence" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..a4abce23a5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_humanities" +"task": "mmlu_it_it_logical_fallacies" +"task_alias": "it_it logical fallacies" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..642d2171f6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_stem" +"task": "mmlu_it_it_machine_learning" +"task_alias": "it_it machine learning" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_management.yaml new file mode 100644 index 0000000000..34454a1fa0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_other" +"task": "mmlu_it_it_management" +"task_alias": "it_it management" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..4b6f793a7d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_other" +"task": "mmlu_it_it_marketing" +"task_alias": "it_it marketing" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..b6fd27d51b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_other" +"task": "mmlu_it_it_medical_genetics" +"task_alias": "it_it medical genetics" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..7ba397bd03 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_other" +"task": "mmlu_it_it_miscellaneous" +"task_alias": "it_it miscellaneous" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..5137737089 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_humanities" +"task": "mmlu_it_it_moral_disputes" +"task_alias": "it_it moral disputes" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..f67e8e4943 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_humanities" +"task": "mmlu_it_it_moral_scenarios" +"task_alias": "it_it moral scenarios" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..881fb43d34 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_other" +"task": "mmlu_it_it_nutrition" +"task_alias": "it_it nutrition" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..7ec0340235 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_humanities" +"task": "mmlu_it_it_philosophy" +"task_alias": "it_it philosophy" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..753faa37bc --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_humanities" +"task": "mmlu_it_it_prehistory" +"task_alias": "it_it prehistory" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..017efb42e9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_other" +"task": "mmlu_it_it_professional_accounting" +"task_alias": "it_it professional accounting" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..c83a1c4258 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_humanities" +"task": "mmlu_it_it_professional_law" +"task_alias": "it_it professional law" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..16986bfa68 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_other" +"task": "mmlu_it_it_professional_medicine" +"task_alias": "it_it professional medicine" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..2a2403d2ca --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_social_sciences" +"task": "mmlu_it_it_professional_psychology" +"task_alias": "it_it professional psychology" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..8c279ed8d5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_social_sciences" +"task": "mmlu_it_it_public_relations" +"task_alias": "it_it public relations" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..6dc46ce0fc --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_social_sciences" +"task": "mmlu_it_it_security_studies" +"task_alias": "it_it security studies" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..ac0626aa9c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_social_sciences" +"task": "mmlu_it_it_sociology" +"task_alias": "it_it sociology" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..70f857db5b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_social_sciences" +"task": "mmlu_it_it_us_foreign_policy" +"task_alias": "it_it us foreign policy" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..ed15dcab9b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_other" +"task": "mmlu_it_it_virology" +"task_alias": "it_it virology" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..61d9118502 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/it_it/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the Italian (Italy) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_it_it_humanities" +"task": "mmlu_it_it_world_religions" +"task_alias": "it_it world religions" +"test_split": "IT_IT" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/_openai_mmlu_ja_jp.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/_openai_mmlu_ja_jp.yaml new file mode 100644 index 0000000000..eff70c5b0a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/_openai_mmlu_ja_jp.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_ja_jp +group_alias: ja_jp +metadata: + version: 1.0.0 +task: +- mmlu_ja_jp_tasks diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..7c4416a451 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_abstract_algebra" +"task_alias": "ja_jp abstract algebra" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..2e87d9acf0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_anatomy" +"task_alias": "ja_jp anatomy" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..6937839ffd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_astronomy" +"task_alias": "ja_jp astronomy" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..2635eebce3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_other" +"task": "mmlu_ja_jp_business_ethics" +"task_alias": "ja_jp business ethics" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..f610886f31 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_other" +"task": "mmlu_ja_jp_clinical_knowledge" +"task_alias": "ja_jp clinical knowledge" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..44fa434ccb --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_college_biology" +"task_alias": "ja_jp college biology" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..f9e76f6496 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_college_chemistry" +"task_alias": "ja_jp college chemistry" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..04990a2c14 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_college_computer_science" +"task_alias": "ja_jp college computer science" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..60ce368a27 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_college_mathematics" +"task_alias": "ja_jp college mathematics" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..6256a74f6c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_other" +"task": "mmlu_ja_jp_college_medicine" +"task_alias": "ja_jp college medicine" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..4277f06ea1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_college_physics" +"task_alias": "ja_jp college physics" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..7e720d6a3f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_computer_security" +"task_alias": "ja_jp computer security" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..1e66a29d8b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_conceptual_physics" +"task_alias": "ja_jp conceptual physics" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..4b7aa1413e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_social_sciences" +"task": "mmlu_ja_jp_econometrics" +"task_alias": "ja_jp econometrics" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..e574f9eed0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_electrical_engineering" +"task_alias": "ja_jp electrical engineering" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..e0b14ff41e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_elementary_mathematics" +"task_alias": "ja_jp elementary mathematics" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..9be17b8a2e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_humanities" +"task": "mmlu_ja_jp_formal_logic" +"task_alias": "ja_jp formal logic" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..06f2511340 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_other" +"task": "mmlu_ja_jp_global_facts" +"task_alias": "ja_jp global facts" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..c97c8f8d60 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_high_school_biology" +"task_alias": "ja_jp high school biology" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..0330fd38de --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_high_school_chemistry" +"task_alias": "ja_jp high school chemistry" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..4fd6f6cee0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_high_school_computer_science" +"task_alias": "ja_jp high school computer science" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..29c900c6c6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_humanities" +"task": "mmlu_ja_jp_high_school_european_history" +"task_alias": "ja_jp high school european history" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..267b75745c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_social_sciences" +"task": "mmlu_ja_jp_high_school_geography" +"task_alias": "ja_jp high school geography" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..5b59998cd9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_social_sciences" +"task": "mmlu_ja_jp_high_school_government_and_politics" +"task_alias": "ja_jp high school government and politics" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..e3b15e1f66 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_social_sciences" +"task": "mmlu_ja_jp_high_school_macroeconomics" +"task_alias": "ja_jp high school macroeconomics" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..5efdd62cac --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_high_school_mathematics" +"task_alias": "ja_jp high school mathematics" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..cc29db199f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_social_sciences" +"task": "mmlu_ja_jp_high_school_microeconomics" +"task_alias": "ja_jp high school microeconomics" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..730e68309d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_high_school_physics" +"task_alias": "ja_jp high school physics" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..49f9b2e144 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_social_sciences" +"task": "mmlu_ja_jp_high_school_psychology" +"task_alias": "ja_jp high school psychology" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..ef4bf9f51e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_high_school_statistics" +"task_alias": "ja_jp high school statistics" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..a8dd2deb48 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_humanities" +"task": "mmlu_ja_jp_high_school_us_history" +"task_alias": "ja_jp high school us history" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..bf3c4e8b1c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_humanities" +"task": "mmlu_ja_jp_high_school_world_history" +"task_alias": "ja_jp high school world history" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..55b1527d6c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_other" +"task": "mmlu_ja_jp_human_aging" +"task_alias": "ja_jp human aging" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..bd181387ab --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_social_sciences" +"task": "mmlu_ja_jp_human_sexuality" +"task_alias": "ja_jp human sexuality" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..e6eb9fcd36 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_humanities" +"task": "mmlu_ja_jp_international_law" +"task_alias": "ja_jp international law" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..647450b418 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_humanities" +"task": "mmlu_ja_jp_jurisprudence" +"task_alias": "ja_jp jurisprudence" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..689ffb25b1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_humanities" +"task": "mmlu_ja_jp_logical_fallacies" +"task_alias": "ja_jp logical fallacies" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..6d74556298 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_stem" +"task": "mmlu_ja_jp_machine_learning" +"task_alias": "ja_jp machine learning" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_management.yaml new file mode 100644 index 0000000000..150c1cc70d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_other" +"task": "mmlu_ja_jp_management" +"task_alias": "ja_jp management" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..f553a1481f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_other" +"task": "mmlu_ja_jp_marketing" +"task_alias": "ja_jp marketing" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..c44e3c8b2b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_other" +"task": "mmlu_ja_jp_medical_genetics" +"task_alias": "ja_jp medical genetics" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..b1293acf3e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_other" +"task": "mmlu_ja_jp_miscellaneous" +"task_alias": "ja_jp miscellaneous" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..939a315546 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_humanities" +"task": "mmlu_ja_jp_moral_disputes" +"task_alias": "ja_jp moral disputes" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..069eba8927 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_humanities" +"task": "mmlu_ja_jp_moral_scenarios" +"task_alias": "ja_jp moral scenarios" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..2a17983f96 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_other" +"task": "mmlu_ja_jp_nutrition" +"task_alias": "ja_jp nutrition" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..b4aec0d898 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_humanities" +"task": "mmlu_ja_jp_philosophy" +"task_alias": "ja_jp philosophy" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..b8ad3b3488 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_humanities" +"task": "mmlu_ja_jp_prehistory" +"task_alias": "ja_jp prehistory" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..1803b8c1c3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_other" +"task": "mmlu_ja_jp_professional_accounting" +"task_alias": "ja_jp professional accounting" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..fe4086cecd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_humanities" +"task": "mmlu_ja_jp_professional_law" +"task_alias": "ja_jp professional law" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..03f10e3e1f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_other" +"task": "mmlu_ja_jp_professional_medicine" +"task_alias": "ja_jp professional medicine" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..f9eded0475 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_social_sciences" +"task": "mmlu_ja_jp_professional_psychology" +"task_alias": "ja_jp professional psychology" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..9a14fec10c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_social_sciences" +"task": "mmlu_ja_jp_public_relations" +"task_alias": "ja_jp public relations" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..923a12b7b4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_social_sciences" +"task": "mmlu_ja_jp_security_studies" +"task_alias": "ja_jp security studies" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..f609b41cdd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_social_sciences" +"task": "mmlu_ja_jp_sociology" +"task_alias": "ja_jp sociology" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..fed04f8f1f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_social_sciences" +"task": "mmlu_ja_jp_us_foreign_policy" +"task_alias": "ja_jp us foreign policy" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..74ed4de5b0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_other" +"task": "mmlu_ja_jp_virology" +"task_alias": "ja_jp virology" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..db04310102 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ja_jp/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the Japanese (Japan) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ja_jp_humanities" +"task": "mmlu_ja_jp_world_religions" +"task_alias": "ja_jp world religions" +"test_split": "JA_JP" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/_openai_mmlu_ko_kr.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/_openai_mmlu_ko_kr.yaml new file mode 100644 index 0000000000..4bcd6b2cdc --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/_openai_mmlu_ko_kr.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_ko_kr +group_alias: ko_kr +metadata: + version: 1.0.0 +task: +- mmlu_ko_kr_tasks diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..19daf86735 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_abstract_algebra" +"task_alias": "ko_kr abstract algebra" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..bf543671f5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_anatomy" +"task_alias": "ko_kr anatomy" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..df59068775 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_astronomy" +"task_alias": "ko_kr astronomy" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..f184b18494 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_other" +"task": "mmlu_ko_kr_business_ethics" +"task_alias": "ko_kr business ethics" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..0d8d0982ad --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_other" +"task": "mmlu_ko_kr_clinical_knowledge" +"task_alias": "ko_kr clinical knowledge" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..57ea02b209 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_college_biology" +"task_alias": "ko_kr college biology" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..cbbace16dd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_college_chemistry" +"task_alias": "ko_kr college chemistry" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..f0bfa69031 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_college_computer_science" +"task_alias": "ko_kr college computer science" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..fc2f721828 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_college_mathematics" +"task_alias": "ko_kr college mathematics" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..8309831619 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_other" +"task": "mmlu_ko_kr_college_medicine" +"task_alias": "ko_kr college medicine" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..73acd8f8ed --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_college_physics" +"task_alias": "ko_kr college physics" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..ecd5626188 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_computer_security" +"task_alias": "ko_kr computer security" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..4b3cffbb4e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_conceptual_physics" +"task_alias": "ko_kr conceptual physics" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..ffb763c5e0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_social_sciences" +"task": "mmlu_ko_kr_econometrics" +"task_alias": "ko_kr econometrics" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..f2bf7eb447 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_electrical_engineering" +"task_alias": "ko_kr electrical engineering" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..53041882ce --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_elementary_mathematics" +"task_alias": "ko_kr elementary mathematics" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..9c3676e2f2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_humanities" +"task": "mmlu_ko_kr_formal_logic" +"task_alias": "ko_kr formal logic" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..f2a5f99e80 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_other" +"task": "mmlu_ko_kr_global_facts" +"task_alias": "ko_kr global facts" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..1d84c5289a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_high_school_biology" +"task_alias": "ko_kr high school biology" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..4534660e62 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_high_school_chemistry" +"task_alias": "ko_kr high school chemistry" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..16d0d4bcf8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_high_school_computer_science" +"task_alias": "ko_kr high school computer science" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..8cd016fa8d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_humanities" +"task": "mmlu_ko_kr_high_school_european_history" +"task_alias": "ko_kr high school european history" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..f489f5beb9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_social_sciences" +"task": "mmlu_ko_kr_high_school_geography" +"task_alias": "ko_kr high school geography" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..627c78b189 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_social_sciences" +"task": "mmlu_ko_kr_high_school_government_and_politics" +"task_alias": "ko_kr high school government and politics" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..6b499a1150 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_social_sciences" +"task": "mmlu_ko_kr_high_school_macroeconomics" +"task_alias": "ko_kr high school macroeconomics" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..115003f44b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_high_school_mathematics" +"task_alias": "ko_kr high school mathematics" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..ffdf7c9111 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_social_sciences" +"task": "mmlu_ko_kr_high_school_microeconomics" +"task_alias": "ko_kr high school microeconomics" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..bf679dc80c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_high_school_physics" +"task_alias": "ko_kr high school physics" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..276d9a35a0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_social_sciences" +"task": "mmlu_ko_kr_high_school_psychology" +"task_alias": "ko_kr high school psychology" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..c0c7c42b39 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_high_school_statistics" +"task_alias": "ko_kr high school statistics" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..f6f1c7fd5d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_humanities" +"task": "mmlu_ko_kr_high_school_us_history" +"task_alias": "ko_kr high school us history" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..66b93e320c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_humanities" +"task": "mmlu_ko_kr_high_school_world_history" +"task_alias": "ko_kr high school world history" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..7c32c73ae3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_other" +"task": "mmlu_ko_kr_human_aging" +"task_alias": "ko_kr human aging" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..779bec4cf7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_social_sciences" +"task": "mmlu_ko_kr_human_sexuality" +"task_alias": "ko_kr human sexuality" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..ba0873be17 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_humanities" +"task": "mmlu_ko_kr_international_law" +"task_alias": "ko_kr international law" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..698e432dae --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_humanities" +"task": "mmlu_ko_kr_jurisprudence" +"task_alias": "ko_kr jurisprudence" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..b121e55c63 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_humanities" +"task": "mmlu_ko_kr_logical_fallacies" +"task_alias": "ko_kr logical fallacies" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..6a787de940 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_stem" +"task": "mmlu_ko_kr_machine_learning" +"task_alias": "ko_kr machine learning" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_management.yaml new file mode 100644 index 0000000000..9f45dc67bb --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_other" +"task": "mmlu_ko_kr_management" +"task_alias": "ko_kr management" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..8d7c27fa0d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_other" +"task": "mmlu_ko_kr_marketing" +"task_alias": "ko_kr marketing" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..2251dc52b4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_other" +"task": "mmlu_ko_kr_medical_genetics" +"task_alias": "ko_kr medical genetics" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..f8f38c2022 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_other" +"task": "mmlu_ko_kr_miscellaneous" +"task_alias": "ko_kr miscellaneous" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..eaeabc4b53 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_humanities" +"task": "mmlu_ko_kr_moral_disputes" +"task_alias": "ko_kr moral disputes" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..9dfedf24ff --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_humanities" +"task": "mmlu_ko_kr_moral_scenarios" +"task_alias": "ko_kr moral scenarios" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..b75b3af058 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_other" +"task": "mmlu_ko_kr_nutrition" +"task_alias": "ko_kr nutrition" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..df40effbdc --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_humanities" +"task": "mmlu_ko_kr_philosophy" +"task_alias": "ko_kr philosophy" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..49fccd27f5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_humanities" +"task": "mmlu_ko_kr_prehistory" +"task_alias": "ko_kr prehistory" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..dfe40b5c59 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_other" +"task": "mmlu_ko_kr_professional_accounting" +"task_alias": "ko_kr professional accounting" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..33ac736382 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_humanities" +"task": "mmlu_ko_kr_professional_law" +"task_alias": "ko_kr professional law" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..b7d1b5f622 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_other" +"task": "mmlu_ko_kr_professional_medicine" +"task_alias": "ko_kr professional medicine" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..211d400e4f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_social_sciences" +"task": "mmlu_ko_kr_professional_psychology" +"task_alias": "ko_kr professional psychology" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..37c907515a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_social_sciences" +"task": "mmlu_ko_kr_public_relations" +"task_alias": "ko_kr public relations" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..3662341f10 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_social_sciences" +"task": "mmlu_ko_kr_security_studies" +"task_alias": "ko_kr security studies" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..5bd119e69f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_social_sciences" +"task": "mmlu_ko_kr_sociology" +"task_alias": "ko_kr sociology" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..06a68d07e3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_social_sciences" +"task": "mmlu_ko_kr_us_foreign_policy" +"task_alias": "ko_kr us foreign policy" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..606e6e087d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_other" +"task": "mmlu_ko_kr_virology" +"task_alias": "ko_kr virology" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..7b0cd0aff4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/ko_kr/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the Korean (South Korea) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_ko_kr_humanities" +"task": "mmlu_ko_kr_world_religions" +"task_alias": "ko_kr world religions" +"test_split": "KO_KR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/_openai_mmlu_pt_br.yaml b/lm_eval/tasks/openai_mmlu/pt_br/_openai_mmlu_pt_br.yaml new file mode 100644 index 0000000000..2f22eb4b0d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/_openai_mmlu_pt_br.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_pt_br +group_alias: pt_br +metadata: + version: 1.0.0 +task: +- mmlu_pt_br_tasks diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..6b701520a5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_abstract_algebra" +"task_alias": "pt_br abstract algebra" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..8d93ce831f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_anatomy" +"task_alias": "pt_br anatomy" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..96b21cc35a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_astronomy" +"task_alias": "pt_br astronomy" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..4d12f06713 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_other" +"task": "mmlu_pt_br_business_ethics" +"task_alias": "pt_br business ethics" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..55621f5681 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_other" +"task": "mmlu_pt_br_clinical_knowledge" +"task_alias": "pt_br clinical knowledge" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..397b362ba4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_college_biology" +"task_alias": "pt_br college biology" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..39b444d778 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_college_chemistry" +"task_alias": "pt_br college chemistry" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..bdb43dd04f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_college_computer_science" +"task_alias": "pt_br college computer science" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..57953ebb43 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_college_mathematics" +"task_alias": "pt_br college mathematics" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..b15466154d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_other" +"task": "mmlu_pt_br_college_medicine" +"task_alias": "pt_br college medicine" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..ecf0fcc625 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_college_physics" +"task_alias": "pt_br college physics" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..e3ea906171 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_computer_security" +"task_alias": "pt_br computer security" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..3a1e2f976c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_conceptual_physics" +"task_alias": "pt_br conceptual physics" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..027244d5ea --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_social_sciences" +"task": "mmlu_pt_br_econometrics" +"task_alias": "pt_br econometrics" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..4ec0a7c89c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_electrical_engineering" +"task_alias": "pt_br electrical engineering" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..6d2abd8073 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_elementary_mathematics" +"task_alias": "pt_br elementary mathematics" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..cc4bdc539d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_humanities" +"task": "mmlu_pt_br_formal_logic" +"task_alias": "pt_br formal logic" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..3befa821ec --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_other" +"task": "mmlu_pt_br_global_facts" +"task_alias": "pt_br global facts" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..f8e5b4e51c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_high_school_biology" +"task_alias": "pt_br high school biology" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..ebce9f581c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_high_school_chemistry" +"task_alias": "pt_br high school chemistry" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..4db3d69a05 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_high_school_computer_science" +"task_alias": "pt_br high school computer science" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..285c7685f4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_humanities" +"task": "mmlu_pt_br_high_school_european_history" +"task_alias": "pt_br high school european history" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..46b0b9cc1c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_social_sciences" +"task": "mmlu_pt_br_high_school_geography" +"task_alias": "pt_br high school geography" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..ddfa9f7b13 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_social_sciences" +"task": "mmlu_pt_br_high_school_government_and_politics" +"task_alias": "pt_br high school government and politics" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..2f5357d096 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_social_sciences" +"task": "mmlu_pt_br_high_school_macroeconomics" +"task_alias": "pt_br high school macroeconomics" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..72b1945a69 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_high_school_mathematics" +"task_alias": "pt_br high school mathematics" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..aab2c0af6f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_social_sciences" +"task": "mmlu_pt_br_high_school_microeconomics" +"task_alias": "pt_br high school microeconomics" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..75733a4300 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_high_school_physics" +"task_alias": "pt_br high school physics" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..e3e50ed0f8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_social_sciences" +"task": "mmlu_pt_br_high_school_psychology" +"task_alias": "pt_br high school psychology" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..9bcce518a6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_high_school_statistics" +"task_alias": "pt_br high school statistics" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..bccbeb3684 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_humanities" +"task": "mmlu_pt_br_high_school_us_history" +"task_alias": "pt_br high school us history" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..715df9166a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_humanities" +"task": "mmlu_pt_br_high_school_world_history" +"task_alias": "pt_br high school world history" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..c674b81116 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_other" +"task": "mmlu_pt_br_human_aging" +"task_alias": "pt_br human aging" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..ef2bb0d3ed --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_social_sciences" +"task": "mmlu_pt_br_human_sexuality" +"task_alias": "pt_br human sexuality" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..296000c33a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_humanities" +"task": "mmlu_pt_br_international_law" +"task_alias": "pt_br international law" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..0c5a044fbf --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_humanities" +"task": "mmlu_pt_br_jurisprudence" +"task_alias": "pt_br jurisprudence" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..160ddc3d3a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_humanities" +"task": "mmlu_pt_br_logical_fallacies" +"task_alias": "pt_br logical fallacies" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..c9d8573a8f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_stem" +"task": "mmlu_pt_br_machine_learning" +"task_alias": "pt_br machine learning" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_management.yaml new file mode 100644 index 0000000000..0fd436c28d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_other" +"task": "mmlu_pt_br_management" +"task_alias": "pt_br management" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..c38a5428ec --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_other" +"task": "mmlu_pt_br_marketing" +"task_alias": "pt_br marketing" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..c54729ab79 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_other" +"task": "mmlu_pt_br_medical_genetics" +"task_alias": "pt_br medical genetics" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..4be7c20ef2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_other" +"task": "mmlu_pt_br_miscellaneous" +"task_alias": "pt_br miscellaneous" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..6ccec19de3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_humanities" +"task": "mmlu_pt_br_moral_disputes" +"task_alias": "pt_br moral disputes" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..2c73c384dd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_humanities" +"task": "mmlu_pt_br_moral_scenarios" +"task_alias": "pt_br moral scenarios" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..f912d6e1a2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_other" +"task": "mmlu_pt_br_nutrition" +"task_alias": "pt_br nutrition" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..b4a4596d81 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_humanities" +"task": "mmlu_pt_br_philosophy" +"task_alias": "pt_br philosophy" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..00edd78313 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_humanities" +"task": "mmlu_pt_br_prehistory" +"task_alias": "pt_br prehistory" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..f30c5ae877 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_other" +"task": "mmlu_pt_br_professional_accounting" +"task_alias": "pt_br professional accounting" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..350ad3491a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_humanities" +"task": "mmlu_pt_br_professional_law" +"task_alias": "pt_br professional law" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..86b3c171a7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_other" +"task": "mmlu_pt_br_professional_medicine" +"task_alias": "pt_br professional medicine" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..6d20d51af8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_social_sciences" +"task": "mmlu_pt_br_professional_psychology" +"task_alias": "pt_br professional psychology" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..606e4b556d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_social_sciences" +"task": "mmlu_pt_br_public_relations" +"task_alias": "pt_br public relations" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..9c16841938 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_social_sciences" +"task": "mmlu_pt_br_security_studies" +"task_alias": "pt_br security studies" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..b1edfba056 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_social_sciences" +"task": "mmlu_pt_br_sociology" +"task_alias": "pt_br sociology" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..a270de615c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_social_sciences" +"task": "mmlu_pt_br_us_foreign_policy" +"task_alias": "pt_br us foreign policy" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..9254fd164c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_other" +"task": "mmlu_pt_br_virology" +"task_alias": "pt_br virology" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..c47b568299 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/pt_br/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the Portuguese (Brazil) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_pt_br_humanities" +"task": "mmlu_pt_br_world_religions" +"task_alias": "pt_br world religions" +"test_split": "PT_BR" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/_openai_mmlu_sw_ke.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/_openai_mmlu_sw_ke.yaml new file mode 100644 index 0000000000..5cfb043e82 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/_openai_mmlu_sw_ke.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_sw_ke +group_alias: sw_ke +metadata: + version: 1.0.0 +task: +- mmlu_sw_ke_tasks diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..4c603463b8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_abstract_algebra" +"task_alias": "sw_ke abstract algebra" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..8bd2c28a47 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_anatomy" +"task_alias": "sw_ke anatomy" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..42ef2c0dd2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_astronomy" +"task_alias": "sw_ke astronomy" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..e4c9c2df38 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_other" +"task": "mmlu_sw_ke_business_ethics" +"task_alias": "sw_ke business ethics" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..7687d315c6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_other" +"task": "mmlu_sw_ke_clinical_knowledge" +"task_alias": "sw_ke clinical knowledge" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..97906e3659 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_college_biology" +"task_alias": "sw_ke college biology" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..01ef507403 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_college_chemistry" +"task_alias": "sw_ke college chemistry" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..79087013e7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_college_computer_science" +"task_alias": "sw_ke college computer science" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..dae241bbef --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_college_mathematics" +"task_alias": "sw_ke college mathematics" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..8f76c482d8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_other" +"task": "mmlu_sw_ke_college_medicine" +"task_alias": "sw_ke college medicine" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..a7167161d1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_college_physics" +"task_alias": "sw_ke college physics" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..9d913a284a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_computer_security" +"task_alias": "sw_ke computer security" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..2a036e937e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_conceptual_physics" +"task_alias": "sw_ke conceptual physics" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..9a844cc36f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_social_sciences" +"task": "mmlu_sw_ke_econometrics" +"task_alias": "sw_ke econometrics" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..fb2579f974 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_electrical_engineering" +"task_alias": "sw_ke electrical engineering" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..c67fa36f2c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_elementary_mathematics" +"task_alias": "sw_ke elementary mathematics" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..f0eb8b8cd1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_humanities" +"task": "mmlu_sw_ke_formal_logic" +"task_alias": "sw_ke formal logic" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..e5bb7c8c2b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_other" +"task": "mmlu_sw_ke_global_facts" +"task_alias": "sw_ke global facts" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..786e4762c1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_high_school_biology" +"task_alias": "sw_ke high school biology" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..8302a939bb --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_high_school_chemistry" +"task_alias": "sw_ke high school chemistry" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..c93d1b586b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_high_school_computer_science" +"task_alias": "sw_ke high school computer science" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..baaae1dc5c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_humanities" +"task": "mmlu_sw_ke_high_school_european_history" +"task_alias": "sw_ke high school european history" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..8e4e1496a6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_social_sciences" +"task": "mmlu_sw_ke_high_school_geography" +"task_alias": "sw_ke high school geography" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..f7f425ff1a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_social_sciences" +"task": "mmlu_sw_ke_high_school_government_and_politics" +"task_alias": "sw_ke high school government and politics" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..10ed732957 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_social_sciences" +"task": "mmlu_sw_ke_high_school_macroeconomics" +"task_alias": "sw_ke high school macroeconomics" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..aeb53c0331 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_high_school_mathematics" +"task_alias": "sw_ke high school mathematics" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..d2df728b5b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_social_sciences" +"task": "mmlu_sw_ke_high_school_microeconomics" +"task_alias": "sw_ke high school microeconomics" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..b63d4bfd6a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_high_school_physics" +"task_alias": "sw_ke high school physics" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..def478161b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_social_sciences" +"task": "mmlu_sw_ke_high_school_psychology" +"task_alias": "sw_ke high school psychology" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..832300bd16 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_high_school_statistics" +"task_alias": "sw_ke high school statistics" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..e72584c61b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_humanities" +"task": "mmlu_sw_ke_high_school_us_history" +"task_alias": "sw_ke high school us history" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..8ef010eeaf --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_humanities" +"task": "mmlu_sw_ke_high_school_world_history" +"task_alias": "sw_ke high school world history" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..fe1d7e399b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_other" +"task": "mmlu_sw_ke_human_aging" +"task_alias": "sw_ke human aging" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..dcffa093b9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_social_sciences" +"task": "mmlu_sw_ke_human_sexuality" +"task_alias": "sw_ke human sexuality" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..75f7ed9caa --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_humanities" +"task": "mmlu_sw_ke_international_law" +"task_alias": "sw_ke international law" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..15e581eb1d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_humanities" +"task": "mmlu_sw_ke_jurisprudence" +"task_alias": "sw_ke jurisprudence" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..192ff0fcf6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_humanities" +"task": "mmlu_sw_ke_logical_fallacies" +"task_alias": "sw_ke logical fallacies" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..4c56f7381e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_stem" +"task": "mmlu_sw_ke_machine_learning" +"task_alias": "sw_ke machine learning" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_management.yaml new file mode 100644 index 0000000000..21c50227ee --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_other" +"task": "mmlu_sw_ke_management" +"task_alias": "sw_ke management" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..a6d7d6b22b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_other" +"task": "mmlu_sw_ke_marketing" +"task_alias": "sw_ke marketing" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..e2fd68d125 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_other" +"task": "mmlu_sw_ke_medical_genetics" +"task_alias": "sw_ke medical genetics" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..cd632eb84f --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_other" +"task": "mmlu_sw_ke_miscellaneous" +"task_alias": "sw_ke miscellaneous" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..56322b5ca3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_humanities" +"task": "mmlu_sw_ke_moral_disputes" +"task_alias": "sw_ke moral disputes" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..c0ad600db4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_humanities" +"task": "mmlu_sw_ke_moral_scenarios" +"task_alias": "sw_ke moral scenarios" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..a6d5be1f0b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_other" +"task": "mmlu_sw_ke_nutrition" +"task_alias": "sw_ke nutrition" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..eac4161911 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_humanities" +"task": "mmlu_sw_ke_philosophy" +"task_alias": "sw_ke philosophy" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..c7ab0f1e1b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_humanities" +"task": "mmlu_sw_ke_prehistory" +"task_alias": "sw_ke prehistory" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..b9bdb1813a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_other" +"task": "mmlu_sw_ke_professional_accounting" +"task_alias": "sw_ke professional accounting" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..24babec253 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_humanities" +"task": "mmlu_sw_ke_professional_law" +"task_alias": "sw_ke professional law" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..adefb668e4 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_other" +"task": "mmlu_sw_ke_professional_medicine" +"task_alias": "sw_ke professional medicine" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..1040621059 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_social_sciences" +"task": "mmlu_sw_ke_professional_psychology" +"task_alias": "sw_ke professional psychology" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..061921488d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_social_sciences" +"task": "mmlu_sw_ke_public_relations" +"task_alias": "sw_ke public relations" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..a1a78dd045 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_social_sciences" +"task": "mmlu_sw_ke_security_studies" +"task_alias": "sw_ke security studies" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..6dd2777277 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_social_sciences" +"task": "mmlu_sw_ke_sociology" +"task_alias": "sw_ke sociology" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..deab5d00de --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_social_sciences" +"task": "mmlu_sw_ke_us_foreign_policy" +"task_alias": "sw_ke us foreign policy" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..b2218e822d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_other" +"task": "mmlu_sw_ke_virology" +"task_alias": "sw_ke virology" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..122d0eb607 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/sw_ke/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the Swahili (Kenya) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_sw_ke_humanities" +"task": "mmlu_sw_ke_world_religions" +"task_alias": "sw_ke world religions" +"test_split": "SW_KE" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/_openai_mmlu_yo_ng.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/_openai_mmlu_yo_ng.yaml new file mode 100644 index 0000000000..d9f1d96816 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/_openai_mmlu_yo_ng.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_yo_ng +group_alias: yo_ng +metadata: + version: 1.0.0 +task: +- mmlu_yo_ng_tasks diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..02d0f0483a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_abstract_algebra" +"task_alias": "yo_ng abstract algebra" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..0af08a600e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_anatomy" +"task_alias": "yo_ng anatomy" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..9c5d2dc6a0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_astronomy" +"task_alias": "yo_ng astronomy" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..f1d004cceb --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_other" +"task": "mmlu_yo_ng_business_ethics" +"task_alias": "yo_ng business ethics" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..408a6514c8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_other" +"task": "mmlu_yo_ng_clinical_knowledge" +"task_alias": "yo_ng clinical knowledge" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..824a2ebbd2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_college_biology" +"task_alias": "yo_ng college biology" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..7dd4b439e6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_college_chemistry" +"task_alias": "yo_ng college chemistry" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..c580839eff --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_college_computer_science" +"task_alias": "yo_ng college computer science" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..21883debda --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_college_mathematics" +"task_alias": "yo_ng college mathematics" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..a93cafe7e5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_other" +"task": "mmlu_yo_ng_college_medicine" +"task_alias": "yo_ng college medicine" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..8fa5fab6ca --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_college_physics" +"task_alias": "yo_ng college physics" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..591298bb11 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_computer_security" +"task_alias": "yo_ng computer security" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..809ec8f007 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_conceptual_physics" +"task_alias": "yo_ng conceptual physics" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..cf0abe4740 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_social_sciences" +"task": "mmlu_yo_ng_econometrics" +"task_alias": "yo_ng econometrics" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..376afdedb8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_electrical_engineering" +"task_alias": "yo_ng electrical engineering" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..8b511ef6aa --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_elementary_mathematics" +"task_alias": "yo_ng elementary mathematics" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..3d7d27b333 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_humanities" +"task": "mmlu_yo_ng_formal_logic" +"task_alias": "yo_ng formal logic" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..a73017b825 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_other" +"task": "mmlu_yo_ng_global_facts" +"task_alias": "yo_ng global facts" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..26f27a4e97 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_high_school_biology" +"task_alias": "yo_ng high school biology" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..82185d5e3e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_high_school_chemistry" +"task_alias": "yo_ng high school chemistry" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..a6816fcba7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_high_school_computer_science" +"task_alias": "yo_ng high school computer science" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..11dc894e04 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_humanities" +"task": "mmlu_yo_ng_high_school_european_history" +"task_alias": "yo_ng high school european history" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..a50c5c4fa2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_social_sciences" +"task": "mmlu_yo_ng_high_school_geography" +"task_alias": "yo_ng high school geography" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..f1f83b4cca --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_social_sciences" +"task": "mmlu_yo_ng_high_school_government_and_politics" +"task_alias": "yo_ng high school government and politics" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..c5b274b430 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_social_sciences" +"task": "mmlu_yo_ng_high_school_macroeconomics" +"task_alias": "yo_ng high school macroeconomics" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..7c61d27a84 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_high_school_mathematics" +"task_alias": "yo_ng high school mathematics" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..649d4c67d5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_social_sciences" +"task": "mmlu_yo_ng_high_school_microeconomics" +"task_alias": "yo_ng high school microeconomics" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..0a9dcc14d6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_high_school_physics" +"task_alias": "yo_ng high school physics" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..34f43d4901 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_social_sciences" +"task": "mmlu_yo_ng_high_school_psychology" +"task_alias": "yo_ng high school psychology" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..8fa97f4e7b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_high_school_statistics" +"task_alias": "yo_ng high school statistics" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..6f24df0c91 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_humanities" +"task": "mmlu_yo_ng_high_school_us_history" +"task_alias": "yo_ng high school us history" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..bde0fbe86c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_humanities" +"task": "mmlu_yo_ng_high_school_world_history" +"task_alias": "yo_ng high school world history" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..2e2df6d714 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_other" +"task": "mmlu_yo_ng_human_aging" +"task_alias": "yo_ng human aging" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..4c991571d0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_social_sciences" +"task": "mmlu_yo_ng_human_sexuality" +"task_alias": "yo_ng human sexuality" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..568ccd4694 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_humanities" +"task": "mmlu_yo_ng_international_law" +"task_alias": "yo_ng international law" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..a380ca1b09 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_humanities" +"task": "mmlu_yo_ng_jurisprudence" +"task_alias": "yo_ng jurisprudence" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..e0ee1602af --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_humanities" +"task": "mmlu_yo_ng_logical_fallacies" +"task_alias": "yo_ng logical fallacies" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..c6cda55314 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_stem" +"task": "mmlu_yo_ng_machine_learning" +"task_alias": "yo_ng machine learning" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_management.yaml new file mode 100644 index 0000000000..40e312967b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_other" +"task": "mmlu_yo_ng_management" +"task_alias": "yo_ng management" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..fc021f5f0e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_other" +"task": "mmlu_yo_ng_marketing" +"task_alias": "yo_ng marketing" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..1e183ebcb5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_other" +"task": "mmlu_yo_ng_medical_genetics" +"task_alias": "yo_ng medical genetics" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..2ce7451d9c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_other" +"task": "mmlu_yo_ng_miscellaneous" +"task_alias": "yo_ng miscellaneous" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..c3b5a06f9e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_humanities" +"task": "mmlu_yo_ng_moral_disputes" +"task_alias": "yo_ng moral disputes" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..22f3422995 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_humanities" +"task": "mmlu_yo_ng_moral_scenarios" +"task_alias": "yo_ng moral scenarios" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..6d7336dcd6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_other" +"task": "mmlu_yo_ng_nutrition" +"task_alias": "yo_ng nutrition" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..24db5084ca --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_humanities" +"task": "mmlu_yo_ng_philosophy" +"task_alias": "yo_ng philosophy" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..b135979935 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_humanities" +"task": "mmlu_yo_ng_prehistory" +"task_alias": "yo_ng prehistory" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..2968924026 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_other" +"task": "mmlu_yo_ng_professional_accounting" +"task_alias": "yo_ng professional accounting" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..82712a36eb --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_humanities" +"task": "mmlu_yo_ng_professional_law" +"task_alias": "yo_ng professional law" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..6dc1c21329 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_other" +"task": "mmlu_yo_ng_professional_medicine" +"task_alias": "yo_ng professional medicine" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..0ae99a13b3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_social_sciences" +"task": "mmlu_yo_ng_professional_psychology" +"task_alias": "yo_ng professional psychology" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..5cf0f25a9b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_social_sciences" +"task": "mmlu_yo_ng_public_relations" +"task_alias": "yo_ng public relations" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..3a9fec2d7e --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_social_sciences" +"task": "mmlu_yo_ng_security_studies" +"task_alias": "yo_ng security studies" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..b07c2fcec2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_social_sciences" +"task": "mmlu_yo_ng_sociology" +"task_alias": "yo_ng sociology" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..719dc280e5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_social_sciences" +"task": "mmlu_yo_ng_us_foreign_policy" +"task_alias": "yo_ng us foreign policy" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..e8103f7bd7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_other" +"task": "mmlu_yo_ng_virology" +"task_alias": "yo_ng virology" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..4c203029bd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/yo_ng/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the Yoruba (Nigeria) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_yo_ng_humanities" +"task": "mmlu_yo_ng_world_religions" +"task_alias": "yo_ng world religions" +"test_split": "YO_NG" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/_openai_mmlu_zh_cn.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/_openai_mmlu_zh_cn.yaml new file mode 100644 index 0000000000..98f3630ff7 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/_openai_mmlu_zh_cn.yaml @@ -0,0 +1,9 @@ +aggregate_metric_list: +- metric: acc + weight_by_size: true +group: openai_mmlu_zh_cn +group_alias: zh_cn +metadata: + version: 1.0.0 +task: +- mmlu_zh_cn_tasks diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_abstract_algebra.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..18efa16f52 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_abstract_algebra.yaml @@ -0,0 +1,8 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_abstract_algebra" +"task_alias": "zh_cn abstract algebra" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_anatomy.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_anatomy.yaml new file mode 100644 index 0000000000..10d22c5d49 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_anatomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy\ + \ in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_anatomy" +"task_alias": "zh_cn anatomy" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_astronomy.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_astronomy.yaml new file mode 100644 index 0000000000..48cae68a99 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_astronomy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy\ + \ in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_astronomy" +"task_alias": "zh_cn astronomy" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_business_ethics.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..106b62a1f6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_business_ethics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_other" +"task": "mmlu_zh_cn_business_ethics" +"task_alias": "zh_cn business ethics" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..42dcb4db18 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_clinical_knowledge.yaml @@ -0,0 +1,8 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_other" +"task": "mmlu_zh_cn_clinical_knowledge" +"task_alias": "zh_cn clinical knowledge" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_biology.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_biology.yaml new file mode 100644 index 0000000000..707f886e7b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_college_biology" +"task_alias": "zh_cn college biology" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_chemistry.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..0b7733cbc2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_college_chemistry" +"task_alias": "zh_cn college chemistry" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_computer_science.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..0916393acf --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_college_computer_science" +"task_alias": "zh_cn college computer science" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_mathematics.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..2f9cb7546b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_college_mathematics" +"task_alias": "zh_cn college mathematics" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_medicine.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..45a6b3f43c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_other" +"task": "mmlu_zh_cn_college_medicine" +"task_alias": "zh_cn college medicine" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_physics.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_physics.yaml new file mode 100644 index 0000000000..71502e01d8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_college_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_college_physics" +"task_alias": "zh_cn college physics" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_computer_security.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_computer_security.yaml new file mode 100644 index 0000000000..bf02e2ad45 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_computer_security.yaml @@ -0,0 +1,8 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_computer_security" +"task_alias": "zh_cn computer security" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_conceptual_physics.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..9aff34ef53 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_conceptual_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_conceptual_physics" +"task_alias": "zh_cn conceptual physics" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_econometrics.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_econometrics.yaml new file mode 100644 index 0000000000..6e4e29e610 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_econometrics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics\ + \ in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_social_sciences" +"task": "mmlu_zh_cn_econometrics" +"task_alias": "zh_cn econometrics" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_electrical_engineering.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..a031f799e9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_electrical_engineering.yaml @@ -0,0 +1,8 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_electrical_engineering" +"task_alias": "zh_cn electrical engineering" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..a22ed20b47 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_elementary_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_elementary_mathematics" +"task_alias": "zh_cn elementary mathematics" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_formal_logic.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..ff2f88df32 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_formal_logic.yaml @@ -0,0 +1,8 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_humanities" +"task": "mmlu_zh_cn_formal_logic" +"task_alias": "zh_cn formal logic" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_global_facts.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_global_facts.yaml new file mode 100644 index 0000000000..dad8d6a465 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_global_facts.yaml @@ -0,0 +1,8 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_other" +"task": "mmlu_zh_cn_global_facts" +"task_alias": "zh_cn global facts" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_biology.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..e21dbb6dd8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_biology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_high_school_biology" +"task_alias": "zh_cn high school biology" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..31d771b8d3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_chemistry.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_high_school_chemistry" +"task_alias": "zh_cn high school chemistry" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..6865327031 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_computer_science.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_high_school_computer_science" +"task_alias": "zh_cn high school computer science" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_european_history.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..2afb7b66ad --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_european_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_humanities" +"task": "mmlu_zh_cn_high_school_european_history" +"task_alias": "zh_cn high school european history" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_geography.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..904cb42615 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_geography.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_social_sciences" +"task": "mmlu_zh_cn_high_school_geography" +"task_alias": "zh_cn high school geography" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..c95d47385a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_social_sciences" +"task": "mmlu_zh_cn_high_school_government_and_politics" +"task_alias": "zh_cn high school government and politics" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..6a74b61256 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_social_sciences" +"task": "mmlu_zh_cn_high_school_macroeconomics" +"task_alias": "zh_cn high school macroeconomics" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..8f1ebc8486 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_mathematics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_high_school_mathematics" +"task_alias": "zh_cn high school mathematics" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..cf848a584d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_social_sciences" +"task": "mmlu_zh_cn_high_school_microeconomics" +"task_alias": "zh_cn high school microeconomics" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_physics.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..6386cecba5 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_physics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_high_school_physics" +"task_alias": "zh_cn high school physics" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_psychology.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..a38490c19b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_social_sciences" +"task": "mmlu_zh_cn_high_school_psychology" +"task_alias": "zh_cn high school psychology" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_statistics.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..788e347922 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_statistics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_high_school_statistics" +"task_alias": "zh_cn high school statistics" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_us_history.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..0c82619ac3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_us_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_humanities" +"task": "mmlu_zh_cn_high_school_us_history" +"task_alias": "zh_cn high school us history" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_world_history.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..2c9a122d50 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_high_school_world_history.yaml @@ -0,0 +1,8 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_humanities" +"task": "mmlu_zh_cn_high_school_world_history" +"task_alias": "zh_cn high school world history" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_human_aging.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_human_aging.yaml new file mode 100644 index 0000000000..414eebba81 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_human_aging.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_other" +"task": "mmlu_zh_cn_human_aging" +"task_alias": "zh_cn human aging" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_human_sexuality.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..54e5b1e552 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_human_sexuality.yaml @@ -0,0 +1,8 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_social_sciences" +"task": "mmlu_zh_cn_human_sexuality" +"task_alias": "zh_cn human sexuality" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_international_law.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_international_law.yaml new file mode 100644 index 0000000000..4d7a59e2a1 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_international_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_humanities" +"task": "mmlu_zh_cn_international_law" +"task_alias": "zh_cn international law" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_jurisprudence.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..68d4d9590d --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_jurisprudence.yaml @@ -0,0 +1,8 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence\ + \ in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_humanities" +"task": "mmlu_zh_cn_jurisprudence" +"task_alias": "zh_cn jurisprudence" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_logical_fallacies.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..462fd0923b --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_logical_fallacies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_humanities" +"task": "mmlu_zh_cn_logical_fallacies" +"task_alias": "zh_cn logical fallacies" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_machine_learning.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..3ee9fffbe8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_machine_learning.yaml @@ -0,0 +1,8 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_stem" +"task": "mmlu_zh_cn_machine_learning" +"task_alias": "zh_cn machine learning" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_management.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_management.yaml new file mode 100644 index 0000000000..6ed8153b23 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_management.yaml @@ -0,0 +1,8 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management\ + \ in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_other" +"task": "mmlu_zh_cn_management" +"task_alias": "zh_cn management" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_marketing.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_marketing.yaml new file mode 100644 index 0000000000..e6a548292a --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_marketing.yaml @@ -0,0 +1,8 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing\ + \ in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_other" +"task": "mmlu_zh_cn_marketing" +"task_alias": "zh_cn marketing" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_medical_genetics.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..ee468b5338 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_medical_genetics.yaml @@ -0,0 +1,8 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_other" +"task": "mmlu_zh_cn_medical_genetics" +"task_alias": "zh_cn medical genetics" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_miscellaneous.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..95503b243c --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_miscellaneous.yaml @@ -0,0 +1,8 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous\ + \ in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_other" +"task": "mmlu_zh_cn_miscellaneous" +"task_alias": "zh_cn miscellaneous" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_moral_disputes.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..06277c2389 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_moral_disputes.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_humanities" +"task": "mmlu_zh_cn_moral_disputes" +"task_alias": "zh_cn moral disputes" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_moral_scenarios.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..56b44b3fc0 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_moral_scenarios.yaml @@ -0,0 +1,8 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_humanities" +"task": "mmlu_zh_cn_moral_scenarios" +"task_alias": "zh_cn moral scenarios" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_nutrition.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_nutrition.yaml new file mode 100644 index 0000000000..f1803dfcfd --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_nutrition.yaml @@ -0,0 +1,8 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition\ + \ in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_other" +"task": "mmlu_zh_cn_nutrition" +"task_alias": "zh_cn nutrition" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_philosophy.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_philosophy.yaml new file mode 100644 index 0000000000..8795d4a516 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_philosophy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy\ + \ in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_humanities" +"task": "mmlu_zh_cn_philosophy" +"task_alias": "zh_cn philosophy" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_prehistory.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_prehistory.yaml new file mode 100644 index 0000000000..7165c1d097 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_prehistory.yaml @@ -0,0 +1,8 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory\ + \ in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_humanities" +"task": "mmlu_zh_cn_prehistory" +"task_alias": "zh_cn prehistory" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_accounting.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..c454e9bbe8 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_accounting.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_other" +"task": "mmlu_zh_cn_professional_accounting" +"task_alias": "zh_cn professional accounting" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_law.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_law.yaml new file mode 100644 index 0000000000..1357ab2ad6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_law.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_humanities" +"task": "mmlu_zh_cn_professional_law" +"task_alias": "zh_cn professional law" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_medicine.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..c480f837e2 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_medicine.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_other" +"task": "mmlu_zh_cn_professional_medicine" +"task_alias": "zh_cn professional medicine" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_psychology.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..011e07fe99 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_professional_psychology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_social_sciences" +"task": "mmlu_zh_cn_professional_psychology" +"task_alias": "zh_cn professional psychology" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_public_relations.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_public_relations.yaml new file mode 100644 index 0000000000..780da6d8d9 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_public_relations.yaml @@ -0,0 +1,8 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_social_sciences" +"task": "mmlu_zh_cn_public_relations" +"task_alias": "zh_cn public relations" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_security_studies.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_security_studies.yaml new file mode 100644 index 0000000000..d062214f34 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_security_studies.yaml @@ -0,0 +1,8 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_social_sciences" +"task": "mmlu_zh_cn_security_studies" +"task_alias": "zh_cn security studies" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_sociology.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_sociology.yaml new file mode 100644 index 0000000000..19eb689688 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_sociology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology\ + \ in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_social_sciences" +"task": "mmlu_zh_cn_sociology" +"task_alias": "zh_cn sociology" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..6692ca16b6 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_us_foreign_policy.yaml @@ -0,0 +1,8 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_social_sciences" +"task": "mmlu_zh_cn_us_foreign_policy" +"task_alias": "zh_cn us foreign policy" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_virology.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_virology.yaml new file mode 100644 index 0000000000..be591a5199 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_virology.yaml @@ -0,0 +1,8 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology\ + \ in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_other" +"task": "mmlu_zh_cn_virology" +"task_alias": "zh_cn virology" +"test_split": "ZH_CN" diff --git a/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_world_religions.yaml b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_world_religions.yaml new file mode 100644 index 0000000000..019305ddf3 --- /dev/null +++ b/lm_eval/tasks/openai_mmlu/zh_cn/openai_mmlu_world_religions.yaml @@ -0,0 +1,8 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions in the Chinese (China) language.\n\n" +"include": "../_default_template.yaml" +"tag": "mmlu_zh_cn_humanities" +"task": "mmlu_zh_cn_world_religions" +"task_alias": "zh_cn world religions" +"test_split": "ZH_CN"