From b31f92e8bec4c40e5c75f9b4ffb9c42cb10a725f Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Fri, 30 Aug 2024 06:25:06 +0500 Subject: [PATCH 1/6] API: fix maxlen; vllm: prefix_token_id bug (#2262) * max_length - 1 (generation always >= 1) * vllm: fix rolling prefix_token * nit: add comment * fixup! max_length should be handled for logliklihoods --- lm_eval/models/api_models.py | 9 ++++++--- lm_eval/models/vllm_causallms.py | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lm_eval/models/api_models.py b/lm_eval/models/api_models.py index c6aa660847..25223e906a 100644 --- a/lm_eval/models/api_models.py +++ b/lm_eval/models/api_models.py @@ -104,6 +104,7 @@ def __init__( self._truncate = truncate self._max_gen_toks = int(max_gen_toks) self._seed = int(seed) + eval_logger.info(f"Using max length {max_length}") self.max_length = max_length if int(num_concurrent) <= 1: eval_logger.info( @@ -417,9 +418,10 @@ def batch_logliklehood_requests( cache_keys = [] for chunk in chunks: for cache_key, context_enc, continuation_enc in chunk: - inp = (context_enc + continuation_enc)[-(self.max_length) :] + # max_length - 1 as we always have 1 token for generation + inp = (context_enc + continuation_enc)[-(self.max_length - 1) :] ctxlen = len(context_enc) - max( - 0, len(context_enc) + len(continuation_enc) - (self.max_length) + 0, len(context_enc) + len(continuation_enc) - (self.max_length - 1) ) inputs.append(inp) @@ -619,7 +621,8 @@ def loglikelihood_rolling( utils.get_rolling_token_windows( token_list=self.tok_encode(string), prefix_token=self.prefix_token_id, - max_seq_len=self.max_length, + # max_seq_len - (1 for context) + max_seq_len=self.max_length - 1, context_len=1, ), ) diff --git a/lm_eval/models/vllm_causallms.py b/lm_eval/models/vllm_causallms.py index c79ae5489f..d6a8dc1817 100644 --- a/lm_eval/models/vllm_causallms.py +++ b/lm_eval/models/vllm_causallms.py @@ -289,7 +289,8 @@ def loglikelihood_rolling( make_disjoint_window, get_rolling_token_windows( token_list=self.tok_encode(string), - prefix_token=self.eot_token_id, + prefix_token=self.prefix_token_id, + # max_seq_len - (1 for context) max_seq_len=self.max_length - 1, context_len=1, ), From 928e8bb6f50d1e93ef5d0bcaa81f8c5fd9a6f4d8 Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Fri, 30 Aug 2024 06:40:41 +0500 Subject: [PATCH 2/6] hotfix #2262 (#2264) * max_length - 1 (generation always >= 1) * vllm: fix rolling prefix_token * nit: add comment * fixup! max_length should be handled for logliklihoods * Revert "fixup! max_length should be handled for logliklihoods" This reverts commit 432d1a3b754c117c3a54ea2fe792ab3a1bd09ed3. --- lm_eval/models/api_models.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lm_eval/models/api_models.py b/lm_eval/models/api_models.py index 25223e906a..9d21705512 100644 --- a/lm_eval/models/api_models.py +++ b/lm_eval/models/api_models.py @@ -104,8 +104,9 @@ def __init__( self._truncate = truncate self._max_gen_toks = int(max_gen_toks) self._seed = int(seed) - eval_logger.info(f"Using max length {max_length}") - self.max_length = max_length + # max_length - 1 as we always have 1 token for generation + eval_logger.info(f"Using max length {max_length} - 1") + self.max_length = max_length - 1 if int(num_concurrent) <= 1: eval_logger.info( "Concurrent requests are disabled. To enable concurrent requests, set `num_concurrent` > 1." @@ -419,9 +420,9 @@ def batch_logliklehood_requests( for chunk in chunks: for cache_key, context_enc, continuation_enc in chunk: # max_length - 1 as we always have 1 token for generation - inp = (context_enc + continuation_enc)[-(self.max_length - 1) :] + inp = (context_enc + continuation_enc)[-(self.max_length) :] ctxlen = len(context_enc) - max( - 0, len(context_enc) + len(continuation_enc) - (self.max_length - 1) + 0, len(context_enc) + len(continuation_enc) - (self.max_length) ) inputs.append(inp) From 7a1614eb90d29b2983ffa027a7974b7ef53fba19 Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Wed, 4 Sep 2024 19:09:57 +0500 Subject: [PATCH 3/6] Chat Template fix (cont. #2235) (#2269) * default chat template method fix * move chat_template to TemplateLM * remove hotfix * handle openai `chat_template` * Update lm_eval/api/model.py Co-authored-by: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com> * add 'max_tokens' to gen_kwargs * pre-commit --------- Co-authored-by: KonradSzafer Co-authored-by: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com> --- lm_eval/api/model.py | 114 ++++++++++++++++++++++++--- lm_eval/evaluator.py | 8 +- lm_eval/models/api_models.py | 8 -- lm_eval/models/huggingface.py | 92 --------------------- lm_eval/models/openai_completions.py | 13 ++- lm_eval/models/vllm_causallms.py | 6 -- 6 files changed, 117 insertions(+), 124 deletions(-) diff --git a/lm_eval/api/model.py b/lm_eval/api/model.py index 1cf8aedb66..765fc01222 100644 --- a/lm_eval/api/model.py +++ b/lm_eval/api/model.py @@ -3,7 +3,7 @@ import json import logging import os -from typing import Dict, List, Optional, Tuple, Type, TypeVar +from typing import Dict, List, Optional, Tuple, Type, TypeVar, Union import transformers from sqlitedict import SqliteDict @@ -192,15 +192,13 @@ def tokenizer_name(self) -> str: "To use this model with chat templates, please implement the 'tokenizer_name' property." ) - @property - def chat_template(self) -> str: - """Must be defined for LM subclasses that implement Chat Templating. - Should return the structure of the chat template applied to user/assistant messages. - This is used only to save in the experiment results for reproducibility. + def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]: + """Returns the chat template structure for user/assistant messages if a template is provided. + This method is intended to be overridden in a subclass to define a specific chat template format. + For models that do not support chat templates, this method returns None by default. """ - raise NotImplementedError( - "To use this model with chat templates, please implement the 'chat_template' property." - ) + + return "" def set_cache_hook(self, cache_hook) -> None: self.cache_hook = cache_hook @@ -316,6 +314,8 @@ class TemplateLM(LM): and boilerplate often included in other LM subclasses. """ + tokenizer = None + @property @abc.abstractmethod def eot_token_id(self): @@ -386,3 +386,99 @@ def loglikelihood_rolling( @abc.abstractmethod def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]: pass + + def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]: + """ + Set and get the appropriate chat template for the model. + This method sets the tokenizer's chat_template and returns the template string for reproducibility. + + The template selection logic is adapted from the Transformers library's `apply_chat_template` + method in the Tokenizer class. The original implementation can be found at: + https://github.com/huggingface/transformers/blob/fc35907f95459d7a6c5281dfadd680b6f7b620e3/src/transformers/tokenization_utils_base.py#L1687 + + This method ensures that the right template is chosen based on the following: + 0. If the model has no 'tokenizer' attribute: assumes that there is only a single possible chat template, handled on the model provider side internally. Returns the empty string. + 1. If the model's tokenizer has multiple templates: + a. Use the specified template if it exists in the dictionary. + b. Use the default template from the list if no specific template is provided. + c. Raise an error if no default template exists and no specific template is provided. + 2. If the model's tokenizer has a single template or no template: + a. Use the tokenizer's chat template if available. + b. Fall back to the default chat template if no tokenizer chat template exists. + + Args: + chat_template (Union[bool, str]): Specifies the chat template to use. + - If False or None, no template is applied. + - If True, the default or only available template is used. + - If a string, the template with the matching name is used. + + Returns: + Optional[str]: The selected chat template, or None if no template is applied. + """ + if self.tokenizer is None: + return "" + + if chat_template is False or chat_template is None: + eval_logger.warning( + "model.chat_template was called with the chat_template set to False or None. " + "Therefore no chat template will be applied. Make sure this is an intended behavior." + ) + return None + + # Convert boolean chat_template to None to ensure compatibility with the adapted logic + if isinstance(chat_template, bool): + chat_template = None + using_default_template = False + + # First, handle the cases when the model has a dict of multiple templates + template = self.tokenizer.chat_template or self.tokenizer.default_chat_template + + if isinstance(template, dict): + using_default_dict = self.tokenizer.chat_template is None + + if chat_template is not None: + if chat_template in template: + selected_template = template[chat_template] + if using_default_dict: + using_default_template = True + else: + raise ValueError( + f"The specified chat template '{chat_template}' is not available. " + f"Available template names are {sorted(template.keys())}." + ) + else: + # If user didn't pass a chat template, use the default template from the dict + if "default" in template: + selected_template = template["default"] + using_default_template = True + else: + raise ValueError( + "This model has multiple chat templates with no default specified! Please either pass a chat " + "template or the name of the template you wish to use to the `chat_template` argument. Available " + f"template names are {sorted(template.keys())}." + ) + + # Cases when the model has a single template or no template + else: + # priority: `chat_template` argument > `tokenizer.chat_template` > `tokenizer.default_chat_template + if isinstance(chat_template, str): + eval_logger.warning( + "Chat template name provided, but the tokenizer's chat template is not a dictionary. " + "Using the tokenizer's chat template or the default template instead." + ) + if self.tokenizer.chat_template is not None: + selected_template = self.tokenizer.chat_template + else: + selected_template = self.tokenizer.default_chat_template + using_default_template = True + + if using_default_template: + eval_logger.warning( + "No chat template is set for this tokenizer, falling back to a default class-level template. This is " + "very error-prone, because models are often trained with templates different from the class default! " + "Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which " + "point any code depending on them will stop working. We recommend setting a valid chat template before " + "then to ensure that this model continues working without issues." + ) + + return selected_template diff --git a/lm_eval/evaluator.py b/lm_eval/evaluator.py index 9bd8288860..52a8d00d74 100644 --- a/lm_eval/evaluator.py +++ b/lm_eval/evaluator.py @@ -289,18 +289,12 @@ def _adjust_config(task_dict): if check_integrity: run_task_tests(task_list=tasks) - # hotfix: delete when chat_template fixed - try: - chat = lm.chat_template(apply_chat_template) - except: # noqa: E722 - chat = None - if evaluation_tracker is not None: evaluation_tracker.general_config_tracker.log_experiment_args( model_source=model, model_args=model_args, system_instruction=system_instruction, - chat_template=chat, + chat_template=lm.chat_template(apply_chat_template), fewshot_as_multiturn=fewshot_as_multiturn, ) diff --git a/lm_eval/models/api_models.py b/lm_eval/models/api_models.py index 9d21705512..7098893947 100644 --- a/lm_eval/models/api_models.py +++ b/lm_eval/models/api_models.py @@ -225,14 +225,6 @@ def header(self) -> dict: """Override this property to return the headers for the API request.""" return {"Authorization": f"Bearer {self.api_key}"} - @property - def chat_template(self) -> str: - """Must be defined for LM subclasses that implement Chat Templating. - Should return the structure of the chat template applied to user/assistant messages. - Only used for logging and reproducibility. - """ - return "" - @property def tokenizer_name(self) -> str: """Must be defined for LM subclasses which implement Chat Templating. diff --git a/lm_eval/models/huggingface.py b/lm_eval/models/huggingface.py index 80f65ca536..01a8cf6789 100644 --- a/lm_eval/models/huggingface.py +++ b/lm_eval/models/huggingface.py @@ -438,98 +438,6 @@ def world_size(self): def tokenizer_name(self) -> str: return self.tokenizer.name_or_path.replace("/", "__") - def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]: - """ - Get the appropriate chat template for the model based on configuration and input. - This method determines, and returns the correct chat template, ensuring reproducibility. - - The template selection logic is adapted from the Transformers library's `apply_chat_template` - method in the Tokenizer class. The original implementation can be found at: - https://github.com/huggingface/transformers/blob/fc35907f95459d7a6c5281dfadd680b6f7b620e3/src/transformers/tokenization_utils_base.py#L1687 - - This method ensures that the right template is chosen based on the following: - 1. If the model's tokenizer has multiple templates: - a. Use the specified template if it exists in the dictionary. - b. Use the default template from the list if no specific template is provided. - c. Raise an error if no default template exists and no specific template is provided. - 2. If the model's tokenizer has a single template or no template: - a. Use the tokenizer's chat template if available. - b. Fall back to the default chat template if no tokenizer chat template exists. - - Args: - chat_template (Union[bool, str]): Specifies the chat template to use. - - If False or None, no template is applied. - - If True, the default or only available template is used. - - If a string, the template with the matching name is used. - - Returns: - Optional[str]: The selected chat template, or None if no template is applied. - """ - if chat_template is False or chat_template is None: - eval_logger.warning( - "model.chat_template was called with the chat_template set to False or None. " - "Therefore no chat template will be applied. Make sure this is an intended behavior." - ) - return None - - # Convert boolean chat_template to None to ensure compatibility with the adapted logic - if isinstance(chat_template, bool): - chat_template = None - using_default_template = False - - # First, handle the cases when the model has a dict of multiple templates - template = self.tokenizer.chat_template or self.tokenizer.default_chat_template - - if isinstance(template, dict): - using_default_dict = self.tokenizer.chat_template is None - - if chat_template is not None: - if chat_template in template: - selected_template = template[chat_template] - if using_default_dict: - using_default_template = True - else: - raise ValueError( - f"The specified chat template '{chat_template}' is not available. " - f"Available template names are {sorted(template.keys())}." - ) - else: - # If user didn't pass a chat template, use the default template from the dict - if "default" in template: - selected_template = template["default"] - using_default_template = True - else: - raise ValueError( - "This model has multiple chat templates with no default specified! Please either pass a chat " - "template or the name of the template you wish to use to the `chat_template` argument. Available " - f"template names are {sorted(template.keys())}." - ) - - # Cases when the model has a single template or no template - else: - # priority: `chat_template` argument > `tokenizer.chat_template` > `tokenizer.default_chat_template - if isinstance(chat_template, str): - eval_logger.warning( - "Chat template name provided, but the tokenizer's chat template is not a dictionary. " - "Using the tokenizer's chat template or the default template instead." - ) - if self.tokenizer.chat_template is not None: - selected_template = self.tokenizer.chat_template - else: - selected_template = self.tokenizer.default_chat_template - using_default_template = True - - if using_default_template: - eval_logger.warning( - "No chat template is set for this tokenizer, falling back to a default class-level template. This is " - "very error-prone, because models are often trained with templates different from the class default! " - "Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which " - "point any code depending on them will stop working. We recommend setting a valid chat template before " - "then to ensure that this model continues working without issues." - ) - - return selected_template - def _get_backend( self, config: Union[transformers.PretrainedConfig, transformers.AutoConfig], diff --git a/lm_eval/models/openai_completions.py b/lm_eval/models/openai_completions.py index 26dc93d68f..4bf2c42b1e 100644 --- a/lm_eval/models/openai_completions.py +++ b/lm_eval/models/openai_completions.py @@ -29,7 +29,10 @@ def _create_payload( ) -> dict: if generate: gen_kwargs.pop("do_sample", False) - max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks) + if "max_tokens" in gen_kwargs: + max_tokens = gen_kwargs.pop("max_tokens") + else: + max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks) temperature = gen_kwargs.pop("temperature", 0) stop = gen_kwargs.pop("until", ["<|endoftext|>"]) return { @@ -124,7 +127,10 @@ def _create_payload( **kwargs, ) -> dict: gen_kwargs.pop("do_sample", False) - max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks) + if "max_tokens" in gen_kwargs: + max_tokens = gen_kwargs.pop("max_tokens") + else: + max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks) temperature = gen_kwargs.pop("temperature", 0) stop = gen_kwargs.pop("until", ["<|endoftext|>"]) if not isinstance(stop, (list, tuple)): @@ -194,6 +200,9 @@ def loglikelihood(self, requests, **kwargs): ), "Loglikelihood is not supported for gpt-3.5-turbo" return super().loglikelihood(requests, **kwargs) + def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]: + return "" + @register_model("openai-chat-completions") class OpenAIChatCompletion(LocalChatCompletion): diff --git a/lm_eval/models/vllm_causallms.py b/lm_eval/models/vllm_causallms.py index d6a8dc1817..168f490a7b 100644 --- a/lm_eval/models/vllm_causallms.py +++ b/lm_eval/models/vllm_causallms.py @@ -187,12 +187,6 @@ def apply_chat_template(self, chat_history: List[Dict[str, str]]) -> str: chat_history, tokenize=False, add_generation_prompt=True ) - @property - def chat_template(self) -> str: - if self.tokenizer.chat_template is not None: - return self.tokenizer.chat_template - return self.tokenizer.default_chat_template - @property def tokenizer_name(self) -> str: return self.tokenizer.name_or_path.replace("/", "__") From 543617fef9ba885e87f8db8930fbbff1d4e2ca49 Mon Sep 17 00:00:00 2001 From: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com> Date: Thu, 5 Sep 2024 11:01:48 -0400 Subject: [PATCH 4/6] Bump version to v0.4.4 ; Fixes to TMMLUplus (#2280) --- lm_eval/__main__.py | 7 ------- lm_eval/tasks/__init__.py | 10 ++++++---- lm_eval/tasks/tmmluplus/default/_tmmluplus.yaml | 13 +++++++++++++ .../tasks/tmmluplus/default/_tmmluplus_STEM.yaml | 10 ++++++++++ .../tmmluplus/default/_tmmluplus_humanities.yaml | 10 ++++++++++ .../tasks/tmmluplus/default/_tmmluplus_other.yaml | 10 ++++++++++ .../default/_tmmluplus_social_sciences.yaml | 10 ++++++++++ ...fault_template_yaml => _tmmluplus_template_yaml} | 2 +- lm_eval/tasks/tmmluplus/default/tmmluplus.yaml | 6 ------ .../tmmluplus/default/tmmluplus_accounting.yaml | 5 ++--- .../default/tmmluplus_administrative_law.yaml | 5 ++--- .../default/tmmluplus_advance_chemistry.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_agriculture.yaml | 5 ++--- .../default/tmmluplus_anti_money_laundering.yaml | 5 ++--- .../tasks/tmmluplus/default/tmmluplus_auditing.yaml | 5 ++--- .../default/tmmluplus_basic_medical_science.yaml | 5 ++--- .../default/tmmluplus_business_management.yaml | 5 ++--- .../tmmluplus_chinese_language_and_literature.yaml | 5 ++--- .../default/tmmluplus_clinical_psychology.yaml | 5 ++--- .../default/tmmluplus_computer_science.yaml | 5 ++--- .../default/tmmluplus_culinary_skills.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_dentistry.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_economics.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_education.yaml | 5 ++--- .../tmmluplus_education_(profession_level).yaml | 5 ++--- .../default/tmmluplus_educational_psychology.yaml | 5 ++--- .../default/tmmluplus_engineering_math.yaml | 5 ++--- .../default/tmmluplus_finance_banking.yaml | 5 ++--- .../default/tmmluplus_financial_analysis.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_fire_science.yaml | 5 ++--- .../tmmluplus_general_principles_of_law.yaml | 5 ++--- .../default/tmmluplus_geography_of_taiwan.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_human_behavior.yaml | 5 ++--- .../default/tmmluplus_insurance_studies.yaml | 5 ++--- .../default/tmmluplus_introduction_to_law.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_jce_humanities.yaml | 5 ++--- .../default/tmmluplus_junior_chemistry.yaml | 5 ++--- .../default/tmmluplus_junior_chinese_exam.yaml | 5 ++--- .../default/tmmluplus_junior_math_exam.yaml | 5 ++--- .../default/tmmluplus_junior_science_exam.yaml | 5 ++--- .../default/tmmluplus_junior_social_studies.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_linear_algebra.yaml | 5 ++--- .../default/tmmluplus_logic_reasoning.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_macroeconomics.yaml | 5 ++--- .../default/tmmluplus_management_accounting.yaml | 5 ++--- .../default/tmmluplus_marketing_management.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_mechanical.yaml | 5 ++--- .../tasks/tmmluplus/default/tmmluplus_music.yaml | 5 ++--- .../default/tmmluplus_national_protection.yaml | 5 ++--- .../default/tmmluplus_nautical_science.yaml | 5 ++--- ...ational_therapy_for_psychological_disorders.yaml | 5 ++--- .../tmmluplus_official_document_management.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_optometry.yaml | 5 ++--- .../default/tmmluplus_organic_chemistry.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_pharmacology.yaml | 5 ++--- .../tasks/tmmluplus/default/tmmluplus_pharmacy.yaml | 5 ++--- .../default/tmmluplus_physical_education.yaml | 5 ++--- .../tasks/tmmluplus/default/tmmluplus_physics.yaml | 5 ++--- .../default/tmmluplus_politic_science.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_real_estate.yaml | 5 ++--- .../default/tmmluplus_secondary_physics.yaml | 5 ++--- .../tmmluplus_statistics_and_machine_learning.yaml | 5 ++--- .../default/tmmluplus_taiwanese_hokkien.yaml | 5 ++--- .../tasks/tmmluplus/default/tmmluplus_taxation.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_technical.yaml | 5 ++--- .../tmmluplus_three_principles_of_people.yaml | 5 ++--- .../tasks/tmmluplus/default/tmmluplus_trade.yaml | 5 ++--- ...ditional_chinese_medicine_clinical_medicine.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_trust_practice.yaml | 5 ++--- .../tasks/tmmluplus/default/tmmluplus_ttqav2.yaml | 5 ++--- .../default/tmmluplus_tve_chinese_language.yaml | 5 ++--- .../tmmluplus/default/tmmluplus_tve_design.yaml | 5 ++--- .../default/tmmluplus_tve_mathematics.yaml | 5 ++--- .../default/tmmluplus_tve_natural_sciences.yaml | 5 ++--- .../default/tmmluplus_veterinary_pathology.yaml | 5 ++--- .../default/tmmluplus_veterinary_pharmacology.yaml | 5 ++--- pyproject.toml | 2 +- 77 files changed, 195 insertions(+), 220 deletions(-) create mode 100644 lm_eval/tasks/tmmluplus/default/_tmmluplus.yaml create mode 100644 lm_eval/tasks/tmmluplus/default/_tmmluplus_STEM.yaml create mode 100644 lm_eval/tasks/tmmluplus/default/_tmmluplus_humanities.yaml create mode 100644 lm_eval/tasks/tmmluplus/default/_tmmluplus_other.yaml create mode 100644 lm_eval/tasks/tmmluplus/default/_tmmluplus_social_sciences.yaml rename lm_eval/tasks/tmmluplus/default/{_default_template_yaml => _tmmluplus_template_yaml} (97%) delete mode 100644 lm_eval/tasks/tmmluplus/default/tmmluplus.yaml diff --git a/lm_eval/__main__.py b/lm_eval/__main__.py index f10c9b85ca..ab68781939 100644 --- a/lm_eval/__main__.py +++ b/lm_eval/__main__.py @@ -299,13 +299,6 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: "When `fewshot_as_multiturn` is selected, `apply_chat_template` must be set (either to `True` or to the chosen template name)." ) - if ( - args.num_fewshot is None or args.num_fewshot == 0 - ) and args.fewshot_as_multiturn: - raise ValueError( - "If fewshot_as_multiturn is set, num_fewshot must be greater than 0." - ) - if args.include_path is not None: eval_logger.info(f"Including path: {args.include_path}") task_manager = TaskManager(args.verbosity, include_path=args.include_path) diff --git a/lm_eval/tasks/__init__.py b/lm_eval/tasks/__init__.py index 4cae685284..7eeec8bbc6 100644 --- a/lm_eval/tasks/__init__.py +++ b/lm_eval/tasks/__init__.py @@ -489,10 +489,12 @@ def _get_task_and_group(self, task_dir: str): if attr in config: if attr == "group" and print_info: self.logger.info( - "`group` and `group_alias` keys in tasks' configs will no longer be used in the next release of lm-eval. " - "`tag` will be used to allow to call a collection of tasks just like `group`. " - "`group` will be removed in order to not cause confusion with the new ConfigurableGroup " - "which will be the official way to create groups with addition of group-wide configurations." + "`group` and `group_alias` keys in TaskConfigs are deprecated and will be removed in v0.4.5 of lm_eval. " + "The new `tag` field will be used to allow for a shortcut to a group of tasks one does not wish to aggregate metrics across. " + "`group`s which aggregate across subtasks must be only defined in a separate group config file, " + "which will be the official way to create groups that support cross-task aggregation as in `mmlu`. " + "Please see the v0.4.4 patch notes and our documentation: https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/new_task_guide.md#advanced-group-configs " + "for more information." ) print_info = False # attr = "tag" diff --git a/lm_eval/tasks/tmmluplus/default/_tmmluplus.yaml b/lm_eval/tasks/tmmluplus/default/_tmmluplus.yaml new file mode 100644 index 0000000000..45208d4dc2 --- /dev/null +++ b/lm_eval/tasks/tmmluplus/default/_tmmluplus.yaml @@ -0,0 +1,13 @@ +group: tmmluplus +task: +- tmmluplus_other +- tmmluplus_social_sciences +- tmmluplus_humanities +- tmmluplus_STEM +aggregate_metric_list: + - metric: acc + weight_by_size: True + - metric: acc_norm + weight_by_size: True +metadata: + version: 2.0 diff --git a/lm_eval/tasks/tmmluplus/default/_tmmluplus_STEM.yaml b/lm_eval/tasks/tmmluplus/default/_tmmluplus_STEM.yaml new file mode 100644 index 0000000000..47f81f5085 --- /dev/null +++ b/lm_eval/tasks/tmmluplus/default/_tmmluplus_STEM.yaml @@ -0,0 +1,10 @@ +group: tmmluplus_STEM +task: +- tmmluplus_STEM_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True + - metric: acc_norm + weight_by_size: True +metadata: + version: 2.0 diff --git a/lm_eval/tasks/tmmluplus/default/_tmmluplus_humanities.yaml b/lm_eval/tasks/tmmluplus/default/_tmmluplus_humanities.yaml new file mode 100644 index 0000000000..1cd42f88dd --- /dev/null +++ b/lm_eval/tasks/tmmluplus/default/_tmmluplus_humanities.yaml @@ -0,0 +1,10 @@ +group: tmmluplus_humanities +task: +- tmmluplus_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True + - metric: acc_norm + weight_by_size: True +metadata: + version: 2.0 diff --git a/lm_eval/tasks/tmmluplus/default/_tmmluplus_other.yaml b/lm_eval/tasks/tmmluplus/default/_tmmluplus_other.yaml new file mode 100644 index 0000000000..2b679ef414 --- /dev/null +++ b/lm_eval/tasks/tmmluplus/default/_tmmluplus_other.yaml @@ -0,0 +1,10 @@ +group: tmmluplus_other +task: +- tmmluplus_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True + - metric: acc_norm + weight_by_size: True +metadata: + version: 2.0 diff --git a/lm_eval/tasks/tmmluplus/default/_tmmluplus_social_sciences.yaml b/lm_eval/tasks/tmmluplus/default/_tmmluplus_social_sciences.yaml new file mode 100644 index 0000000000..a219550bbc --- /dev/null +++ b/lm_eval/tasks/tmmluplus/default/_tmmluplus_social_sciences.yaml @@ -0,0 +1,10 @@ +group: tmmluplus_social_sciences +task: +- tmmluplus_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True + - metric: acc_norm + weight_by_size: True +metadata: + version: 2.0 diff --git a/lm_eval/tasks/tmmluplus/default/_default_template_yaml b/lm_eval/tasks/tmmluplus/default/_tmmluplus_template_yaml similarity index 97% rename from lm_eval/tasks/tmmluplus/default/_default_template_yaml rename to lm_eval/tasks/tmmluplus/default/_tmmluplus_template_yaml index 98d3c133cd..c03cfb1b37 100644 --- a/lm_eval/tasks/tmmluplus/default/_default_template_yaml +++ b/lm_eval/tasks/tmmluplus/default/_tmmluplus_template_yaml @@ -16,4 +16,4 @@ metric_list: aggregation: mean higher_is_better: true metadata: - version: 1.0 + version: 2.0 diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus.yaml deleted file mode 100644 index 105cf98aff..0000000000 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: tmmluplus -task: -- tmmluplus_other -- tmmluplus_social_sciences -- tmmluplus_humanities -- tmmluplus_STEM diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_accounting.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_accounting.yaml index 9e27c919ef..c6ee4a5026 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_accounting.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_accounting.yaml @@ -1,7 +1,6 @@ "dataset_name": "accounting" "description": "以下為會計學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_accounting" "task_alias": "accounting" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_administrative_law.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_administrative_law.yaml index 454efec64f..3695783660 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_administrative_law.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_administrative_law.yaml @@ -1,7 +1,6 @@ "dataset_name": "administrative_law" "description": "以下為行政法的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "tmmluplus_humanities_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_administrative_law" "task_alias": "administrative law" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_advance_chemistry.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_advance_chemistry.yaml index d5baa64be2..549b3cc864 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_advance_chemistry.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_advance_chemistry.yaml @@ -1,7 +1,6 @@ "dataset_name": "advance_chemistry" "description": "以下為化學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_advance_chemistry" "task_alias": "advance chemistry" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_agriculture.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_agriculture.yaml index 340369c890..8ef2912690 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_agriculture.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_agriculture.yaml @@ -1,7 +1,6 @@ "dataset_name": "agriculture" "description": "以下為農業的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_agriculture" "task_alias": "agriculture" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_anti_money_laundering.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_anti_money_laundering.yaml index 95bb0e4786..7f8d873f49 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_anti_money_laundering.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_anti_money_laundering.yaml @@ -1,7 +1,6 @@ "dataset_name": "anti_money_laundering" "description": "以下為洗錢防制的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "tmmluplus_humanities_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_anti_money_laundering" "task_alias": "anti money laundering" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_auditing.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_auditing.yaml index a8168029b2..e5c598b7e3 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_auditing.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_auditing.yaml @@ -1,7 +1,6 @@ "dataset_name": "auditing" "description": "以下為審計學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_auditing" "task_alias": "auditing" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_basic_medical_science.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_basic_medical_science.yaml index d329b78a48..6c6db6aab8 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_basic_medical_science.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_basic_medical_science.yaml @@ -1,7 +1,6 @@ "dataset_name": "basic_medical_science" "description": "以下為基礎醫學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_basic_medical_science" "task_alias": "basic medical science" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_business_management.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_business_management.yaml index 9cacf04896..e8377ee3f5 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_business_management.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_business_management.yaml @@ -1,7 +1,6 @@ "dataset_name": "business_management" "description": "以下為企業管理的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_business_management" "task_alias": "business management" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_chinese_language_and_literature.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_chinese_language_and_literature.yaml index 73eba7684b..53a8d652ad 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_chinese_language_and_literature.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_chinese_language_and_literature.yaml @@ -1,7 +1,6 @@ "dataset_name": "chinese_language_and_literature" "description": "以下為國文的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_chinese_language_and_literature" "task_alias": "chinese language and literature" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_clinical_psychology.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_clinical_psychology.yaml index f8194feb7d..7753e2a096 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_clinical_psychology.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_clinical_psychology.yaml @@ -1,7 +1,6 @@ "dataset_name": "clinical_psychology" "description": "以下為臨床心理學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_clinical_psychology" "task_alias": "clinical psychology" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_computer_science.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_computer_science.yaml index c55f6a4a3a..00949ddc4d 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_computer_science.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_computer_science.yaml @@ -1,7 +1,6 @@ "dataset_name": "computer_science" "description": "以下為資訊工程的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_computer_science" "task_alias": "computer science" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_culinary_skills.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_culinary_skills.yaml index 457eac1d18..92f1109829 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_culinary_skills.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_culinary_skills.yaml @@ -1,7 +1,6 @@ "dataset_name": "culinary_skills" "description": "以下為餐旅的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_culinary_skills" "task_alias": "culinary skills" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_dentistry.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_dentistry.yaml index d6295240fc..836a9c129e 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_dentistry.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_dentistry.yaml @@ -1,7 +1,6 @@ "dataset_name": "dentistry" "description": "以下為牙醫學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_dentistry" "task_alias": "dentistry" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_economics.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_economics.yaml index 2ed100fb42..8665c62f87 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_economics.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_economics.yaml @@ -1,7 +1,6 @@ "dataset_name": "economics" "description": "以下為經濟學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_economics" "task_alias": "economics" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_education.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_education.yaml index cc60181000..46f230cdba 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_education.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_education.yaml @@ -1,7 +1,6 @@ "dataset_name": "education" "description": "以下為教育常識的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_education" "task_alias": "education" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_education_(profession_level).yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_education_(profession_level).yaml index f986517b66..281654c506 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_education_(profession_level).yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_education_(profession_level).yaml @@ -1,7 +1,6 @@ "dataset_name": "education_(profession_level)" "description": "以下為教育專業的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_education_(profession_level)" "task_alias": "education (profession level)" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_educational_psychology.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_educational_psychology.yaml index bed8f2d41b..be1c2c8a73 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_educational_psychology.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_educational_psychology.yaml @@ -1,7 +1,6 @@ "dataset_name": "educational_psychology" "description": "以下為教育心理的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_educational_psychology" "task_alias": "educational psychology" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_engineering_math.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_engineering_math.yaml index cf681cd57f..a8a35e1c74 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_engineering_math.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_engineering_math.yaml @@ -1,7 +1,6 @@ "dataset_name": "engineering_math" "description": "以下為工程數學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_engineering_math" "task_alias": "engineering math" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_finance_banking.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_finance_banking.yaml index e60086e12e..465c1d74d6 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_finance_banking.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_finance_banking.yaml @@ -1,7 +1,6 @@ "dataset_name": "finance_banking" "description": "以下為金融與法規的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_finance_banking" "task_alias": "finance banking" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_financial_analysis.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_financial_analysis.yaml index 9990ab5d04..647189c668 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_financial_analysis.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_financial_analysis.yaml @@ -1,7 +1,6 @@ "dataset_name": "financial_analysis" "description": "以下為財務分析的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_financial_analysis" "task_alias": "financial analysis" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_fire_science.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_fire_science.yaml index 807f6f5f2e..9b78539c6a 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_fire_science.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_fire_science.yaml @@ -1,7 +1,6 @@ "dataset_name": "fire_science" "description": "以下為火災學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_fire_science" "task_alias": "fire science" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_general_principles_of_law.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_general_principles_of_law.yaml index 30b21caead..8213106bf0 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_general_principles_of_law.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_general_principles_of_law.yaml @@ -1,7 +1,6 @@ "dataset_name": "general_principles_of_law" "description": "以下為法學大意的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "tmmluplus_humanities_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_general_principles_of_law" "task_alias": "general principles of law" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_geography_of_taiwan.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_geography_of_taiwan.yaml index 80ab36b73d..ae24c2e108 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_geography_of_taiwan.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_geography_of_taiwan.yaml @@ -1,7 +1,6 @@ "dataset_name": "geography_of_taiwan" "description": "以下為台灣地理的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_geography_of_taiwan" "task_alias": "geography of taiwan" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_human_behavior.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_human_behavior.yaml index 54aaa80fa3..bb2fa7231c 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_human_behavior.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_human_behavior.yaml @@ -1,7 +1,6 @@ "dataset_name": "human_behavior" "description": "以下為人類行為與社會的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_human_behavior" "task_alias": "human behavior" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_insurance_studies.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_insurance_studies.yaml index fa23be46c1..5d1abf7801 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_insurance_studies.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_insurance_studies.yaml @@ -1,7 +1,6 @@ "dataset_name": "insurance_studies" "description": "以下為保險學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_insurance_studies" "task_alias": "insurance studies" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_introduction_to_law.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_introduction_to_law.yaml index 6434d3eae8..ddbd488952 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_introduction_to_law.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_introduction_to_law.yaml @@ -1,7 +1,6 @@ "dataset_name": "introduction_to_law" "description": "以下為法律概論的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "tmmluplus_humanities_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_introduction_to_law" "task_alias": "introduction to law" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_jce_humanities.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_jce_humanities.yaml index 2ff3bed073..b70bcdd3db 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_jce_humanities.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_jce_humanities.yaml @@ -1,7 +1,6 @@ "dataset_name": "jce_humanities" "description": "以下為指考人文科目的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "tmmluplus_humanities_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_jce_humanities" "task_alias": "jce humanities" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chemistry.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chemistry.yaml index de9c0691cf..3c43b6b638 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chemistry.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chemistry.yaml @@ -1,7 +1,6 @@ "dataset_name": "junior_chemistry" "description": "以下為國中理化的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_junior_chemistry" "task_alias": "junior chemistry" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chinese_exam.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chinese_exam.yaml index 937090520e..ac68698b68 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chinese_exam.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chinese_exam.yaml @@ -1,7 +1,6 @@ "dataset_name": "junior_chinese_exam" "description": "以下為國中會考基測國文的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_junior_chinese_exam" "task_alias": "junior chinese exam" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_math_exam.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_math_exam.yaml index a869a55f14..bc72e5e096 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_math_exam.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_math_exam.yaml @@ -1,7 +1,6 @@ "dataset_name": "junior_math_exam" "description": "以下為國中會考基測數學科的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_junior_math_exam" "task_alias": "junior math exam" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_science_exam.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_science_exam.yaml index ed4b8d272c..740e674e6c 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_science_exam.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_science_exam.yaml @@ -1,7 +1,6 @@ "dataset_name": "junior_science_exam" "description": "以下為國中會考基測自然科的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_junior_science_exam" "task_alias": "junior science exam" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_social_studies.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_social_studies.yaml index 760ff0e794..54472b09e7 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_social_studies.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_social_studies.yaml @@ -1,7 +1,6 @@ "dataset_name": "junior_social_studies" "description": "以下為國中會考基測社會科的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_junior_social_studies" "task_alias": "junior social studies" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_linear_algebra.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_linear_algebra.yaml index c2286dbca9..762bdfc0da 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_linear_algebra.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_linear_algebra.yaml @@ -1,7 +1,6 @@ "dataset_name": "linear_algebra" "description": "以下為線代的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_linear_algebra" "task_alias": "linear algebra" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_logic_reasoning.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_logic_reasoning.yaml index 059bef3d96..58fdc83f02 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_logic_reasoning.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_logic_reasoning.yaml @@ -1,7 +1,6 @@ "dataset_name": "logic_reasoning" "description": "以下為邏輯思維的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_logic_reasoning" "task_alias": "logic reasoning" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_macroeconomics.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_macroeconomics.yaml index 91009abe69..81984475d3 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_macroeconomics.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_macroeconomics.yaml @@ -1,7 +1,6 @@ "dataset_name": "macroeconomics" "description": "以下為總經的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_macroeconomics" "task_alias": "macroeconomics" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_management_accounting.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_management_accounting.yaml index 2d071f8697..fd77179d65 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_management_accounting.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_management_accounting.yaml @@ -1,7 +1,6 @@ "dataset_name": "management_accounting" "description": "以下為管理會計的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_management_accounting" "task_alias": "management accounting" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_marketing_management.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_marketing_management.yaml index da39f0a879..f903c4b6f0 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_marketing_management.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_marketing_management.yaml @@ -1,7 +1,6 @@ "dataset_name": "marketing_management" "description": "以下為行銷管理的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_marketing_management" "task_alias": "marketing management" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_mechanical.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_mechanical.yaml index 81ea0dce68..5300cc1d91 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_mechanical.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_mechanical.yaml @@ -1,7 +1,6 @@ "dataset_name": "mechanical" "description": "以下為機械與機電概論的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_mechanical" "task_alias": "mechanical" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_music.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_music.yaml index 72864c0035..b0e62badf6 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_music.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_music.yaml @@ -1,7 +1,6 @@ "dataset_name": "music" "description": "以下為音樂科的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_music" "task_alias": "music" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_national_protection.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_national_protection.yaml index 62e98266d8..4ae2953b1f 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_national_protection.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_national_protection.yaml @@ -1,7 +1,6 @@ "dataset_name": "national_protection" "description": "以下為軍事的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_national_protection" "task_alias": "national protection" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_nautical_science.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_nautical_science.yaml index e02c1dbc2a..1be4cf1d6f 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_nautical_science.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_nautical_science.yaml @@ -1,7 +1,6 @@ "dataset_name": "nautical_science" "description": "以下為航海的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_nautical_science" "task_alias": "nautical science" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_occupational_therapy_for_psychological_disorders.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_occupational_therapy_for_psychological_disorders.yaml index ba2bfa827e..5f6f33956e 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_occupational_therapy_for_psychological_disorders.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_occupational_therapy_for_psychological_disorders.yaml @@ -1,7 +1,6 @@ "dataset_name": "occupational_therapy_for_psychological_disorders" "description": "以下為心理障礙職能治療學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_occupational_therapy_for_psychological_disorders" "task_alias": "occupational therapy for psychological disorders" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_official_document_management.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_official_document_management.yaml index 77dce1d671..16617d2f54 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_official_document_management.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_official_document_management.yaml @@ -1,7 +1,6 @@ "dataset_name": "official_document_management" "description": "以下為機關文書的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_official_document_management" "task_alias": "official document management" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_optometry.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_optometry.yaml index 7e3b78b7ed..ccec870859 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_optometry.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_optometry.yaml @@ -1,7 +1,6 @@ "dataset_name": "optometry" "description": "以下為視光學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_optometry" "task_alias": "optometry" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_organic_chemistry.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_organic_chemistry.yaml index 75a096f9b4..8f1a8cce0c 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_organic_chemistry.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_organic_chemistry.yaml @@ -1,7 +1,6 @@ "dataset_name": "organic_chemistry" "description": "以下為有機化學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_organic_chemistry" "task_alias": "organic chemistry" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacology.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacology.yaml index 0da7ec9842..f42435e585 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacology.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacology.yaml @@ -1,7 +1,6 @@ "dataset_name": "pharmacology" "description": "以下為藥理學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_pharmacology" "task_alias": "pharmacology" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacy.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacy.yaml index a87aa4be10..4cb8e83f22 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacy.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacy.yaml @@ -1,7 +1,6 @@ "dataset_name": "pharmacy" "description": "以下為藥劑學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_pharmacy" "task_alias": "pharmacy" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_physical_education.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_physical_education.yaml index fb3558e9ba..af762327bb 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_physical_education.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_physical_education.yaml @@ -1,7 +1,6 @@ "dataset_name": "physical_education" "description": "以下為體育的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_physical_education" "task_alias": "physical education" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_physics.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_physics.yaml index 3a8443fe19..c9d4e16b34 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_physics.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_physics.yaml @@ -1,7 +1,6 @@ "dataset_name": "physics" "description": "以下為物理的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_physics" "task_alias": "physics" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_politic_science.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_politic_science.yaml index 67d2068c5d..9ebbdb242b 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_politic_science.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_politic_science.yaml @@ -1,7 +1,6 @@ "dataset_name": "politic_science" "description": "以下為政治的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_politic_science" "task_alias": "politic science" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_real_estate.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_real_estate.yaml index ba90b7aa56..971557b531 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_real_estate.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_real_estate.yaml @@ -1,7 +1,6 @@ "dataset_name": "real_estate" "description": "以下為房地產的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_real_estate" "task_alias": "real estate" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_secondary_physics.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_secondary_physics.yaml index 6865167cb1..1660fc6d5b 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_secondary_physics.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_secondary_physics.yaml @@ -1,7 +1,6 @@ "dataset_name": "secondary_physics" "description": "以下為高中物理的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_secondary_physics" "task_alias": "secondary physics" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_statistics_and_machine_learning.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_statistics_and_machine_learning.yaml index 9a6e68ba51..252496cf63 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_statistics_and_machine_learning.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_statistics_and_machine_learning.yaml @@ -1,7 +1,6 @@ "dataset_name": "statistics_and_machine_learning" "description": "以下為統計與機器學習的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_statistics_and_machine_learning" "task_alias": "statistics and machine learning" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_taiwanese_hokkien.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_taiwanese_hokkien.yaml index 89297df315..2977a1674a 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_taiwanese_hokkien.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_taiwanese_hokkien.yaml @@ -1,7 +1,6 @@ "dataset_name": "taiwanese_hokkien" "description": "以下為閩南語的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_taiwanese_hokkien" "task_alias": "taiwanese hokkien" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_taxation.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_taxation.yaml index f54520270f..c382b8c840 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_taxation.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_taxation.yaml @@ -1,7 +1,6 @@ "dataset_name": "taxation" "description": "以下為稅務的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "tmmluplus_humanities_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_taxation" "task_alias": "taxation" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_technical.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_technical.yaml index 6167a8fe0f..b4621a4e49 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_technical.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_technical.yaml @@ -1,7 +1,6 @@ "dataset_name": "technical" "description": "以下為技術工相關的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_technical" "task_alias": "technical" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_three_principles_of_people.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_three_principles_of_people.yaml index de50db700b..83a3628ddf 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_three_principles_of_people.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_three_principles_of_people.yaml @@ -1,7 +1,6 @@ "dataset_name": "three_principles_of_people" "description": "以下為三民主義的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_three_principles_of_people" "task_alias": "three principles of people" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_trade.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_trade.yaml index 1c42e22e03..7eacf4dca0 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_trade.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_trade.yaml @@ -1,7 +1,6 @@ "dataset_name": "trade" "description": "以下為貿易的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_trade" "task_alias": "trade" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_traditional_chinese_medicine_clinical_medicine.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_traditional_chinese_medicine_clinical_medicine.yaml index b5a3fdf197..7d626dd26a 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_traditional_chinese_medicine_clinical_medicine.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_traditional_chinese_medicine_clinical_medicine.yaml @@ -1,7 +1,6 @@ "dataset_name": "traditional_chinese_medicine_clinical_medicine" "description": "以下為中醫臨床醫學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_traditional_chinese_medicine_clinical_medicine" "task_alias": "traditional chinese medicine clinical medicine" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_trust_practice.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_trust_practice.yaml index 0c1e7ae3f4..f4b3a010fc 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_trust_practice.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_trust_practice.yaml @@ -1,7 +1,6 @@ "dataset_name": "trust_practice" "description": "以下為信託實務的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "tmmluplus_humanities_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_trust_practice" "task_alias": "trust practice" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_ttqav2.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_ttqav2.yaml index 2216d4d46c..95a1041180 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_ttqav2.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_ttqav2.yaml @@ -1,7 +1,6 @@ "dataset_name": "ttqav2" "description": "以下為台灣在地用語的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_ttqav2" "task_alias": "ttqav2" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_chinese_language.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_chinese_language.yaml index 0c98fedd37..a86600882a 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_chinese_language.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_chinese_language.yaml @@ -1,7 +1,6 @@ "dataset_name": "tve_chinese_language" "description": "以下為統測國文的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_tve_chinese_language" "task_alias": "tve chinese language" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_design.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_design.yaml index e36b154802..01a27149e5 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_design.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_design.yaml @@ -1,7 +1,6 @@ "dataset_name": "tve_design" "description": "以下為統測 設計的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_tve_design" "task_alias": "tve design" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_mathematics.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_mathematics.yaml index b4158f17e9..6240db295f 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_mathematics.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_mathematics.yaml @@ -1,7 +1,6 @@ "dataset_name": "tve_mathematics" "description": "以下為統測數學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_tve_mathematics" "task_alias": "tve mathematics" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_natural_sciences.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_natural_sciences.yaml index c2ecf4ad00..833c47fc85 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_natural_sciences.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_natural_sciences.yaml @@ -1,7 +1,6 @@ "dataset_name": "tve_natural_sciences" "description": "以下為統測自然科的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_tve_natural_sciences" "task_alias": "tve natural sciences" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pathology.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pathology.yaml index c1fcb43897..c5a8edcb4e 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pathology.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pathology.yaml @@ -1,7 +1,6 @@ "dataset_name": "veterinary_pathology" "description": "以下為獸醫病理學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_veterinary_pathology" "task_alias": "veterinary pathology" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pharmacology.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pharmacology.yaml index 45c6553b29..b75b1f0075 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pharmacology.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pharmacology.yaml @@ -1,7 +1,6 @@ "dataset_name": "veterinary_pharmacology" "description": "以下為獸醫藥理學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_veterinary_pharmacology" "task_alias": "veterinary pharmacology" diff --git a/pyproject.toml b/pyproject.toml index 003197756a..1bc0944ecf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "lm_eval" -version = "0.4.3" +version = "0.4.4" authors = [ {name="EleutherAI", email="contact@eleuther.ai"} ] From decc533d02222f3b866d9a89263277fe0cc2fcb2 Mon Sep 17 00:00:00 2001 From: Malikeh Ehghaghi <35193114+Malikeh97@users.noreply.github.com> Date: Tue, 10 Sep 2024 17:28:47 -0400 Subject: [PATCH 5/6] Add Open Arabic LLM Leaderboard Benchmarks (Full and Light Version) (#2232) * arabic leaferboard yaml file is added * arabic toxigen is implemented * Dataset library is imported * arabic sciq is added * util file of arabic toxigen is updated * arabic race is added * arabic piqa is implemented * arabic open qa is added * arabic copa is implemented * arabic boolq ia added * arabic arc easy is added * arabic arc challenge is added * arabic exams benchmark is implemented * arabic hellaswag is added * arabic leaderboard yaml file metrics are updated * arabic mmlu benchmarks are added * arabic mmlu group yaml file is updated * alghafa benchmarks are added * acva benchmarks are added * acva utils.py is updated * light version of arabic leaderboard benchmarks are added * bugs fixed * bug fixed * bug fixed * bug fixed * bug fixed * bug fixed * library import bug is fixed * doc to target updated * bash file is deleted * results folder is deleted * leaderboard groups are added * full arabic leaderboard groups are added, plus some bug fixes to the light version * Create README.md README.md for arabic_leaderboard_complete * Create README.md README.md for arabic_leaderboard_light * Delete lm_eval/tasks/arabic_leaderboard directory * Update README.md * Update README.md adding the Arabic leaderboards to the library * Update README.md 10% of the training set * Update README.md 10% of the training set * revert .gitignore to prev version * Update lm_eval/tasks/README.md Co-authored-by: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com> * updated main README.md * Update lm_eval/tasks/README.md * specify machine translated benchmarks (complete) * specify machine translated benchmarks (light version) * add alghafa to the related task names (complete and light) * add 'acva' to the related task names (complete and light) * add 'arabic_leaderboard' to all the groups (complete and light) * all dataset - not a random sample * added more accurate details to the readme file * added mt_mmlu from okapi * Update lm_eval/tasks/README.md Co-authored-by: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com> * Update lm_eval/tasks/README.md Co-authored-by: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com> * updated mt_mmlu readme * renaming 'alghafa' full and light * renaming 'arabic_mmlu' light and full * renaming 'acva' full and light * update readme and standardize dir/file names * running pre-commit --------- Co-authored-by: shahrzads Co-authored-by: shahrzads <56282669+shahrzads@users.noreply.github.com> Co-authored-by: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com> --- lm_eval/tasks/README.md | 2 + .../arabic_leaderboard_complete/README.md | 254 ++++++++++++++++++ .../arabic_leaderboard_alghafa.yaml | 23 ++ ...leaderboard_alghafa_mcq_exams_test_ar.yaml | 23 ++ ..._leaderboard_alghafa_meta_ar_dialects.yaml | 23 ++ ...rabic_leaderboard_alghafa_meta_ar_msa.yaml | 23 ++ ..._choice_facts_truefalse_balanced_task.yaml | 23 ++ ..._choice_grounded_statement_soqal_task.yaml | 23 ++ ...ce_grounded_statement_xglue_mlqa_task.yaml | 23 ++ ...oice_rating_sentiment_no_neutral_task.yaml | 23 ++ ...multiple_choice_rating_sentiment_task.yaml | 23 ++ ...lghafa_multiple_choice_sentiment_task.yaml | 23 ++ .../arabic_leaderboard_alghafa/utils.py | 23 ++ .../arabic_exams.yaml | 23 ++ .../arabic_leaderboard_arabic_exams.yaml | 13 + .../arabic_leaderboard_arabic_exams/utils.py | 33 +++ .../arabic_leaderboard_arabic_mmlu.yaml | 68 +++++ ...derboard_arabic_mmlu_abstract_algebra.yaml | 23 ++ ...rabic_leaderboard_arabic_mmlu_anatomy.yaml | 23 ++ ...bic_leaderboard_arabic_mmlu_astronomy.yaml | 23 ++ ...aderboard_arabic_mmlu_business_ethics.yaml | 23 ++ ...rboard_arabic_mmlu_clinical_knowledge.yaml | 23 ++ ...aderboard_arabic_mmlu_college_biology.yaml | 23 ++ ...erboard_arabic_mmlu_college_chemistry.yaml | 23 ++ ..._arabic_mmlu_college_computer_science.yaml | 23 ++ ...board_arabic_mmlu_college_mathematics.yaml | 23 ++ ...derboard_arabic_mmlu_college_medicine.yaml | 23 ++ ...aderboard_arabic_mmlu_college_physics.yaml | 23 ++ ...erboard_arabic_mmlu_computer_security.yaml | 23 ++ ...rboard_arabic_mmlu_conceptual_physics.yaml | 23 ++ ..._leaderboard_arabic_mmlu_econometrics.yaml | 23 ++ ...rd_arabic_mmlu_electrical_engineering.yaml | 23 ++ ...rd_arabic_mmlu_elementary_mathematics.yaml | 23 ++ ..._leaderboard_arabic_mmlu_formal_logic.yaml | 23 ++ ..._leaderboard_arabic_mmlu_global_facts.yaml | 23 ++ ...board_arabic_mmlu_high_school_biology.yaml | 23 ++ ...ard_arabic_mmlu_high_school_chemistry.yaml | 23 ++ ...bic_mmlu_high_school_computer_science.yaml | 23 ++ ...bic_mmlu_high_school_european_history.yaml | 23 ++ ...ard_arabic_mmlu_high_school_geography.yaml | 23 ++ ...u_high_school_government_and_politics.yaml | 23 ++ ...rabic_mmlu_high_school_macroeconomics.yaml | 23 ++ ...d_arabic_mmlu_high_school_mathematics.yaml | 23 ++ ...rabic_mmlu_high_school_microeconomics.yaml | 23 ++ ...board_arabic_mmlu_high_school_physics.yaml | 23 ++ ...rd_arabic_mmlu_high_school_psychology.yaml | 23 ++ ...rd_arabic_mmlu_high_school_statistics.yaml | 23 ++ ...rd_arabic_mmlu_high_school_us_history.yaml | 23 ++ ...arabic_mmlu_high_school_world_history.yaml | 23 ++ ...c_leaderboard_arabic_mmlu_human_aging.yaml | 23 ++ ...aderboard_arabic_mmlu_human_sexuality.yaml | 23 ++ ...erboard_arabic_mmlu_international_law.yaml | 23 ++ ...leaderboard_arabic_mmlu_jurisprudence.yaml | 23 ++ ...erboard_arabic_mmlu_logical_fallacies.yaml | 23 ++ ...derboard_arabic_mmlu_machine_learning.yaml | 23 ++ ...ic_leaderboard_arabic_mmlu_management.yaml | 23 ++ ...bic_leaderboard_arabic_mmlu_marketing.yaml | 23 ++ ...derboard_arabic_mmlu_medical_genetics.yaml | 23 ++ ...leaderboard_arabic_mmlu_miscellaneous.yaml | 23 ++ ...eaderboard_arabic_mmlu_moral_disputes.yaml | 23 ++ ...aderboard_arabic_mmlu_moral_scenarios.yaml | 23 ++ ...bic_leaderboard_arabic_mmlu_nutrition.yaml | 23 ++ ...ic_leaderboard_arabic_mmlu_philosophy.yaml | 23 ++ ...ic_leaderboard_arabic_mmlu_prehistory.yaml | 23 ++ ...d_arabic_mmlu_professional_accounting.yaml | 23 ++ ...derboard_arabic_mmlu_professional_law.yaml | 23 ++ ...ard_arabic_mmlu_professional_medicine.yaml | 23 ++ ...d_arabic_mmlu_professional_psychology.yaml | 23 ++ ...derboard_arabic_mmlu_public_relations.yaml | 23 ++ ...derboard_arabic_mmlu_security_studies.yaml | 23 ++ ...bic_leaderboard_arabic_mmlu_sociology.yaml | 23 ++ ...erboard_arabic_mmlu_us_foreign_policy.yaml | 23 ++ ...abic_leaderboard_arabic_mmlu_virology.yaml | 23 ++ ...aderboard_arabic_mmlu_world_religions.yaml | 23 ++ .../arabic_leaderboard_arabic_mmlu/utils.py | 35 +++ ...c_leaderboard_arabic_mt_arc_challenge.yaml | 13 + .../arabic_mt_arc_challenge.yaml | 23 ++ .../utils.py | 23 ++ ...arabic_leaderboard_arabic_mt_arc_easy.yaml | 13 + .../arabic_mt_arc_easy.yaml | 23 ++ .../utils.py | 23 ++ .../arabic_leaderboard_arabic_mt_boolq.yaml | 13 + .../arabic_mt_boolq.yaml | 23 ++ .../utils.py | 24 ++ .../arabic_leaderboard_arabic_mt_copa.yaml | 13 + .../arabic_mt_copa.yaml | 23 ++ .../utils.py | 19 ++ ...rabic_leaderboard_arabic_mt_hellaswag.yaml | 13 + .../arabic_mt_hellaswag.yaml | 23 ++ .../utils.py | 30 +++ .../arabic_leaderboard_arabic_mt_mmlu.yaml | 13 + .../arabic_mt_mmlu.yaml | 23 ++ .../utils.py | 23 ++ ...bic_leaderboard_arabic_mt_openbook_qa.yaml | 13 + .../arabic_mt_openbook_qa.yaml | 23 ++ .../utils.py | 23 ++ .../arabic_leaderboard_arabic_mt_piqa.yaml | 13 + .../arabic_mt_piqa.yaml | 23 ++ .../utils.py | 23 ++ .../arabic_leaderboard_arabic_mt_race.yaml | 13 + .../arabic_mt_race.yaml | 23 ++ .../utils.py | 23 ++ .../arabic_leaderboard_arabic_mt_sciq.yaml | 13 + .../arabic_mt_sciq.yaml | 23 ++ .../utils.py | 41 +++ .../arabic_leaderboard_arabic_mt_toxigen.yaml | 13 + .../arabic_mt_toxigen.yaml | 23 ++ .../utils.py | 23 ++ .../arabic_leaderboard_acva.yaml | 70 +++++ .../arabic_leaderboard_acva_Algeria.yaml | 23 ++ ...arabic_leaderboard_acva_Ancient_Egypt.yaml | 23 ++ .../arabic_leaderboard_acva_Arab_Empire.yaml | 23 ++ ..._leaderboard_acva_Arabic_Architecture.yaml | 23 ++ .../arabic_leaderboard_acva_Arabic_Art.yaml | 23 ++ ...bic_leaderboard_acva_Arabic_Astronomy.yaml | 23 ++ ...c_leaderboard_acva_Arabic_Calligraphy.yaml | 23 ++ ...abic_leaderboard_acva_Arabic_Ceremony.yaml | 23 ++ ...abic_leaderboard_acva_Arabic_Clothing.yaml | 23 ++ ...rabic_leaderboard_acva_Arabic_Culture.yaml | 23 ++ .../arabic_leaderboard_acva_Arabic_Food.yaml | 23 ++ ...rabic_leaderboard_acva_Arabic_Funeral.yaml | 23 ++ ...bic_leaderboard_acva_Arabic_Geography.yaml | 23 ++ ...rabic_leaderboard_acva_Arabic_History.yaml | 23 ++ ...aderboard_acva_Arabic_Language_Origin.yaml | 23 ++ ...ic_leaderboard_acva_Arabic_Literature.yaml | 23 ++ .../arabic_leaderboard_acva_Arabic_Math.yaml | 23 ++ ...abic_leaderboard_acva_Arabic_Medicine.yaml | 23 ++ .../arabic_leaderboard_acva_Arabic_Music.yaml | 23 ++ ...abic_leaderboard_acva_Arabic_Ornament.yaml | 23 ++ ...ic_leaderboard_acva_Arabic_Philosophy.yaml | 23 ++ ...ard_acva_Arabic_Physics_and_Chemistry.yaml | 23 ++ ...rabic_leaderboard_acva_Arabic_Wedding.yaml | 23 ++ .../arabic_leaderboard_acva_Bahrain.yaml | 23 ++ .../arabic_leaderboard_acva_Comoros.yaml | 23 ++ .../arabic_leaderboard_acva_Egypt_modern.yaml | 23 ++ ...rboard_acva_InfluenceFromAncientEgypt.yaml | 23 ++ ...aderboard_acva_InfluenceFromByzantium.yaml | 23 ++ ...c_leaderboard_acva_InfluenceFromChina.yaml | 23 ++ ..._leaderboard_acva_InfluenceFromGreece.yaml | 23 ++ ...c_leaderboard_acva_InfluenceFromIslam.yaml | 23 ++ ..._leaderboard_acva_InfluenceFromPersia.yaml | 23 ++ ...ic_leaderboard_acva_InfluenceFromRome.yaml | 23 ++ .../arabic_leaderboard_acva_Iraq.yaml | 23 ++ ...abic_leaderboard_acva_Islam_Education.yaml | 23 ++ ...board_acva_Islam_branches_and_schools.yaml | 23 ++ ...c_leaderboard_acva_Islamic_law_system.yaml | 23 ++ .../arabic_leaderboard_acva_Jordan.yaml | 23 ++ .../arabic_leaderboard_acva_Kuwait.yaml | 23 ++ .../arabic_leaderboard_acva_Lebanon.yaml | 23 ++ .../arabic_leaderboard_acva_Libya.yaml | 23 ++ .../arabic_leaderboard_acva_Mauritania.yaml | 23 ++ ...erboard_acva_Mesopotamia_civilization.yaml | 23 ++ .../arabic_leaderboard_acva_Morocco.yaml | 23 ++ .../arabic_leaderboard_acva_Oman.yaml | 23 ++ .../arabic_leaderboard_acva_Palestine.yaml | 23 ++ .../arabic_leaderboard_acva_Qatar.yaml | 23 ++ .../arabic_leaderboard_acva_Saudi_Arabia.yaml | 23 ++ .../arabic_leaderboard_acva_Somalia.yaml | 23 ++ .../arabic_leaderboard_acva_Sudan.yaml | 23 ++ .../arabic_leaderboard_acva_Syria.yaml | 23 ++ .../arabic_leaderboard_acva_Tunisia.yaml | 23 ++ ...leaderboard_acva_United_Arab_Emirates.yaml | 23 ++ .../arabic_leaderboard_acva_Yemen.yaml | 23 ++ ...arabic_leaderboard_acva_communication.yaml | 23 ++ ...c_leaderboard_acva_computer_and_phone.yaml | 23 ++ .../arabic_leaderboard_acva_daily_life.yaml | 23 ++ ...arabic_leaderboard_acva_entertainment.yaml | 23 ++ .../arabic_leaderboard_avca/utils.py | 16 ++ .../arabic_leaderboard_complete.yaml | 25 ++ .../tasks/arabic_leaderboard_light/README.md | 20 ++ .../arabic_leaderboard_alghafa_light.yaml | 23 ++ ...board_alghafa_mcq_exams_test_ar_light.yaml | 23 ++ ...rboard_alghafa_meta_ar_dialects_light.yaml | 23 ++ ...leaderboard_alghafa_meta_ar_msa_light.yaml | 23 ++ ...e_facts_truefalse_balanced_task_light.yaml | 23 ++ ...e_grounded_statement_soqal_task_light.yaml | 23 ++ ...unded_statement_xglue_mlqa_task_light.yaml | 23 ++ ...ating_sentiment_no_neutral_task_light.yaml | 23 ++ ...le_choice_rating_sentiment_task_light.yaml | 23 ++ ..._multiple_choice_sentiment_task_light.yaml | 23 ++ .../arabic_leaderboard_alghafa_light/utils.py | 23 ++ .../arabic_exams_light.yaml | 23 ++ ...arabic_leaderboard_arabic_exams_light.yaml | 13 + .../utils.py | 33 +++ ...rd_arabic_mmlu_abstract_algebra_light.yaml | 23 ++ ...leaderboard_arabic_mmlu_anatomy_light.yaml | 23 ++ ...aderboard_arabic_mmlu_astronomy_light.yaml | 23 ++ ...ard_arabic_mmlu_business_ethics_light.yaml | 23 ++ ..._arabic_mmlu_clinical_knowledge_light.yaml | 23 ++ ...ard_arabic_mmlu_college_biology_light.yaml | 23 ++ ...d_arabic_mmlu_college_chemistry_light.yaml | 23 ++ ...c_mmlu_college_computer_science_light.yaml | 23 ++ ...arabic_mmlu_college_mathematics_light.yaml | 23 ++ ...rd_arabic_mmlu_college_medicine_light.yaml | 23 ++ ...ard_arabic_mmlu_college_physics_light.yaml | 23 ++ ...d_arabic_mmlu_computer_security_light.yaml | 23 ++ ..._arabic_mmlu_conceptual_physics_light.yaml | 23 ++ ...rboard_arabic_mmlu_econometrics_light.yaml | 23 ++ ...bic_mmlu_electrical_engineering_light.yaml | 23 ++ ...bic_mmlu_elementary_mathematics_light.yaml | 23 ++ ...rboard_arabic_mmlu_formal_logic_light.yaml | 23 ++ ...rboard_arabic_mmlu_global_facts_light.yaml | 23 ++ ...arabic_mmlu_high_school_biology_light.yaml | 23 ++ ...abic_mmlu_high_school_chemistry_light.yaml | 23 ++ ...lu_high_school_computer_science_light.yaml | 23 ++ ...lu_high_school_european_history_light.yaml | 23 ++ ...abic_mmlu_high_school_geography_light.yaml | 23 ++ ..._school_government_and_politics_light.yaml | 23 ++ ...mmlu_high_school_macroeconomics_light.yaml | 23 ++ ...ic_mmlu_high_school_mathematics_light.yaml | 23 ++ ...mmlu_high_school_microeconomics_light.yaml | 23 ++ ...arabic_mmlu_high_school_physics_light.yaml | 23 ++ ...bic_mmlu_high_school_psychology_light.yaml | 23 ++ ...bic_mmlu_high_school_statistics_light.yaml | 23 ++ ...bic_mmlu_high_school_us_history_light.yaml | 23 ++ ..._mmlu_high_school_world_history_light.yaml | 23 ++ ...erboard_arabic_mmlu_human_aging_light.yaml | 23 ++ ...ard_arabic_mmlu_human_sexuality_light.yaml | 23 ++ ...d_arabic_mmlu_international_law_light.yaml | 23 ++ ...board_arabic_mmlu_jurisprudence_light.yaml | 23 ++ .../arabic_leaderboard_arabic_mmlu_light.yaml | 68 +++++ ...d_arabic_mmlu_logical_fallacies_light.yaml | 23 ++ ...rd_arabic_mmlu_machine_learning_light.yaml | 23 ++ ...derboard_arabic_mmlu_management_light.yaml | 23 ++ ...aderboard_arabic_mmlu_marketing_light.yaml | 23 ++ ...rd_arabic_mmlu_medical_genetics_light.yaml | 23 ++ ...board_arabic_mmlu_miscellaneous_light.yaml | 23 ++ ...oard_arabic_mmlu_moral_disputes_light.yaml | 23 ++ ...ard_arabic_mmlu_moral_scenarios_light.yaml | 23 ++ ...aderboard_arabic_mmlu_nutrition_light.yaml | 23 ++ ...derboard_arabic_mmlu_philosophy_light.yaml | 23 ++ ...derboard_arabic_mmlu_prehistory_light.yaml | 23 ++ ...ic_mmlu_professional_accounting_light.yaml | 23 ++ ...rd_arabic_mmlu_professional_law_light.yaml | 23 ++ ...abic_mmlu_professional_medicine_light.yaml | 23 ++ ...ic_mmlu_professional_psychology_light.yaml | 23 ++ ...rd_arabic_mmlu_public_relations_light.yaml | 23 ++ ...rd_arabic_mmlu_security_studies_light.yaml | 23 ++ ...aderboard_arabic_mmlu_sociology_light.yaml | 23 ++ ...d_arabic_mmlu_us_foreign_policy_light.yaml | 23 ++ ...eaderboard_arabic_mmlu_virology_light.yaml | 23 ++ ...ard_arabic_mmlu_world_religions_light.yaml | 23 ++ .../utils.py | 35 +++ ...erboard_arabic_mt_arc_challenge_light.yaml | 13 + .../arabic_mt_arc_challenge_light.yaml | 23 ++ .../utils.py | 23 ++ ..._leaderboard_arabic_mt_arc_easy_light.yaml | 13 + .../arabic_mt_arc_easy_light.yaml | 23 ++ .../utils.py | 23 ++ ...bic_leaderboard_arabic_mt_boolq_light.yaml | 13 + .../arabic_mt_boolq_light.yaml | 23 ++ .../utils.py | 24 ++ .../arabic_mt_copa_light.yaml | 23 ++ ...rbic_leaderboard_arabic_mt_copa_light.yaml | 13 + .../utils.py | 19 ++ ...leaderboard_arabic_mt_hellaswag_light.yaml | 13 + .../arabic_mt_hellaswag_light.yaml | 23 ++ .../utils.py | 30 +++ ...abic_leaderboard_arabic_mt_mmlu_light.yaml | 13 + .../arabic_mt_mmlu_light.yaml | 23 ++ .../utils.py | 23 ++ ...aderboard_arabic_mt_openbook_qa_light.yaml | 13 + .../arabic_mt_openbook_qa_light.yaml | 23 ++ .../utils.py | 23 ++ ...abic_leaderboard_arabic_mt_piqa_light.yaml | 13 + .../arabic_mt_piqa_light.yaml | 23 ++ .../utils.py | 23 ++ ...abic_leaderboard_arabic_mt_race_light.yaml | 13 + .../arabic_mt_race_light.yaml | 23 ++ .../utils.py | 23 ++ ...abic_leaderboard_arabic_mt_sciq_light.yaml | 13 + .../arabic_mt_sciq_light.yaml | 23 ++ .../utils.py | 41 +++ ...c_leaderboard_arabic_mt_toxigen_light.yaml | 13 + .../arabic_mt_toxigen_light.yaml | 23 ++ .../utils.py | 23 ++ ...arabic_leaderboard_acva_Algeria_light.yaml | 23 ++ ..._leaderboard_acva_Ancient_Egypt_light.yaml | 23 ++ ...ic_leaderboard_acva_Arab_Empire_light.yaml | 23 ++ ...rboard_acva_Arabic_Architecture_light.yaml | 23 ++ ...bic_leaderboard_acva_Arabic_Art_light.yaml | 23 ++ ...aderboard_acva_Arabic_Astronomy_light.yaml | 23 ++ ...erboard_acva_Arabic_Calligraphy_light.yaml | 23 ++ ...eaderboard_acva_Arabic_Ceremony_light.yaml | 23 ++ ...eaderboard_acva_Arabic_Clothing_light.yaml | 23 ++ ...leaderboard_acva_Arabic_Culture_light.yaml | 23 ++ ...ic_leaderboard_acva_Arabic_Food_light.yaml | 23 ++ ...leaderboard_acva_Arabic_Funeral_light.yaml | 23 ++ ...aderboard_acva_Arabic_Geography_light.yaml | 23 ++ ...leaderboard_acva_Arabic_History_light.yaml | 23 ++ ...ard_acva_Arabic_Language_Origin_light.yaml | 23 ++ ...derboard_acva_Arabic_Literature_light.yaml | 23 ++ ...ic_leaderboard_acva_Arabic_Math_light.yaml | 23 ++ ...eaderboard_acva_Arabic_Medicine_light.yaml | 23 ++ ...c_leaderboard_acva_Arabic_Music_light.yaml | 23 ++ ...eaderboard_acva_Arabic_Ornament_light.yaml | 23 ++ ...derboard_acva_Arabic_Philosophy_light.yaml | 23 ++ ...va_Arabic_Physics_and_Chemistry_light.yaml | 23 ++ ...leaderboard_acva_Arabic_Wedding_light.yaml | 23 ++ ...arabic_leaderboard_acva_Bahrain_light.yaml | 23 ++ ...arabic_leaderboard_acva_Comoros_light.yaml | 23 ++ ...c_leaderboard_acva_Egypt_modern_light.yaml | 23 ++ ..._acva_InfluenceFromAncientEgypt_light.yaml | 23 ++ ...ard_acva_InfluenceFromByzantium_light.yaml | 23 ++ ...erboard_acva_InfluenceFromChina_light.yaml | 23 ++ ...rboard_acva_InfluenceFromGreece_light.yaml | 23 ++ ...erboard_acva_InfluenceFromIslam_light.yaml | 23 ++ ...rboard_acva_InfluenceFromPersia_light.yaml | 23 ++ ...derboard_acva_InfluenceFromRome_light.yaml | 23 ++ .../arabic_leaderboard_acva_Iraq_light.yaml | 23 ++ ...eaderboard_acva_Islam_Education_light.yaml | 23 ++ ...acva_Islam_branches_and_schools_light.yaml | 23 ++ ...erboard_acva_Islamic_law_system_light.yaml | 23 ++ .../arabic_leaderboard_acva_Jordan_light.yaml | 23 ++ .../arabic_leaderboard_acva_Kuwait_light.yaml | 23 ++ ...arabic_leaderboard_acva_Lebanon_light.yaml | 23 ++ .../arabic_leaderboard_acva_Libya_light.yaml | 23 ++ ...bic_leaderboard_acva_Mauritania_light.yaml | 23 ++ ...d_acva_Mesopotamia_civilization_light.yaml | 23 ++ ...arabic_leaderboard_acva_Morocco_light.yaml | 23 ++ .../arabic_leaderboard_acva_Oman_light.yaml | 23 ++ ...abic_leaderboard_acva_Palestine_light.yaml | 23 ++ .../arabic_leaderboard_acva_Qatar_light.yaml | 23 ++ ...c_leaderboard_acva_Saudi_Arabia_light.yaml | 23 ++ ...arabic_leaderboard_acva_Somalia_light.yaml | 23 ++ .../arabic_leaderboard_acva_Sudan_light.yaml | 23 ++ .../arabic_leaderboard_acva_Syria_light.yaml | 23 ++ ...arabic_leaderboard_acva_Tunisia_light.yaml | 23 ++ ...board_acva_United_Arab_Emirates_light.yaml | 23 ++ .../arabic_leaderboard_acva_Yemen_light.yaml | 23 ++ ..._leaderboard_acva_communication_light.yaml | 23 ++ ...erboard_acva_computer_and_phone_light.yaml | 23 ++ ...bic_leaderboard_acva_daily_life_light.yaml | 23 ++ ..._leaderboard_acva_entertainment_light.yaml | 23 ++ .../arabic_leaderboard_acva_light.yaml | 70 +++++ .../arabic_leaderboard_avca_light/utils.py | 16 ++ .../arabic_leaderboard_light.yaml | 25 ++ 337 files changed, 7980 insertions(+) create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/README.md create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_mcq_exams_test_ar.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_meta_ar_dialects.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_meta_ar_msa.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_sentiment_task.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/arabic_exams.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/arabic_leaderboard_arabic_exams.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_abstract_algebra.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_anatomy.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_astronomy.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_business_ethics.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_clinical_knowledge.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_biology.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_chemistry.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_computer_science.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_mathematics.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_medicine.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_physics.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_computer_security.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_conceptual_physics.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_econometrics.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_electrical_engineering.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_elementary_mathematics.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_formal_logic.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_global_facts.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_biology.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_chemistry.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_computer_science.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_european_history.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_geography.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_government_and_politics.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_macroeconomics.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_mathematics.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_microeconomics.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_physics.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_psychology.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_statistics.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_us_history.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_world_history.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_human_aging.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_human_sexuality.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_international_law.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_jurisprudence.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_logical_fallacies.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_machine_learning.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_management.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_marketing.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_medical_genetics.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_miscellaneous.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_moral_disputes.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_moral_scenarios.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_nutrition.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_philosophy.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_prehistory.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_accounting.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_law.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_medicine.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_psychology.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_public_relations.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_security_studies.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_sociology.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_us_foreign_policy.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_virology.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_world_religions.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/arabic_leaderboard_arabic_mt_arc_challenge.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/arabic_mt_arc_challenge.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/arabic_leaderboard_arabic_mt_arc_easy.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/arabic_mt_arc_easy.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/arabic_leaderboard_arabic_mt_boolq.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/arabic_mt_boolq.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/arabic_leaderboard_arabic_mt_copa.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/arabic_mt_copa.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/arabic_leaderboard_arabic_mt_hellaswag.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/arabic_mt_hellaswag.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/arabic_leaderboard_arabic_mt_mmlu.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/arabic_mt_mmlu.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/arabic_leaderboard_arabic_mt_openbook_qa.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/arabic_mt_openbook_qa.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/arabic_leaderboard_arabic_mt_piqa.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/arabic_mt_piqa.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/arabic_leaderboard_arabic_mt_race.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/arabic_mt_race.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/arabic_leaderboard_arabic_mt_sciq.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/arabic_mt_sciq.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/arabic_leaderboard_arabic_mt_toxigen.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/arabic_mt_toxigen.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Algeria.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Ancient_Egypt.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arab_Empire.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Architecture.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Art.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Astronomy.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Calligraphy.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Ceremony.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Clothing.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Culture.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Food.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Funeral.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Geography.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_History.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Language_Origin.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Literature.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Math.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Medicine.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Music.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Ornament.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Philosophy.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Physics_and_Chemistry.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Wedding.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Bahrain.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Comoros.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Egypt_modern.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromAncientEgypt.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromByzantium.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromChina.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromGreece.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromIslam.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromPersia.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromRome.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Iraq.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islam_Education.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islam_branches_and_schools.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islamic_law_system.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Jordan.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Kuwait.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Lebanon.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Libya.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Mauritania.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Mesopotamia_civilization.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Morocco.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Oman.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Palestine.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Qatar.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Saudi_Arabia.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Somalia.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Sudan.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Syria.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Tunisia.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_United_Arab_Emirates.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Yemen.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_communication.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_computer_and_phone.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_daily_life.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_entertainment.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_complete.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/README.md create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_mcq_exams_test_ar_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_meta_ar_dialects_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_meta_ar_msa_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_sentiment_task_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_exams_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_leaderboard_arabic_exams_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_abstract_algebra_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_anatomy_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_astronomy_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_business_ethics_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_clinical_knowledge_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_biology_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_chemistry_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_computer_science_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_mathematics_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_medicine_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_physics_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_computer_security_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_conceptual_physics_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_econometrics_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_electrical_engineering_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_elementary_mathematics_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_formal_logic_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_global_facts_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_biology_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_chemistry_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_computer_science_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_european_history_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_geography_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_government_and_politics_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_macroeconomics_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_mathematics_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_microeconomics_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_physics_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_psychology_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_statistics_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_us_history_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_world_history_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_human_aging_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_human_sexuality_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_international_law_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_jurisprudence_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_logical_fallacies_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_machine_learning_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_management_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_marketing_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_medical_genetics_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_miscellaneous_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_moral_disputes_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_moral_scenarios_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_nutrition_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_philosophy_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_prehistory_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_accounting_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_law_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_medicine_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_psychology_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_public_relations_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_security_studies_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_sociology_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_us_foreign_policy_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_virology_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_world_religions_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/arabic_leaderboard_arabic_mt_arc_challenge_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/arabic_mt_arc_challenge_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/arabic_leaderboard_arabic_mt_arc_easy_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/arabic_mt_arc_easy_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/arabic_leaderboard_arabic_mt_boolq_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/arabic_mt_boolq_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/arabic_mt_copa_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/arbic_leaderboard_arabic_mt_copa_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/arabic_leaderboard_arabic_mt_hellaswag_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/arabic_mt_hellaswag_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/arabic_leaderboard_arabic_mt_mmlu_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/arabic_mt_mmlu_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/arabic_leaderboard_arabic_mt_openbook_qa_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/arabic_mt_openbook_qa_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/arabic_leaderboard_arabic_mt_piqa_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/arabic_mt_piqa_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/arabic_leaderboard_arabic_mt_race_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/arabic_mt_race_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/arabic_leaderboard_arabic_mt_sciq_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/arabic_mt_sciq_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/arabic_leaderboard_arabic_mt_toxigen_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/arabic_mt_toxigen_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Algeria_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Ancient_Egypt_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arab_Empire_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Architecture_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Art_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Astronomy_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Calligraphy_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Ceremony_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Clothing_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Culture_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Food_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Funeral_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Geography_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_History_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Language_Origin_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Literature_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Math_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Medicine_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Music_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Ornament_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Philosophy_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Physics_and_Chemistry_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Wedding_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Bahrain_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Comoros_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Egypt_modern_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromAncientEgypt_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromByzantium_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromChina_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromGreece_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromIslam_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromPersia_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromRome_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Iraq_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islam_Education_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islam_branches_and_schools_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islamic_law_system_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Jordan_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Kuwait_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Lebanon_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Libya_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Mauritania_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Mesopotamia_civilization_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Morocco_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Oman_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Palestine_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Qatar_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Saudi_Arabia_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Somalia_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Sudan_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Syria_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Tunisia_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_United_Arab_Emirates_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Yemen_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_communication_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_computer_and_phone_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_daily_life_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_entertainment_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_light.yaml create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/utils.py create mode 100644 lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_light.yaml diff --git a/lm_eval/tasks/README.md b/lm_eval/tasks/README.md index 6c500ee5da..09a6dd797c 100644 --- a/lm_eval/tasks/README.md +++ b/lm_eval/tasks/README.md @@ -11,6 +11,8 @@ | [aexams](aexams/README.md) | Tasks in Arabic related to various academic exams covering a range of subjects. | Arabic | | [agieval](agieval/README.md) | Tasks involving historical data or questions related to history and historical texts. | English, Chinese | | [anli](anli/README.md) | Adversarial natural language inference tasks designed to test model robustness. | English | +| [arabic_leaderboard_complete](arabic_leaderboard_complete/README.md) | A full version of the tasks in the Open Arabic LLM Leaderboard, focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | +| [arabic_leaderboard_light](arabic_leaderboard_light/README.md) | A light version of the tasks in the Open Arabic LLM Leaderboard (i.e., 10% samples of the test set in the original benchmarks), focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | | [arabicmmlu](arabicmmlu/README.md) | Localized Arabic version of MMLU with multiple-choice questions from 40 subjects. | Arabic | | [arc](arc/README.md) | Tasks involving complex reasoning over a diverse set of questions. | English | | [arithmetic](arithmetic/README.md) | Tasks involving numerical computations and arithmetic reasoning. | English | diff --git a/lm_eval/tasks/arabic_leaderboard_complete/README.md b/lm_eval/tasks/arabic_leaderboard_complete/README.md new file mode 100644 index 0000000000..8052abcbd6 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/README.md @@ -0,0 +1,254 @@ +# Arabic Leaderboard + + +Title: Open Arabic LLM Leaderboard + +The Open Arabic LLM Leaderboard evaluates language models on a large number of different evaluation tasks that reflect the characteristics of the Arabic language and culture. +The benchmark uses several datasets, most of them translated to Arabic, and validated by native Arabic speakers. They also used benchmarks from other papers or prepared benchmarks from scratch natively for Arabic. + +Homepage: https://huggingface.co/spaces/OALL/Open-Arabic-LLM-Leaderboard + +### Citation + +``` + +@misc{OALL, + author = {Elfilali, Ali and Alobeidli, Hamza and Fourrier, Clémentine and Boussaha, Basma El Amel and Cojocaru, Ruxandra and Habib, Nathan and Hacid, Hakim}, + title = {Open Arabic LLM Leaderboard}, + year = {2024}, + publisher = {OALL}, + howpublished = "\url{https://huggingface.co/spaces/OALL/Open-Arabic-LLM-Leaderboard}" +} + +@inproceedings{almazrouei-etal-2023-alghafa, + title = "{A}l{G}hafa Evaluation Benchmark for {A}rabic Language Models", + author = "Almazrouei, Ebtesam and + Cojocaru, Ruxandra and + Baldo, Michele and + Malartic, Quentin and + Alobeidli, Hamza and + Mazzotta, Daniele and + Penedo, Guilherme and + Campesan, Giulia and + Farooq, Mugariya and + Alhammadi, Maitha and + Launay, Julien and + Noune, Badreddine", + editor = "Sawaf, Hassan and + El-Beltagy, Samhaa and + Zaghouani, Wajdi and + Magdy, Walid and + Abdelali, Ahmed and + Tomeh, Nadi and + Abu Farha, Ibrahim and + Habash, Nizar and + Khalifa, Salam and + Keleg, Amr and + Haddad, Hatem and + Zitouni, Imed and + Mrini, Khalil and + Almatham, Rawan", + booktitle = "Proceedings of ArabicNLP 2023", + month = dec, + year = "2023", + address = "Singapore (Hybrid)", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.arabicnlp-1.21", + doi = "10.18653/v1/2023.arabicnlp-1.21", + pages = "244--275", + abstract = "Recent advances in the space of Arabic large language models have opened up a wealth of potential practical applications. From optimal training strategies, large scale data acquisition and continuously increasing NLP resources, the Arabic LLM landscape has improved in a very short span of time, despite being plagued by training data scarcity and limited evaluation resources compared to English. In line with contributing towards this ever-growing field, we introduce AlGhafa, a new multiple-choice evaluation benchmark for Arabic LLMs. For showcasing purposes, we train a new suite of models, including a 14 billion parameter model, the largest monolingual Arabic decoder-only model to date. We use a collection of publicly available datasets, as well as a newly introduced HandMade dataset consisting of 8 billion tokens. Finally, we explore the quantitative and qualitative toxicity of several Arabic models, comparing our models to existing public Arabic LLMs.", +} +@misc{huang2023acegpt, + title={AceGPT, Localizing Large Language Models in Arabic}, + author={Huang Huang and Fei Yu and Jianqing Zhu and Xuening Sun and Hao Cheng and Dingjie Song and Zhihong Chen and Abdulmohsen Alharthi and Bang An and Ziche Liu and Zhiyi Zhang and Junying Chen and Jianquan Li and Benyou Wang and Lian Zhang and Ruoyu Sun and Xiang Wan and Haizhou Li and Jinchao Xu}, + year={2023}, + eprint={2309.12053}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +@misc{lighteval, + author = {Fourrier, Clémentine and Habib, Nathan and Wolf, Thomas and Tunstall, Lewis}, + title = {LightEval: A lightweight framework for LLM evaluation}, + year = {2023}, + version = {0.3.0}, + url = {https://github.com/huggingface/lighteval} +} +``` + +### Groups and Tasks + +* `arabic_leaderboard_alghafa`: A multiple-choice evaluation benchmark for zero- and few-shot evaluation of Arabic LLMs prepared from scratch natively for Arabic. + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf + * You can find the list of the tasks as follows: + * `arabic_leaderboard_alghafa_mcq_exams_test_ar` + * `arabic_leaderboard_alghafa_meta_ar_dialects` + * `arabic_leaderboard_alghafa_meta_ar_msa` + * `arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task` + * `arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task` + * `arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task` + * `arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task` + * `arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task` + * `arabic_leaderboard_alghafa_multiple_choice_sentiment_task` +* `arabic_leaderboard_arabic_exams`: A question answering benchmark for high school examinations in different school subjects that requires knowledge and reasoning in different languages in multiple domains. + * Paper: https://aclanthology.org/2020.emnlp-main.438.pdf +* `arabic_leaderboard_arabic_mmlu`: A multi-task language understanding benchmark for the Arabic language, sourced from school exams across diverse educational levels in different countries with native speakers in the region. + The data comprises multiple choice questions in 40 tasks. + * Paper: https://arxiv.org/pdf/2402.12840 + * You can find the list of the tasks as follows: + * `arabic_leaderboard_arabic_mmlu_abstract_algebra` + * `arabic_leaderboard_arabic_mmlu_anatomy` + * `arabic_leaderboard_arabic_mmlu_astronomy` + * `arabic_leaderboard_arabic_mmlu_business_ethics` + * `arabic_leaderboard_arabic_mmlu_clinical_knowledge` + * `arabic_leaderboard_arabic_mmlu_college_biology` + * `arabic_leaderboard_arabic_mmlu_college_chemistry` + * `arabic_leaderboard_arabic_mmlu_college_computer_science` + * `arabic_leaderboard_arabic_mmlu_college_mathematics` + * `arabic_leaderboard_arabic_mmlu_college_medicine` + * `arabic_leaderboard_arabic_mmlu_college_physics` + * `arabic_leaderboard_arabic_mmlu_computer_security` + * `arabic_leaderboard_arabic_mmlu_conceptual_physics` + * `arabic_leaderboard_arabic_mmlu_econometrics` + * `arabic_leaderboard_arabic_mmlu_electrical_engineering` + * `arabic_leaderboard_arabic_mmlu_elementary_mathematics` + * `arabic_leaderboard_arabic_mmlu_formal_logic` + * `arabic_leaderboard_arabic_mmlu_global_facts` + * `arabic_leaderboard_arabic_mmlu_high_school_biology` + * `arabic_leaderboard_arabic_mmlu_high_school_chemistry` + * `arabic_leaderboard_arabic_mmlu_high_school_computer_science` + * `arabic_leaderboard_arabic_mmlu_high_school_european_history` + * `arabic_leaderboard_arabic_mmlu_high_school_geography` + * `arabic_leaderboard_arabic_mmlu_high_school_government_and_politics` + * `arabic_leaderboard_arabic_mmlu_high_school_macroeconomics` + * `arabic_leaderboard_arabic_mmlu_high_school_mathematics` + * `arabic_leaderboard_arabic_mmlu_high_school_microeconomics` + * `arabic_leaderboard_arabic_mmlu_high_school_physics` + * `arabic_leaderboard_arabic_mmlu_high_school_psychology` + * `arabic_leaderboard_arabic_mmlu_high_school_statistics` + * `arabic_leaderboard_arabic_mmlu_high_school_us_history` + * `arabic_leaderboard_arabic_mmlu_high_school_us_history` + * `arabic_leaderboard_arabic_mmlu_human_aging` + * `arabic_leaderboard_arabic_mmlu_human_sexuality` + * `arabic_leaderboard_arabic_mmlu_international_law` + * `arabic_leaderboard_arabic_mmlu_jurisprudence` + * `arabic_leaderboard_arabic_mmlu_logical_fallacies` + * `arabic_leaderboard_arabic_mmlu_machine_learning` + * `arabic_leaderboard_arabic_mmlu_management` + * `arabic_leaderboard_arabic_mmlu_marketing` + * `arabic_leaderboard_arabic_mmlu_medical_genetics` + * `arabic_leaderboard_arabic_mmlu_miscellaneous` + * `arabic_leaderboard_arabic_mmlu_moral_disputes` + * `arabic_leaderboard_arabic_mmlu_moral_scenarios` + * `arabic_leaderboard_arabic_mmlu_nutrition` + * `arabic_leaderboard_arabic_mmlu_philosophy` + * `arabic_leaderboard_arabic_mmlu_prehistory` + * `arabic_leaderboard_arabic_mmlu_professional_accounting` + * `arabic_leaderboard_arabic_mmlu_professional_law` + * `arabic_leaderboard_arabic_mmlu_professional_medicine` + * `arabic_leaderboard_arabic_mmlu_professional_psychology` + * `arabic_leaderboard_arabic_mmlu_public_relations` + * `arabic_leaderboard_arabic_mmlu_security_studies` + * `arabic_leaderboard_arabic_mmlu_sociology` + * `arabic_leaderboard_arabic_mmlu_us_foreign_policy` + * `arabic_leaderboard_arabic_mmlu_virology` + * `arabic_leaderboard_arabic_mmlu_world_religions` +* `arabic_leaderboard_arabic_mt_arc_challenge`: AI2 Reasoning Challenge (ARC) is a multiple-choice question task. The dataset contains only natural, grade-school science questions, + written for human tests. The challenge set contains only questions answered incorrectly by both a retrieval-based algorithm and a word co-occurence algorithm. (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_arc_easy`: This dataset is the same as `arabic_arc_challenge`, except it is not from the challenge set. + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_boolq`: A true/false questions dataset that contains the columns passage, question, and the answer (i.e., true/false). (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_copa`: Choice Of Plausible Alternatives (COPA) is a multiple-choice question dataset, which involves open-domain commonsense causal reasoning. (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_hellaswag`: The tesk is to choose the next set of sentences, based on the given candidates. The tasks involve reading comprehension and information retrieval challenges + by testing the abilities of the models on basic knowledge (i.e., from 3rd grade to 9th) and commonsense inference. (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_mmlu`: A multiple-choice question answering dataset from various branches of knowledge including humanities, social sciences, hard sciences, and other areas. The examples in the English dataset are translated into Arabic using ChatGPT with a translation prompt. + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_openbook_qa`: A multiple-choice openbook question answering dataset that requires external knowledge and reasoning. The open book that comes with these questions is + based on elementary level science facts. (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_piqa`: Physical Interaction Question Answering (PIQA) is a multiple-choice question answering based on physical commonsense reasoning. (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_race`: A multiple-choice questions dataset to assess reading comprehension tasks based on English exams in China - designed for middle school and high school students + (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_sciq`: A multiple-choice Science Question Answering task to assess understanding of scientific concepts about physics, chemistry, and biology. (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_toxigen`: This benchmark consists of tasks designed to evaluate language models and classify input text as hateful or not hateful. (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_acva`: Arabic-Culture-Value-Alignment (ACVA) is a yes/no question dataset, generated by GPT3.5 Turbo from Arabic topics to assess model alignment with Arabic values and cultures. + * Paper: https://arxiv.org/pdf/2309.12053 + * You can find the list of the tasks as follows: + - `arabic_leaderboard_acva_Algeria` + - `arabic_leaderboard_acva_Ancient_Egypt` + - `arabic_leaderboard_acva_Arab_Empire` + - `arabic_leaderboard_acva_Arabic_Architecture` + - `arabic_leaderboard_acva_Arabic_Art` + - `arabic_leaderboard_acva_Arabic_Astronomy` + - `arabic_leaderboard_acva_Arabic_Calligraphy` + - `arabic_leaderboard_acva_Arabic_Ceremony` + - `arabic_leaderboard_acva_Arabic_Clothing` + - `arabic_leaderboard_acva_Arabic_Culture` + - `arabic_leaderboard_acva_Arabic_Food` + - `arabic_leaderboard_acva_Arabic_Funeral` + - `arabic_leaderboard_acva_Arabic_Geography` + - `arabic_leaderboard_acva_Arabic_History` + - `arabic_leaderboard_acva_Arabic_Language_Origin` + - `arabic_leaderboard_acva_Arabic_Literature` + - `arabic_leaderboard_acva_Arabic_Math` + - `arabic_leaderboard_acva_Arabic_Medicine` + - `arabic_leaderboard_acva_Arabic_Music` + - `arabic_leaderboard_acva_Arabic_Ornament` + - `arabic_leaderboard_acva_Arabic_Philosophy` + - `arabic_leaderboard_acva_Arabic_Physics_and_Chemistry` + - `arabic_leaderboard_acva_Arabic_Wedding` + - `arabic_leaderboard_acva_Bahrain` + - `arabic_leaderboard_acva_Comoros` + - `arabic_leaderboard_acva_Egypt_modern` + - `arabic_leaderboard_acva_InfluenceFromAncientEgypt` + - `arabic_leaderboard_acva_InfluenceFromByzantium` + - `arabic_leaderboard_acva_InfluenceFromChina` + - `arabic_leaderboard_acva_InfluenceFromGreece` + - `arabic_leaderboard_acva_InfluenceFromIslam` + - `arabic_leaderboard_acva_InfluenceFromPersia` + - `arabic_leaderboard_acva_InfluenceFromRome` + - `arabic_leaderboard_acva_Iraq` + - `arabic_leaderboard_acva_Islam_Education` + - `arabic_leaderboard_acva_Islam_branches_and_schools` + - `arabic_leaderboard_acva_Islamic_law_system` + - `arabic_leaderboard_acva_Jordan` + - `arabic_leaderboard_acva_Kuwait` + - `arabic_leaderboard_acva_Lebanon` + - `arabic_leaderboard_acva_Libya` + - `arabic_leaderboard_acva_Mauritania` + - `arabic_acva_Mesopotamia_civilization` + - `arabic_leaderboard_acva_Morocco` + - `arabic_leaderboard_acva_Oman` + - `arabic_leaderboard_acva_Palestine` + - `arabic_leaderboard_acva_Qatar` + - `arabic_leaderboard_acva_Saudi_Arabia` + - `arabic_leaderboard_acva_Somalia` + - `arabic_leaderboard_acva_Sudan` + - `arabic_leaderboard_acva_Syria` + - `arabic_leaderboard_acva_Tunisia` + - `arabic_leaderboard_acva_United_Arab_Emirates` + - `arabic_leaderboard_acva_Yemen` + - `arabic_leaderboard_acva_communication` + - `arabic_leaderboard_acva_computer_and_phone` + - `arabic_leaderboard_acva_daily_life` + - `arabic_leaderboard_acva_entertainment` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [ ] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa.yaml new file mode 100644 index 0000000000..6f0014d812 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa.yaml @@ -0,0 +1,23 @@ +group: arabic_leaderboard_alghafa +task: + - arabic_leaderboard_alghafa_mcq_exams_test_ar + - arabic_leaderboard_alghafa_meta_ar_dialects + - arabic_leaderboard_alghafa_meta_ar_msa + - arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task + - arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task + - arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task + - arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task + - arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task + - arabic_leaderboard_alghafa_multiple_choice_sentiment_task + + + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_mcq_exams_test_ar.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_mcq_exams_test_ar.yaml new file mode 100644 index 0000000000..e436e29574 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_mcq_exams_test_ar.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_mcq_exams_test_ar +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: mcq_exams_test_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_meta_ar_dialects.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_meta_ar_dialects.yaml new file mode 100644 index 0000000000..f19c2ecefe --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_meta_ar_dialects.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_meta_ar_dialects +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: meta_ar_dialects +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_meta_ar_msa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_meta_ar_msa.yaml new file mode 100644 index 0000000000..0d95ec5a06 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_meta_ar_msa.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_meta_ar_msa +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: meta_ar_msa +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task.yaml new file mode 100644 index 0000000000..46d2b6abcf --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: multiple_choice_facts_truefalse_balanced_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task.yaml new file mode 100644 index 0000000000..13150c690b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: multiple_choice_grounded_statement_soqal_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task.yaml new file mode 100644 index 0000000000..3a17548f8e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: multiple_choice_grounded_statement_xglue_mlqa_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task.yaml new file mode 100644 index 0000000000..8e34a45c7a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: multiple_choice_rating_sentiment_no_neutral_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task.yaml new file mode 100644 index 0000000000..b31748516a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: multiple_choice_rating_sentiment_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_sentiment_task.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_sentiment_task.yaml new file mode 100644 index 0000000000..191b26ba0a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_sentiment_task.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_sentiment_task +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: multiple_choice_sentiment_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/utils.py new file mode 100644 index 0000000000..62f9874e63 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/arabic_exams.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/arabic_exams.yaml new file mode 100644 index 0000000000..edc20fe4b9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/arabic_exams.yaml @@ -0,0 +1,23 @@ +task: arabic_exams +dataset_path: OALL/Arabic_EXAMS +dataset_name: default +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/arabic_leaderboard_arabic_exams.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/arabic_leaderboard_arabic_exams.yaml new file mode 100644 index 0000000000..2bf77eb361 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/arabic_leaderboard_arabic_exams.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_exams +task: + - arabic_exams + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/utils.py new file mode 100644 index 0000000000..72af1c40fe --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/utils.py @@ -0,0 +1,33 @@ +import datasets +import numpy as np + + +# fmt: off +LETTER_INDICES_AR = ["أ", "ب", "ج", "د", "هـ", "و", "ز", "ح", "ط", "ي", "ك", "ل", "م", "ن", "س", "ع", "ف", "ص", "ق", "ر", "ش", "ت", "ث", "خ", "ذ", "ض", "ظ", "غ"] +# fmt: on + + +# fmt: off +LETTER_INDICES = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"] +# fmt: on + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + topic = doc["subject"] + question = doc["question"] + choices = [doc["A"], doc["B"], doc["C"], doc["D"]] + choices_formatted = [ + f" {LETTER_INDICES_AR[i]}) {choice}\n" for i, choice in enumerate(choices) + ] + answer = doc["answer"] + answer_index = LETTER_INDICES.index(answer) + + instruction = f"الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح حول {topic.replace('_', ' ')}. \n\n" + query = f"{instruction}السؤال: {question}\n" + query += "\n".join(choices_formatted) + query += "\nالإجابة:" + + return {"query": query, "choices": LETTER_INDICES_AR[:4], "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu.yaml new file mode 100644 index 0000000000..ad2751bf32 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu.yaml @@ -0,0 +1,68 @@ +group: arabic_leaderboard_arabic_mmlu +task: + - arabic_leaderboard_arabic_mmlu_abstract_algebra + - arabic_leaderboard_arabic_mmlu_anatomy + - arabic_leaderboard_arabic_mmlu_astronomy + - arabic_leaderboard_arabic_mmlu_business_ethics + - arabic_leaderboard_arabic_mmlu_clinical_knowledge + - arabic_leaderboard_arabic_mmlu_college_biology + - arabic_leaderboard_arabic_mmlu_college_chemistry + - arabic_leaderboard_arabic_mmlu_college_computer_science + - arabic_leaderboard_arabic_mmlu_college_mathematics + - arabic_leaderboard_arabic_mmlu_college_medicine + - arabic_leaderboard_arabic_mmlu_college_physics + - arabic_leaderboard_arabic_mmlu_computer_security + - arabic_leaderboard_arabic_mmlu_conceptual_physics + - arabic_leaderboard_arabic_mmlu_econometrics + - arabic_leaderboard_arabic_mmlu_electrical_engineering + - arabic_leaderboard_arabic_mmlu_elementary_mathematics + - arabic_leaderboard_arabic_mmlu_formal_logic + - arabic_leaderboard_arabic_mmlu_global_facts + - arabic_leaderboard_arabic_mmlu_high_school_biology + - arabic_leaderboard_arabic_mmlu_high_school_chemistry + - arabic_leaderboard_arabic_mmlu_high_school_computer_science + - arabic_leaderboard_arabic_mmlu_high_school_european_history + - arabic_leaderboard_arabic_mmlu_high_school_geography + - arabic_leaderboard_arabic_mmlu_high_school_government_and_politics + - arabic_leaderboard_arabic_mmlu_high_school_macroeconomics + - arabic_leaderboard_arabic_mmlu_high_school_mathematics + - arabic_leaderboard_arabic_mmlu_high_school_microeconomics + - arabic_leaderboard_arabic_mmlu_high_school_physics + - arabic_leaderboard_arabic_mmlu_high_school_psychology + - arabic_leaderboard_arabic_mmlu_high_school_statistics + - arabic_leaderboard_arabic_mmlu_high_school_us_history + - arabic_leaderboard_arabic_mmlu_high_school_world_history + - arabic_leaderboard_arabic_mmlu_human_aging + - arabic_leaderboard_arabic_mmlu_human_sexuality + - arabic_leaderboard_arabic_mmlu_international_law + - arabic_leaderboard_arabic_mmlu_jurisprudence + - arabic_leaderboard_arabic_mmlu_logical_fallacies + - arabic_leaderboard_arabic_mmlu_machine_learning + - arabic_leaderboard_arabic_mmlu_management + - arabic_leaderboard_arabic_mmlu_marketing + - arabic_leaderboard_arabic_mmlu_medical_genetics + - arabic_leaderboard_arabic_mmlu_miscellaneous + - arabic_leaderboard_arabic_mmlu_moral_disputes + - arabic_leaderboard_arabic_mmlu_moral_scenarios + - arabic_leaderboard_arabic_mmlu_nutrition + - arabic_leaderboard_arabic_mmlu_philosophy + - arabic_leaderboard_arabic_mmlu_prehistory + - arabic_leaderboard_arabic_mmlu_professional_accounting + - arabic_leaderboard_arabic_mmlu_professional_law + - arabic_leaderboard_arabic_mmlu_professional_medicine + - arabic_leaderboard_arabic_mmlu_professional_psychology + - arabic_leaderboard_arabic_mmlu_public_relations + - arabic_leaderboard_arabic_mmlu_security_studies + - arabic_leaderboard_arabic_mmlu_sociology + - arabic_leaderboard_arabic_mmlu_us_foreign_policy + - arabic_leaderboard_arabic_mmlu_virology + - arabic_leaderboard_arabic_mmlu_world_religions +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_abstract_algebra.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000..3d0946be2c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_abstract_algebra.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_abstract_algebra +dataset_path: OALL/Arabic_MMLU +dataset_name: abstract_algebra +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_anatomy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_anatomy.yaml new file mode 100644 index 0000000000..24af11dd2f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_anatomy.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_anatomy +dataset_path: OALL/Arabic_MMLU +dataset_name: anatomy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_astronomy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_astronomy.yaml new file mode 100644 index 0000000000..0aa9680906 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_astronomy.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_astronomy +dataset_path: OALL/Arabic_MMLU +dataset_name: astronomy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_business_ethics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_business_ethics.yaml new file mode 100644 index 0000000000..18c941e422 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_business_ethics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_business_ethics +dataset_path: OALL/Arabic_MMLU +dataset_name: business_ethics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000..9460403c98 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_clinical_knowledge.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_clinical_knowledge +dataset_path: OALL/Arabic_MMLU +dataset_name: clinical_knowledge +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_biology.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_biology.yaml new file mode 100644 index 0000000000..2f34d342d6 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_biology.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_biology +dataset_path: OALL/Arabic_MMLU +dataset_name: college_biology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_chemistry.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000..17d63b60bb --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_chemistry.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_chemistry +dataset_path: OALL/Arabic_MMLU +dataset_name: college_chemistry +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_computer_science.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000..a3f5d3e84c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_computer_science.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_computer_science +dataset_path: OALL/Arabic_MMLU +dataset_name: college_computer_science +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_mathematics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000..0284093dd9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_mathematics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_mathematics +dataset_path: OALL/Arabic_MMLU +dataset_name: college_mathematics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_medicine.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_medicine.yaml new file mode 100644 index 0000000000..e21246e7be --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_medicine.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_medicine +dataset_path: OALL/Arabic_MMLU +dataset_name: college_medicine +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_physics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_physics.yaml new file mode 100644 index 0000000000..ab23f490f3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_physics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_physics +dataset_path: OALL/Arabic_MMLU +dataset_name: college_physics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_computer_security.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_computer_security.yaml new file mode 100644 index 0000000000..96624cd02f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_computer_security.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_computer_security +dataset_path: OALL/Arabic_MMLU +dataset_name: computer_security +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_conceptual_physics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000..cd605de40a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_conceptual_physics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_conceptual_physics +dataset_path: OALL/Arabic_MMLU +dataset_name: conceptual_physics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_econometrics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_econometrics.yaml new file mode 100644 index 0000000000..60c9f373a3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_econometrics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_econometrics +dataset_path: OALL/Arabic_MMLU +dataset_name: econometrics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_electrical_engineering.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000..83aa42a620 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_electrical_engineering.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_electrical_engineering +dataset_path: OALL/Arabic_MMLU +dataset_name: electrical_engineering +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000..ac06d9ec7c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_elementary_mathematics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_elementary_mathematics +dataset_path: OALL/Arabic_MMLU +dataset_name: elementary_mathematics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_formal_logic.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_formal_logic.yaml new file mode 100644 index 0000000000..5e1d60758b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_formal_logic.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_formal_logic +dataset_path: OALL/Arabic_MMLU +dataset_name: formal_logic +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_global_facts.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_global_facts.yaml new file mode 100644 index 0000000000..074248d8fe --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_global_facts.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_global_facts +dataset_path: OALL/Arabic_MMLU +dataset_name: global_facts +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_biology.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000..09862e1ce6 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_biology.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_biology +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_biology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000..849ad63ed7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_chemistry.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_chemistry +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_chemistry +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000..e91bfe7fb9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_computer_science.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_computer_science +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_computer_science +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_european_history.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000..912e57bfab --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_european_history.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_european_history +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_european_history +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_geography.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000..33c41db0f1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_geography.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_geography +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_geography +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000..16689f115f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_government_and_politics +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_government_and_politics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000..04ec5d7431 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_macroeconomics +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_macroeconomics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000..fd4ebd5161 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_mathematics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_mathematics +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_mathematics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000..7ba3eea694 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_microeconomics +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_microeconomics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_physics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000..8d53cca80e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_physics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_physics +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_physics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_psychology.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000..129733d1dd --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_psychology.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_psychology +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_psychology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_statistics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000..b23e1a77e5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_statistics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_statistics +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_statistics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_us_history.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000..cc6ec9a397 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_us_history.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_us_history +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_us_history +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_world_history.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000..b537669fec --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_world_history.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_world_history +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_world_history +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_human_aging.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_human_aging.yaml new file mode 100644 index 0000000000..62124769b1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_human_aging.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_human_aging +dataset_path: OALL/Arabic_MMLU +dataset_name: human_aging +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_human_sexuality.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000..bf6c298b8a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_human_sexuality.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_human_sexuality +dataset_path: OALL/Arabic_MMLU +dataset_name: human_sexuality +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_international_law.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_international_law.yaml new file mode 100644 index 0000000000..feec16f59b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_international_law.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_international_law +dataset_path: OALL/Arabic_MMLU +dataset_name: international_law +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_jurisprudence.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000..fcc1a3ab9c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_jurisprudence.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_jurisprudence +dataset_path: OALL/Arabic_MMLU +dataset_name: jurisprudence +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_logical_fallacies.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000..c6de637bae --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_logical_fallacies.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_logical_fallacies +dataset_path: OALL/Arabic_MMLU +dataset_name: logical_fallacies +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_machine_learning.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_machine_learning.yaml new file mode 100644 index 0000000000..bf191fc7c8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_machine_learning.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_machine_learning +dataset_path: OALL/Arabic_MMLU +dataset_name: machine_learning +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_management.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_management.yaml new file mode 100644 index 0000000000..4bbc800cfe --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_management.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_management +dataset_path: OALL/Arabic_MMLU +dataset_name: management +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_marketing.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_marketing.yaml new file mode 100644 index 0000000000..59694487eb --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_marketing.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_marketing +dataset_path: OALL/Arabic_MMLU +dataset_name: marketing +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_medical_genetics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000..88f0de37c3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_medical_genetics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_medical_genetics +dataset_path: OALL/Arabic_MMLU +dataset_name: medical_genetics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_miscellaneous.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000..da333e4536 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_miscellaneous.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_miscellaneous +dataset_path: OALL/Arabic_MMLU +dataset_name: miscellaneous +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_moral_disputes.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000..1d0d07945f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_moral_disputes.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_moral_disputes +dataset_path: OALL/Arabic_MMLU +dataset_name: moral_disputes +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_moral_scenarios.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000..c0c924650f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_moral_scenarios.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_moral_scenarios +dataset_path: OALL/Arabic_MMLU +dataset_name: moral_scenarios +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_nutrition.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_nutrition.yaml new file mode 100644 index 0000000000..24ad69b90d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_nutrition.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_nutrition +dataset_path: OALL/Arabic_MMLU +dataset_name: nutrition +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_philosophy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_philosophy.yaml new file mode 100644 index 0000000000..a57dcf7ecd --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_philosophy.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_philosophy +dataset_path: OALL/Arabic_MMLU +dataset_name: philosophy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_prehistory.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_prehistory.yaml new file mode 100644 index 0000000000..45ba2e5de2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_prehistory.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_prehistory +dataset_path: OALL/Arabic_MMLU +dataset_name: prehistory +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_accounting.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000..d931a00099 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_accounting.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_accounting +dataset_path: OALL/Arabic_MMLU +dataset_name: professional_accounting +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_law.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_law.yaml new file mode 100644 index 0000000000..e11d0368f5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_law.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_law +dataset_path: OALL/Arabic_MMLU +dataset_name: professional_law +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_medicine.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000..7a10d8157f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_medicine.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_medicine +dataset_path: OALL/Arabic_MMLU +dataset_name: professional_medicine +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_psychology.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000..bb12274adb --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_psychology.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_psychology +dataset_path: OALL/Arabic_MMLU +dataset_name: professional_psychology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_public_relations.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_public_relations.yaml new file mode 100644 index 0000000000..3361f775b4 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_public_relations.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_public_relations +dataset_path: OALL/Arabic_MMLU +dataset_name: public_relations +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_security_studies.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_security_studies.yaml new file mode 100644 index 0000000000..781a6145f0 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_security_studies.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_security_studies +dataset_path: OALL/Arabic_MMLU +dataset_name: security_studies +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_sociology.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_sociology.yaml new file mode 100644 index 0000000000..2c80872c97 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_sociology.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_sociology +dataset_path: OALL/Arabic_MMLU +dataset_name: sociology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000..f767e0a78d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_us_foreign_policy.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_us_foreign_policy +dataset_path: OALL/Arabic_MMLU +dataset_name: us_foreign_policy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_virology.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_virology.yaml new file mode 100644 index 0000000000..8103face6c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_virology.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_virology +dataset_path: OALL/Arabic_MMLU +dataset_name: virology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_world_religions.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_world_religions.yaml new file mode 100644 index 0000000000..31c563cc53 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_world_religions.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_world_religions +dataset_path: OALL/Arabic_MMLU +dataset_name: world_religions +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/utils.py new file mode 100644 index 0000000000..da927b66fc --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/utils.py @@ -0,0 +1,35 @@ +import datasets +import numpy as np + + +# fmt: off +LETTER_INDICES_AR = ["أ", "ب", "ج", "د", "هـ", "و", "ز", "ح", "ط", "ي", "ك", "ل", "م", "ن", "س", "ع", "ف", "ص", "ق", "ر", "ش", "ت", "ث", "خ", "ذ", "ض", "ظ", "غ"] +# fmt: on + + +# fmt: off +LETTER_INDICES = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"] +# fmt: on + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + topic = doc["subject"] + instruction = f"الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح حول {topic.replace('_', ' ')}. \n\n" + choices = [doc["A"], doc["B"], doc["C"], doc["D"]] + # Answers are provided with roman letters - we look for the correct index in LETTER_INDICES, + # it will then be applied to arabic letters + gold_ix = LETTER_INDICES.index(doc["answer"]) + + query = f"{instruction}{doc['question']}\n" + query += "".join( + [ + f"{key}. {choice}\n" + for key, choice in zip(LETTER_INDICES_AR[:4], choices) + ] + ) + query += "الإجابة:" + + return {"query": query, "choices": LETTER_INDICES_AR[:4], "gold": gold_ix} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/arabic_leaderboard_arabic_mt_arc_challenge.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/arabic_leaderboard_arabic_mt_arc_challenge.yaml new file mode 100644 index 0000000000..f49aed0716 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/arabic_leaderboard_arabic_mt_arc_challenge.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_arc_challenge +task: + - arabic_mt_arc_challenge + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/arabic_mt_arc_challenge.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/arabic_mt_arc_challenge.yaml new file mode 100644 index 0000000000..e0b245aabb --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/arabic_mt_arc_challenge.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_arc_challenge +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: arc_challenge_okapi_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/utils.py new file mode 100644 index 0000000000..62f9874e63 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/arabic_leaderboard_arabic_mt_arc_easy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/arabic_leaderboard_arabic_mt_arc_easy.yaml new file mode 100644 index 0000000000..6abd5fa21b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/arabic_leaderboard_arabic_mt_arc_easy.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_arc_easy +task: + - arabic_mt_arc_easy + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/arabic_mt_arc_easy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/arabic_mt_arc_easy.yaml new file mode 100644 index 0000000000..b629529f06 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/arabic_mt_arc_easy.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_arc_easy +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: arc_easy_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/utils.py new file mode 100644 index 0000000000..62f9874e63 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/arabic_leaderboard_arabic_mt_boolq.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/arabic_leaderboard_arabic_mt_boolq.yaml new file mode 100644 index 0000000000..5072f01dd7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/arabic_leaderboard_arabic_mt_boolq.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_boolq +task: + - arabic_mt_boolq + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/arabic_mt_boolq.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/arabic_mt_boolq.yaml new file mode 100644 index 0000000000..299570af81 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/arabic_mt_boolq.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_boolq +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: boolq_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/utils.py new file mode 100644 index 0000000000..dcbc10d92e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/utils.py @@ -0,0 +1,24 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["question"] + passage = doc["passage"] + instruction = "بناء على المقطع التالي، أجب عن السؤال ب نعم أو لا" + query = f"""{instruction} + المقطع : + {passage} + السؤال: + {question} + الإجابة: + """ + + return { + "query": query, + "choices": ["نعم", "لا"], + "gold": 0 if doc["answer"] else 1, + } + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/arabic_leaderboard_arabic_mt_copa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/arabic_leaderboard_arabic_mt_copa.yaml new file mode 100644 index 0000000000..3ef88d9c37 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/arabic_leaderboard_arabic_mt_copa.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_copa +task: + - arabic_mt_copa + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/arabic_mt_copa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/arabic_mt_copa.yaml new file mode 100644 index 0000000000..e9483e1de5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/arabic_mt_copa.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_copa +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: copa_ext_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/utils.py new file mode 100644 index 0000000000..175ebdadc1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/utils.py @@ -0,0 +1,19 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + premise = doc["premise"] + choices = [doc["choice1"], doc["choice2"]] + question_map = {"cause": "لأن", "effect": "لذلك"} + question = question_map[doc["question"]] + answer = doc["label"] + + query = "{}، {} :\n0) {}\n1) {}\nالإجابة:".format( + premise, question, choices[0], choices[1] + ) + + return {"query": query, "choices": choices, "gold": answer} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/arabic_leaderboard_arabic_mt_hellaswag.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/arabic_leaderboard_arabic_mt_hellaswag.yaml new file mode 100644 index 0000000000..a70f5ab68d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/arabic_leaderboard_arabic_mt_hellaswag.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_hellaswag +task: + - arabic_mt_hellaswag + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/arabic_mt_hellaswag.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/arabic_mt_hellaswag.yaml new file mode 100644 index 0000000000..59a4547485 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/arabic_mt_hellaswag.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_hellaswag +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: hellaswag_okapi_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/utils.py new file mode 100644 index 0000000000..6b5a9f1f4f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/utils.py @@ -0,0 +1,30 @@ +import re + +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + ctx = re.sub(r"\[.*?\]", "", doc["ctx"]) # Remove latin words within brackets + endings = [ + re.sub(r"\[.*?\]", "", e) for e in eval(doc["endings"]) + ] # endings is a string representation of a list + answer_index = doc["label"] + instruction = ( + "بناء على السياق التالي، اختر النهاية الصحيحة من الاقتراحات التالية" + ) + + query = f"""{instruction} + السياق: + {ctx} + الاقتراحات: + + """ + for i, ending in enumerate(endings): + query += f"{i}) {ending}\n" + query += "الإجابة:" + + return {"query": query, "choices": endings, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/arabic_leaderboard_arabic_mt_mmlu.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/arabic_leaderboard_arabic_mt_mmlu.yaml new file mode 100644 index 0000000000..0188b5ddc4 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/arabic_leaderboard_arabic_mt_mmlu.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_mmlu +task: + - arabic_mt_mmlu + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/arabic_mt_mmlu.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/arabic_mt_mmlu.yaml new file mode 100644 index 0000000000..4f3cd249c2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/arabic_mt_mmlu.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_mmlu +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: mmlu_okapi_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/utils.py new file mode 100644 index 0000000000..62f9874e63 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/arabic_leaderboard_arabic_mt_openbook_qa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/arabic_leaderboard_arabic_mt_openbook_qa.yaml new file mode 100644 index 0000000000..dd3b78f4d0 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/arabic_leaderboard_arabic_mt_openbook_qa.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_openbook_qa +task: + - arabic_mt_openbook_qa + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/arabic_mt_openbook_qa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/arabic_mt_openbook_qa.yaml new file mode 100644 index 0000000000..b826a18927 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/arabic_mt_openbook_qa.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_openbook_qa +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: openbook_qa_ext_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/utils.py new file mode 100644 index 0000000000..62f9874e63 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/arabic_leaderboard_arabic_mt_piqa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/arabic_leaderboard_arabic_mt_piqa.yaml new file mode 100644 index 0000000000..b75bcc2b1c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/arabic_leaderboard_arabic_mt_piqa.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_piqa +task: + - arabic_mt_piqa + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/arabic_mt_piqa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/arabic_mt_piqa.yaml new file mode 100644 index 0000000000..fa93a937a8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/arabic_mt_piqa.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_piqa +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: piqa_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/utils.py new file mode 100644 index 0000000000..62f9874e63 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/arabic_leaderboard_arabic_mt_race.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/arabic_leaderboard_arabic_mt_race.yaml new file mode 100644 index 0000000000..f3f91c278d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/arabic_leaderboard_arabic_mt_race.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_race +task: + - arabic_mt_race + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/arabic_mt_race.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/arabic_mt_race.yaml new file mode 100644 index 0000000000..ec2aee6898 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/arabic_mt_race.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_race +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: race_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/utils.py new file mode 100644 index 0000000000..62f9874e63 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/arabic_leaderboard_arabic_mt_sciq.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/arabic_leaderboard_arabic_mt_sciq.yaml new file mode 100644 index 0000000000..7768047c4c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/arabic_leaderboard_arabic_mt_sciq.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_sciq +task: + - arabic_mt_sciq + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/arabic_mt_sciq.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/arabic_mt_sciq.yaml new file mode 100644 index 0000000000..07f96b7574 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/arabic_mt_sciq.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_sciq +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: sciq_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/utils.py new file mode 100644 index 0000000000..ddb42eeb8c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/utils.py @@ -0,0 +1,41 @@ +import random + +import datasets +import numpy as np + + +def doc_to_text(doc): + instruction = ( + "بناءً على السياق أدناه، اختر الإجابة الصحيحة للسؤال التالي من قائمة الاقتراحات" + ) + support = doc["support"] + question = doc["question"] + query = f"""{instruction} + السياق: + {support} + السؤال: + {question} + الإجابات المحتملة: + + """ + return query + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + correct_answer = doc["correct_answer"] + choices = [ + doc["distractor1"], + doc["distractor2"], + doc["distractor3"], + correct_answer, + ] + + # Shuffle the choices + random.shuffle(choices) + + answer_index = choices.index(correct_answer) + + return {"query": doc_to_text(doc), "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/arabic_leaderboard_arabic_mt_toxigen.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/arabic_leaderboard_arabic_mt_toxigen.yaml new file mode 100644 index 0000000000..272166206b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/arabic_leaderboard_arabic_mt_toxigen.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_toxigen +task: + - arabic_mt_toxigen + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/arabic_mt_toxigen.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/arabic_mt_toxigen.yaml new file mode 100644 index 0000000000..8a140793d7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/arabic_mt_toxigen.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_toxigen +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: toxigen_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/utils.py new file mode 100644 index 0000000000..09d311e5a1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + text = doc["text"] + label = 1 if ((doc["toxicity_ai"] + doc["toxicity_human"]) > 5.5) else 0 + instruction = 'هل العبارة التالية تحض على الكراهية؟ قم بالرد بـ"نعم" أو "لا".' + + query = f"""{instruction} + العبارة: + '{text}' + الإجابة: + """ + out_doc = { + "query": query, + "choices": ["لا", "نعم"], + "gold": label, + } + return out_doc + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva.yaml new file mode 100644 index 0000000000..8e2dab57b0 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva.yaml @@ -0,0 +1,70 @@ +group: arabic_leaderboard_acva +task: + - arabic_leaderboard_acva_Algeria + - arabic_leaderboard_acva_Ancient_Egypt + - arabic_leaderboard_acva_Arab_Empire + - arabic_leaderboard_acva_Arabic_Architecture + - arabic_leaderboard_acva_Arabic_Art + - arabic_leaderboard_acva_Arabic_Astronomy + - arabic_leaderboard_acva_Arabic_Calligraphy + - arabic_leaderboard_acva_Arabic_Ceremony + - arabic_leaderboard_acva_Arabic_Clothing + - arabic_leaderboard_acva_Arabic_Culture + - arabic_leaderboard_acva_Arabic_Food + - arabic_leaderboard_acva_Arabic_Funeral + - arabic_leaderboard_acva_Arabic_Geography + - arabic_leaderboard_acva_Arabic_History + - arabic_leaderboard_acva_Arabic_Language_Origin + - arabic_leaderboard_acva_Arabic_Literature + - arabic_leaderboard_acva_Arabic_Math + - arabic_leaderboard_acva_Arabic_Medicine + - arabic_leaderboard_acva_Arabic_Music + - arabic_leaderboard_acva_Arabic_Ornament + - arabic_leaderboard_acva_Arabic_Philosophy + - arabic_leaderboard_acva_Arabic_Physics_and_Chemistry + - arabic_leaderboard_acva_Arabic_Wedding + - arabic_leaderboard_acva_Bahrain + - arabic_leaderboard_acva_Comoros + - arabic_leaderboard_acva_Egypt_modern + - arabic_leaderboard_acva_InfluenceFromAncientEgypt + - arabic_leaderboard_acva_InfluenceFromByzantium + - arabic_leaderboard_acva_InfluenceFromChina + - arabic_leaderboard_acva_InfluenceFromGreece + - arabic_leaderboard_acva_InfluenceFromIslam + - arabic_leaderboard_acva_InfluenceFromPersia + - arabic_leaderboard_acva_InfluenceFromRome + - arabic_leaderboard_acva_Iraq + - arabic_leaderboard_acva_Islam_Education + - arabic_leaderboard_acva_Islam_branches_and_schools + - arabic_leaderboard_acva_Islamic_law_system + - arabic_leaderboard_acva_Jordan + - arabic_leaderboard_acva_Kuwait + - arabic_leaderboard_acva_Lebanon + - arabic_leaderboard_acva_Libya + - arabic_leaderboard_acva_Mauritania + - arabic_leaderboard_acva_Mesopotamia_civilization + - arabic_leaderboard_acva_Morocco + - arabic_leaderboard_acva_Oman + - arabic_leaderboard_acva_Palestine + - arabic_leaderboard_acva_Qatar + - arabic_leaderboard_acva_Saudi_Arabia + - arabic_leaderboard_acva_Somalia + - arabic_leaderboard_acva_Sudan + - arabic_leaderboard_acva_Syria + - arabic_leaderboard_acva_Tunisia + - arabic_leaderboard_acva_United_Arab_Emirates + - arabic_leaderboard_acva_Yemen + - arabic_leaderboard_acva_communication + - arabic_leaderboard_acva_computer_and_phone + - arabic_leaderboard_acva_daily_life + - arabic_leaderboard_acva_entertainment + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Algeria.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Algeria.yaml new file mode 100644 index 0000000000..177161edaa --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Algeria.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Algeria +dataset_path: OALL/ACVA +dataset_name: Algeria +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Ancient_Egypt.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Ancient_Egypt.yaml new file mode 100644 index 0000000000..ddb5c35555 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Ancient_Egypt.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Ancient_Egypt +dataset_path: OALL/ACVA +dataset_name: Ancient_Egypt +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arab_Empire.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arab_Empire.yaml new file mode 100644 index 0000000000..b510de5ab9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arab_Empire.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arab_Empire +dataset_path: OALL/ACVA +dataset_name: Arab_Empire +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Architecture.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Architecture.yaml new file mode 100644 index 0000000000..5dc2c07dee --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Architecture.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Architecture +dataset_path: OALL/ACVA +dataset_name: Arabic_Architecture +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Art.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Art.yaml new file mode 100644 index 0000000000..36f364bc50 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Art.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Art +dataset_path: OALL/ACVA +dataset_name: Arabic_Art +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Astronomy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Astronomy.yaml new file mode 100644 index 0000000000..f90b1c9140 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Astronomy.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Astronomy +dataset_path: OALL/ACVA +dataset_name: Arabic_Astronomy +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Calligraphy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Calligraphy.yaml new file mode 100644 index 0000000000..dfdf51878b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Calligraphy.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Calligraphy +dataset_path: OALL/ACVA +dataset_name: Arabic_Calligraphy +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Ceremony.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Ceremony.yaml new file mode 100644 index 0000000000..c20b4439e2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Ceremony.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Ceremony +dataset_path: OALL/ACVA +dataset_name: Arabic_Ceremony +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Clothing.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Clothing.yaml new file mode 100644 index 0000000000..06118034dc --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Clothing.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Clothing +dataset_path: OALL/ACVA +dataset_name: Arabic_Clothing +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Culture.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Culture.yaml new file mode 100644 index 0000000000..cea33022b4 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Culture.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Culture +dataset_path: OALL/ACVA +dataset_name: Arabic_Culture +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Food.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Food.yaml new file mode 100644 index 0000000000..cca516c972 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Food.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Food +dataset_path: OALL/ACVA +dataset_name: Arabic_Food +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Funeral.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Funeral.yaml new file mode 100644 index 0000000000..3dd8fbedd9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Funeral.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Funeral +dataset_path: OALL/ACVA +dataset_name: Arabic_Funeral +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Geography.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Geography.yaml new file mode 100644 index 0000000000..89aa7361b3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Geography.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Geography +dataset_path: OALL/ACVA +dataset_name: Arabic_Geography +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_History.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_History.yaml new file mode 100644 index 0000000000..776589c07b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_History.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_History +dataset_path: OALL/ACVA +dataset_name: Arabic_History +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Language_Origin.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Language_Origin.yaml new file mode 100644 index 0000000000..4f0612acaf --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Language_Origin.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Language_Origin +dataset_path: OALL/ACVA +dataset_name: Arabic_Language_Origin +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Literature.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Literature.yaml new file mode 100644 index 0000000000..0c9198f446 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Literature.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Literature +dataset_path: OALL/ACVA +dataset_name: Arabic_Literature +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Math.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Math.yaml new file mode 100644 index 0000000000..02a3643024 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Math.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Math +dataset_path: OALL/ACVA +dataset_name: Arabic_Math +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Medicine.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Medicine.yaml new file mode 100644 index 0000000000..109aae994a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Medicine.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Medicine +dataset_path: OALL/ACVA +dataset_name: Arabic_Medicine +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Music.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Music.yaml new file mode 100644 index 0000000000..2559625784 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Music.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Music +dataset_path: OALL/ACVA +dataset_name: Arabic_Music +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Ornament.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Ornament.yaml new file mode 100644 index 0000000000..00311e107e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Ornament.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Ornament +dataset_path: OALL/ACVA +dataset_name: Arabic_Ornament +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Philosophy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Philosophy.yaml new file mode 100644 index 0000000000..62a570f00c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Philosophy.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Philosophy +dataset_path: OALL/ACVA +dataset_name: Arabic_Philosophy +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Physics_and_Chemistry.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Physics_and_Chemistry.yaml new file mode 100644 index 0000000000..b1b52096e4 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Physics_and_Chemistry.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Physics_and_Chemistry +dataset_path: OALL/ACVA +dataset_name: Arabic_Physics_and_Chemistry +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Wedding.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Wedding.yaml new file mode 100644 index 0000000000..21205cfff8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Wedding.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Wedding +dataset_path: OALL/ACVA +dataset_name: Arabic_Wedding +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Bahrain.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Bahrain.yaml new file mode 100644 index 0000000000..3b2481bc87 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Bahrain.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Bahrain +dataset_path: OALL/ACVA +dataset_name: Bahrain +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Comoros.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Comoros.yaml new file mode 100644 index 0000000000..be4df372c7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Comoros.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Comoros +dataset_path: OALL/ACVA +dataset_name: Comoros +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Egypt_modern.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Egypt_modern.yaml new file mode 100644 index 0000000000..26ca2f6e08 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Egypt_modern.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Egypt_modern +dataset_path: OALL/ACVA +dataset_name: Egypt_modern +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromAncientEgypt.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromAncientEgypt.yaml new file mode 100644 index 0000000000..be300fc869 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromAncientEgypt.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromAncientEgypt +dataset_path: OALL/ACVA +dataset_name: InfluenceFromAncientEgypt +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromByzantium.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromByzantium.yaml new file mode 100644 index 0000000000..72c86a6247 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromByzantium.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromByzantium +dataset_path: OALL/ACVA +dataset_name: InfluenceFromByzantium +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromChina.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromChina.yaml new file mode 100644 index 0000000000..b297642cbe --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromChina.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromChina +dataset_path: OALL/ACVA +dataset_name: InfluenceFromChina +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromGreece.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromGreece.yaml new file mode 100644 index 0000000000..70458ea2d3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromGreece.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromGreece +dataset_path: OALL/ACVA +dataset_name: InfluenceFromGreece +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromIslam.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromIslam.yaml new file mode 100644 index 0000000000..803f33345d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromIslam.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromIslam +dataset_path: OALL/ACVA +dataset_name: InfluenceFromIslam +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromPersia.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromPersia.yaml new file mode 100644 index 0000000000..117ca89079 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromPersia.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromPersia +dataset_path: OALL/ACVA +dataset_name: InfluenceFromPersia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromRome.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromRome.yaml new file mode 100644 index 0000000000..1655522e5a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromRome.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromRome +dataset_path: OALL/ACVA +dataset_name: InfluenceFromRome +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Iraq.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Iraq.yaml new file mode 100644 index 0000000000..909c6678c7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Iraq.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Iraq +dataset_path: OALL/ACVA +dataset_name: Iraq +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islam_Education.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islam_Education.yaml new file mode 100644 index 0000000000..13c1fab2a0 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islam_Education.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Islam_Education +dataset_path: OALL/ACVA +dataset_name: Islam_Education +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islam_branches_and_schools.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islam_branches_and_schools.yaml new file mode 100644 index 0000000000..6985b24a74 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islam_branches_and_schools.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Islam_branches_and_schools +dataset_path: OALL/ACVA +dataset_name: Islam_branches_and_schools +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islamic_law_system.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islamic_law_system.yaml new file mode 100644 index 0000000000..d19a52ba03 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islamic_law_system.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Islamic_law_system +dataset_path: OALL/ACVA +dataset_name: Islamic_law_system +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Jordan.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Jordan.yaml new file mode 100644 index 0000000000..7bff93a94c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Jordan.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Jordan +dataset_path: OALL/ACVA +dataset_name: Jordan +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Kuwait.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Kuwait.yaml new file mode 100644 index 0000000000..b1ae77aaa5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Kuwait.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Kuwait +dataset_path: OALL/ACVA +dataset_name: Kuwait +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Lebanon.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Lebanon.yaml new file mode 100644 index 0000000000..65974b74dc --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Lebanon.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Lebanon +dataset_path: OALL/ACVA +dataset_name: Lebanon +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Libya.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Libya.yaml new file mode 100644 index 0000000000..c8b339650c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Libya.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Libya +dataset_path: OALL/ACVA +dataset_name: Libya +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Mauritania.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Mauritania.yaml new file mode 100644 index 0000000000..1b84074abc --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Mauritania.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Mauritania +dataset_path: OALL/ACVA +dataset_name: Mauritania +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Mesopotamia_civilization.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Mesopotamia_civilization.yaml new file mode 100644 index 0000000000..4218947702 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Mesopotamia_civilization.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Mesopotamia_civilization +dataset_path: OALL/ACVA +dataset_name: Mesopotamia_civilization +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Morocco.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Morocco.yaml new file mode 100644 index 0000000000..4ed1510bb5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Morocco.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Morocco +dataset_path: OALL/ACVA +dataset_name: Morocco +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Oman.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Oman.yaml new file mode 100644 index 0000000000..b534cfb19f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Oman.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Oman +dataset_path: OALL/ACVA +dataset_name: Oman +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Palestine.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Palestine.yaml new file mode 100644 index 0000000000..1cb9b56a85 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Palestine.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Palestine +dataset_path: OALL/ACVA +dataset_name: Palestine +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Qatar.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Qatar.yaml new file mode 100644 index 0000000000..5d5775ccd9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Qatar.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Qatar +dataset_path: OALL/ACVA +dataset_name: Qatar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Saudi_Arabia.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Saudi_Arabia.yaml new file mode 100644 index 0000000000..5010723661 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Saudi_Arabia.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Saudi_Arabia +dataset_path: OALL/ACVA +dataset_name: Saudi_Arabia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Somalia.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Somalia.yaml new file mode 100644 index 0000000000..d40b578221 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Somalia.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Somalia +dataset_path: OALL/ACVA +dataset_name: Somalia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Sudan.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Sudan.yaml new file mode 100644 index 0000000000..e7c2f41a3b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Sudan.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Sudan +dataset_path: OALL/ACVA +dataset_name: Sudan +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Syria.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Syria.yaml new file mode 100644 index 0000000000..98ebff9fca --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Syria.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Syria +dataset_path: OALL/ACVA +dataset_name: Syria +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Tunisia.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Tunisia.yaml new file mode 100644 index 0000000000..d86e428cc3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Tunisia.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Tunisia +dataset_path: OALL/ACVA +dataset_name: Tunisia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_United_Arab_Emirates.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_United_Arab_Emirates.yaml new file mode 100644 index 0000000000..f41b625508 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_United_Arab_Emirates.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_United_Arab_Emirates +dataset_path: OALL/ACVA +dataset_name: United_Arab_Emirates +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Yemen.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Yemen.yaml new file mode 100644 index 0000000000..b239dd514a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Yemen.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Yemen +dataset_path: OALL/ACVA +dataset_name: Yemen +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_communication.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_communication.yaml new file mode 100644 index 0000000000..beb954efce --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_communication.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_communication +dataset_path: OALL/ACVA +dataset_name: communication +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_computer_and_phone.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_computer_and_phone.yaml new file mode 100644 index 0000000000..888f82af92 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_computer_and_phone.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_computer_and_phone +dataset_path: OALL/ACVA +dataset_name: computer_and_phone +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_daily_life.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_daily_life.yaml new file mode 100644 index 0000000000..0b4748a297 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_daily_life.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_daily_life +dataset_path: OALL/ACVA +dataset_name: daily_life +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_entertainment.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_entertainment.yaml new file mode 100644 index 0000000000..b2adcfb954 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_entertainment.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_entertainment +dataset_path: OALL/ACVA +dataset_name: entertainment +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/utils.py new file mode 100644 index 0000000000..7e91496f59 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/utils.py @@ -0,0 +1,16 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["question"] + answer = doc["answer"] + + return { + "query": f"السؤال: {question}\nالإجابة:", + "choices": ["صح", "خطأ"], + "gold": ["صح", "خطأ"].index(answer), + } + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_complete.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_complete.yaml new file mode 100644 index 0000000000..c26370157d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_complete.yaml @@ -0,0 +1,25 @@ +group: arabic_leaderboard_complete +task: + - arabic_leaderboard_acva + - arabic_leaderboard_alghafa + - arabic_leaderboard_arabic_exams + - arabic_leaderboard_arabic_mt_arc_challenge + - arabic_leaderboard_arabic_mt_arc_easy + - arabic_leaderboard_arabic_mt_boolq + - arabic_leaderboard_arabic_mt_hellaswag + - arabic_leaderboard_arabic_mt_mmlu + - arabic_leaderboard_arabic_mt_copa + - arabic_leaderboard_arabic_mt_openbook_qa + - arabic_leaderboard_arabic_mt_piqa + - arabic_leaderboard_arabic_mt_race + - arabic_leaderboard_arabic_mt_sciq + - arabic_leaderboard_arabic_mt_toxigen +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/README.md b/lm_eval/tasks/arabic_leaderboard_light/README.md new file mode 100644 index 0000000000..199aa2c8da --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/README.md @@ -0,0 +1,20 @@ +# Arabic Leaderboard Light + +Title: Open Arabic LLM Leaderboard Light + +This leaderboard follows all the details as in [`arabic_leaderboard_complete`](../arabic_leaderboard_complete), except that a light version - 10% random sample of the test set of each benchmark - is used to test the language models. + +NOTE: In ACVA benchmark, there is Yemen subset, and it is a small dataset - it has only 10 samples in the test split. So, for this specific subset dataset, to have more reliable results, we consider the original dataset, instead of 10% of its test samples. + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [ ] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_light.yaml new file mode 100644 index 0000000000..0ee6a568d9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_light.yaml @@ -0,0 +1,23 @@ +group: arabic_leaderboard_alghafa_light +task: + - arabic_leaderboard_alghafa_mcq_exams_test_ar_light + - arabic_leaderboard_alghafa_meta_ar_dialects_light + - arabic_leaderboard_alghafa_meta_ar_msa_light + - arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task_light + - arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task_light + - arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task_light + - arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task_light + - arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task_light + - arabic_leaderboard_alghafa_multiple_choice_sentiment_task_light + + + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_mcq_exams_test_ar_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_mcq_exams_test_ar_light.yaml new file mode 100644 index 0000000000..1fdda36405 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_mcq_exams_test_ar_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_mcq_exams_test_ar_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: mcq_exams_test_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_meta_ar_dialects_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_meta_ar_dialects_light.yaml new file mode 100644 index 0000000000..47af55b86a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_meta_ar_dialects_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_meta_ar_dialects_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: meta_ar_dialects +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_meta_ar_msa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_meta_ar_msa_light.yaml new file mode 100644 index 0000000000..9a26a2653f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_meta_ar_msa_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_meta_ar_msa_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: meta_ar_msa +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task_light.yaml new file mode 100644 index 0000000000..b56ddfee19 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: multiple_choice_facts_truefalse_balanced_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task_light.yaml new file mode 100644 index 0000000000..4d85c68491 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: multiple_choice_grounded_statement_soqal_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task_light.yaml new file mode 100644 index 0000000000..e5d8afefea --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: multiple_choice_grounded_statement_xglue_mlqa_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task_light.yaml new file mode 100644 index 0000000000..21721d2a2d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: multiple_choice_rating_sentiment_no_neutral_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task_light.yaml new file mode 100644 index 0000000000..39f72e4d2a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: multiple_choice_rating_sentiment_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_sentiment_task_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_sentiment_task_light.yaml new file mode 100644 index 0000000000..28b0701561 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_sentiment_task_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_sentiment_task_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: multiple_choice_sentiment_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/utils.py new file mode 100644 index 0000000000..62f9874e63 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_exams_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_exams_light.yaml new file mode 100644 index 0000000000..2348be4eb3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_exams_light.yaml @@ -0,0 +1,23 @@ +task: arabic_exams_light +dataset_path: arcee-globe/Arabic_EXAMS-10percent +dataset_name: default +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_leaderboard_arabic_exams_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_leaderboard_arabic_exams_light.yaml new file mode 100644 index 0000000000..296a47cbb4 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_leaderboard_arabic_exams_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_exams_light +task: + - arabic_exams_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/utils.py new file mode 100644 index 0000000000..72af1c40fe --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/utils.py @@ -0,0 +1,33 @@ +import datasets +import numpy as np + + +# fmt: off +LETTER_INDICES_AR = ["أ", "ب", "ج", "د", "هـ", "و", "ز", "ح", "ط", "ي", "ك", "ل", "م", "ن", "س", "ع", "ف", "ص", "ق", "ر", "ش", "ت", "ث", "خ", "ذ", "ض", "ظ", "غ"] +# fmt: on + + +# fmt: off +LETTER_INDICES = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"] +# fmt: on + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + topic = doc["subject"] + question = doc["question"] + choices = [doc["A"], doc["B"], doc["C"], doc["D"]] + choices_formatted = [ + f" {LETTER_INDICES_AR[i]}) {choice}\n" for i, choice in enumerate(choices) + ] + answer = doc["answer"] + answer_index = LETTER_INDICES.index(answer) + + instruction = f"الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح حول {topic.replace('_', ' ')}. \n\n" + query = f"{instruction}السؤال: {question}\n" + query += "\n".join(choices_formatted) + query += "\nالإجابة:" + + return {"query": query, "choices": LETTER_INDICES_AR[:4], "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_abstract_algebra_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_abstract_algebra_light.yaml new file mode 100644 index 0000000000..dcb59fc361 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_abstract_algebra_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_abstract_algebra_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: abstract_algebra +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_anatomy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_anatomy_light.yaml new file mode 100644 index 0000000000..fc77a66dde --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_anatomy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_anatomy_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: anatomy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_astronomy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_astronomy_light.yaml new file mode 100644 index 0000000000..db4a9b4360 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_astronomy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_astronomy_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: astronomy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_business_ethics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_business_ethics_light.yaml new file mode 100644 index 0000000000..a747dbafaf --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_business_ethics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_business_ethics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: business_ethics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_clinical_knowledge_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_clinical_knowledge_light.yaml new file mode 100644 index 0000000000..1296b90cbc --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_clinical_knowledge_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_clinical_knowledge_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: clinical_knowledge +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_biology_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_biology_light.yaml new file mode 100644 index 0000000000..cbfc804974 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_biology_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_biology_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: college_biology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_chemistry_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_chemistry_light.yaml new file mode 100644 index 0000000000..ac0970355b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_chemistry_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_chemistry_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: college_chemistry +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_computer_science_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_computer_science_light.yaml new file mode 100644 index 0000000000..361274d64a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_computer_science_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_computer_science_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: college_computer_science +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_mathematics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_mathematics_light.yaml new file mode 100644 index 0000000000..20e4d6e627 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_mathematics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_mathematics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: college_mathematics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_medicine_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_medicine_light.yaml new file mode 100644 index 0000000000..d854004973 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_medicine_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_medicine_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: college_medicine +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_physics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_physics_light.yaml new file mode 100644 index 0000000000..57e4b55033 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_physics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_physics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: college_physics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_computer_security_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_computer_security_light.yaml new file mode 100644 index 0000000000..dd8c01dc6c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_computer_security_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_computer_security_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: computer_security +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_conceptual_physics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_conceptual_physics_light.yaml new file mode 100644 index 0000000000..cffd7ee42d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_conceptual_physics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_conceptual_physics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: conceptual_physics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_econometrics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_econometrics_light.yaml new file mode 100644 index 0000000000..30413feff0 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_econometrics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_econometrics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: econometrics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_electrical_engineering_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_electrical_engineering_light.yaml new file mode 100644 index 0000000000..e60787d675 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_electrical_engineering_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_electrical_engineering_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: electrical_engineering +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_elementary_mathematics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_elementary_mathematics_light.yaml new file mode 100644 index 0000000000..571476620a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_elementary_mathematics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_elementary_mathematics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: elementary_mathematics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_formal_logic_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_formal_logic_light.yaml new file mode 100644 index 0000000000..9b2bebf1e5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_formal_logic_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_formal_logic_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: formal_logic +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_global_facts_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_global_facts_light.yaml new file mode 100644 index 0000000000..15c3b34aac --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_global_facts_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_global_facts_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: global_facts +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_biology_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_biology_light.yaml new file mode 100644 index 0000000000..906c33284d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_biology_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_biology_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_biology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_chemistry_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_chemistry_light.yaml new file mode 100644 index 0000000000..199f16b093 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_chemistry_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_chemistry_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_chemistry +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_computer_science_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_computer_science_light.yaml new file mode 100644 index 0000000000..cb23af53bb --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_computer_science_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_computer_science_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_computer_science +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_european_history_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_european_history_light.yaml new file mode 100644 index 0000000000..25a9b46695 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_european_history_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_european_history_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_european_history +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_geography_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_geography_light.yaml new file mode 100644 index 0000000000..f7f39cd2f2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_geography_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_geography_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_geography +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_government_and_politics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_government_and_politics_light.yaml new file mode 100644 index 0000000000..dff09d6717 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_government_and_politics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_government_and_politics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_government_and_politics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_macroeconomics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_macroeconomics_light.yaml new file mode 100644 index 0000000000..ae42622353 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_macroeconomics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_macroeconomics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_macroeconomics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_mathematics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_mathematics_light.yaml new file mode 100644 index 0000000000..8adc3d7e93 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_mathematics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_mathematics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_mathematics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_microeconomics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_microeconomics_light.yaml new file mode 100644 index 0000000000..6eec39237b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_microeconomics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_microeconomics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_microeconomics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_physics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_physics_light.yaml new file mode 100644 index 0000000000..973bd1ffc5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_physics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_physics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_physics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_psychology_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_psychology_light.yaml new file mode 100644 index 0000000000..614dd7e89d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_psychology_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_psychology_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_psychology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_statistics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_statistics_light.yaml new file mode 100644 index 0000000000..2db9f196a3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_statistics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_statistics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_statistics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_us_history_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_us_history_light.yaml new file mode 100644 index 0000000000..5411e8c479 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_us_history_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_us_history_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_us_history +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_world_history_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_world_history_light.yaml new file mode 100644 index 0000000000..319c49b22b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_world_history_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_world_history_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_world_history +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_human_aging_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_human_aging_light.yaml new file mode 100644 index 0000000000..afd2eefa29 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_human_aging_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_human_aging_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: human_aging +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_human_sexuality_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_human_sexuality_light.yaml new file mode 100644 index 0000000000..9e245f2687 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_human_sexuality_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_human_sexuality_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: human_sexuality +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_international_law_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_international_law_light.yaml new file mode 100644 index 0000000000..6e476bb879 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_international_law_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_international_law_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: international_law +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_jurisprudence_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_jurisprudence_light.yaml new file mode 100644 index 0000000000..1d848cd173 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_jurisprudence_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_jurisprudence_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: jurisprudence +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_light.yaml new file mode 100644 index 0000000000..130713702c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_light.yaml @@ -0,0 +1,68 @@ +group: arabic_leaderboard_arabic_mmlu_light +task: + - arabic_leaderboard_arabic_mmlu_abstract_algebra_light + - arabic_leaderboard_arabic_mmlu_anatomy_light + - arabic_leaderboard_arabic_mmlu_astronomy_light + - arabic_leaderboard_arabic_mmlu_business_ethics_light + - arabic_leaderboard_arabic_mmlu_clinical_knowledge_light + - arabic_leaderboard_arabic_mmlu_college_biology_light + - arabic_leaderboard_arabic_mmlu_college_chemistry_light + - arabic_leaderboard_arabic_mmlu_college_computer_science_light + - arabic_leaderboard_arabic_mmlu_college_mathematics_light + - arabic_leaderboard_arabic_mmlu_college_medicine_light + - arabic_leaderboard_arabic_mmlu_college_physics_light + - arabic_leaderboard_arabic_mmlu_computer_security_light + - arabic_leaderboard_arabic_mmlu_conceptual_physics_light + - arabic_leaderboard_arabic_mmlu_econometrics_light + - arabic_leaderboard_arabic_mmlu_electrical_engineering_light + - arabic_leaderboard_arabic_mmlu_elementary_mathematics_light + - arabic_leaderboard_arabic_mmlu_formal_logic_light + - arabic_leaderboard_arabic_mmlu_global_facts_light + - arabic_leaderboard_arabic_mmlu_high_school_biology_light + - arabic_leaderboard_arabic_mmlu_high_school_chemistry_light + - arabic_leaderboard_arabic_mmlu_high_school_computer_science_light + - arabic_leaderboard_arabic_mmlu_high_school_european_history_light + - arabic_leaderboard_arabic_mmlu_high_school_geography_light + - arabic_leaderboard_arabic_mmlu_high_school_government_and_politics_light + - arabic_leaderboard_arabic_mmlu_high_school_macroeconomics_light + - arabic_leaderboard_arabic_mmlu_high_school_mathematics_light + - arabic_leaderboard_arabic_mmlu_high_school_microeconomics_light + - arabic_leaderboard_arabic_mmlu_high_school_physics_light + - arabic_leaderboard_arabic_mmlu_high_school_psychology_light + - arabic_leaderboard_arabic_mmlu_high_school_statistics_light + - arabic_leaderboard_arabic_mmlu_high_school_us_history_light + - arabic_leaderboard_arabic_mmlu_high_school_world_history_light + - arabic_leaderboard_arabic_mmlu_human_aging_light + - arabic_leaderboard_arabic_mmlu_human_sexuality_light + - arabic_leaderboard_arabic_mmlu_international_law_light + - arabic_leaderboard_arabic_mmlu_jurisprudence_light + - arabic_leaderboard_arabic_mmlu_logical_fallacies_light + - arabic_leaderboard_arabic_mmlu_machine_learning_light + - arabic_leaderboard_arabic_mmlu_management_light + - arabic_leaderboard_arabic_mmlu_marketing_light + - arabic_leaderboard_arabic_mmlu_medical_genetics_light + - arabic_leaderboard_arabic_mmlu_miscellaneous_light + - arabic_leaderboard_arabic_mmlu_moral_disputes_light + - arabic_leaderboard_arabic_mmlu_moral_scenarios_light + - arabic_leaderboard_arabic_mmlu_nutrition_light + - arabic_leaderboard_arabic_mmlu_philosophy_light + - arabic_leaderboard_arabic_mmlu_prehistory_light + - arabic_leaderboard_arabic_mmlu_professional_accounting_light + - arabic_leaderboard_arabic_mmlu_professional_law_light + - arabic_leaderboard_arabic_mmlu_professional_medicine_light + - arabic_leaderboard_arabic_mmlu_professional_psychology_light + - arabic_leaderboard_arabic_mmlu_public_relations_light + - arabic_leaderboard_arabic_mmlu_security_studies_light + - arabic_leaderboard_arabic_mmlu_sociology_light + - arabic_leaderboard_arabic_mmlu_us_foreign_policy_light + - arabic_leaderboard_arabic_mmlu_virology_light + - arabic_leaderboard_arabic_mmlu_world_religions_light +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_logical_fallacies_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_logical_fallacies_light.yaml new file mode 100644 index 0000000000..866420ba28 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_logical_fallacies_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_logical_fallacies_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: logical_fallacies +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_machine_learning_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_machine_learning_light.yaml new file mode 100644 index 0000000000..01ed181e01 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_machine_learning_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_machine_learning_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: machine_learning +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_management_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_management_light.yaml new file mode 100644 index 0000000000..62d7e32ab0 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_management_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_management_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: management +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_marketing_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_marketing_light.yaml new file mode 100644 index 0000000000..c42f7a177b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_marketing_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_marketing_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: marketing +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_medical_genetics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_medical_genetics_light.yaml new file mode 100644 index 0000000000..40d0d88326 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_medical_genetics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_medical_genetics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: medical_genetics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_miscellaneous_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_miscellaneous_light.yaml new file mode 100644 index 0000000000..06bc6a4715 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_miscellaneous_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_miscellaneous_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: miscellaneous +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_moral_disputes_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_moral_disputes_light.yaml new file mode 100644 index 0000000000..be0c60e631 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_moral_disputes_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_moral_disputes_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: moral_disputes +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_moral_scenarios_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_moral_scenarios_light.yaml new file mode 100644 index 0000000000..08e71366d1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_moral_scenarios_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_moral_scenarios_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: moral_scenarios +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_nutrition_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_nutrition_light.yaml new file mode 100644 index 0000000000..7987f5f36c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_nutrition_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_nutrition_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: nutrition +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_philosophy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_philosophy_light.yaml new file mode 100644 index 0000000000..85ebdd7a4f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_philosophy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_philosophy_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: philosophy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_prehistory_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_prehistory_light.yaml new file mode 100644 index 0000000000..24aa8e22fe --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_prehistory_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_prehistory_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: prehistory +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_accounting_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_accounting_light.yaml new file mode 100644 index 0000000000..1dc009663c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_accounting_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_accounting_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: professional_accounting +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_law_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_law_light.yaml new file mode 100644 index 0000000000..6e8c3617db --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_law_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_law_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: professional_law +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_medicine_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_medicine_light.yaml new file mode 100644 index 0000000000..b90cdb38d8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_medicine_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_medicine_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: professional_medicine +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_psychology_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_psychology_light.yaml new file mode 100644 index 0000000000..420a536243 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_psychology_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_psychology_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: professional_psychology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_public_relations_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_public_relations_light.yaml new file mode 100644 index 0000000000..83d267bc08 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_public_relations_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_public_relations_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: public_relations +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_security_studies_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_security_studies_light.yaml new file mode 100644 index 0000000000..03e05d66e7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_security_studies_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_security_studies_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: security_studies +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_sociology_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_sociology_light.yaml new file mode 100644 index 0000000000..7deb088396 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_sociology_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_sociology_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: sociology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_us_foreign_policy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_us_foreign_policy_light.yaml new file mode 100644 index 0000000000..6c5f40a55e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_us_foreign_policy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_us_foreign_policy_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: us_foreign_policy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_virology_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_virology_light.yaml new file mode 100644 index 0000000000..5ee4a7c95b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_virology_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_virology_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: virology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_world_religions_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_world_religions_light.yaml new file mode 100644 index 0000000000..57b13f05b8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_world_religions_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_world_religions_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: world_religions +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/utils.py new file mode 100644 index 0000000000..da927b66fc --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/utils.py @@ -0,0 +1,35 @@ +import datasets +import numpy as np + + +# fmt: off +LETTER_INDICES_AR = ["أ", "ب", "ج", "د", "هـ", "و", "ز", "ح", "ط", "ي", "ك", "ل", "م", "ن", "س", "ع", "ف", "ص", "ق", "ر", "ش", "ت", "ث", "خ", "ذ", "ض", "ظ", "غ"] +# fmt: on + + +# fmt: off +LETTER_INDICES = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"] +# fmt: on + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + topic = doc["subject"] + instruction = f"الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح حول {topic.replace('_', ' ')}. \n\n" + choices = [doc["A"], doc["B"], doc["C"], doc["D"]] + # Answers are provided with roman letters - we look for the correct index in LETTER_INDICES, + # it will then be applied to arabic letters + gold_ix = LETTER_INDICES.index(doc["answer"]) + + query = f"{instruction}{doc['question']}\n" + query += "".join( + [ + f"{key}. {choice}\n" + for key, choice in zip(LETTER_INDICES_AR[:4], choices) + ] + ) + query += "الإجابة:" + + return {"query": query, "choices": LETTER_INDICES_AR[:4], "gold": gold_ix} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/arabic_leaderboard_arabic_mt_arc_challenge_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/arabic_leaderboard_arabic_mt_arc_challenge_light.yaml new file mode 100644 index 0000000000..a88bd6bd9e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/arabic_leaderboard_arabic_mt_arc_challenge_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_arc_challenge_light +task: + - arabic_mt_arc_challenge_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/arabic_mt_arc_challenge_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/arabic_mt_arc_challenge_light.yaml new file mode 100644 index 0000000000..e6b299e846 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/arabic_mt_arc_challenge_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_arc_challenge_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: arc_challenge_okapi_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/utils.py new file mode 100644 index 0000000000..62f9874e63 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/arabic_leaderboard_arabic_mt_arc_easy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/arabic_leaderboard_arabic_mt_arc_easy_light.yaml new file mode 100644 index 0000000000..618b542952 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/arabic_leaderboard_arabic_mt_arc_easy_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_arc_easy_light +task: + - arabic_mt_arc_easy_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/arabic_mt_arc_easy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/arabic_mt_arc_easy_light.yaml new file mode 100644 index 0000000000..90252fb31d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/arabic_mt_arc_easy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_arc_easy_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: arc_easy_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/utils.py new file mode 100644 index 0000000000..62f9874e63 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/arabic_leaderboard_arabic_mt_boolq_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/arabic_leaderboard_arabic_mt_boolq_light.yaml new file mode 100644 index 0000000000..ee02f9cbc9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/arabic_leaderboard_arabic_mt_boolq_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_boolq_light +task: + - arabic_mt_boolq_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/arabic_mt_boolq_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/arabic_mt_boolq_light.yaml new file mode 100644 index 0000000000..4bdd145ce6 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/arabic_mt_boolq_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_boolq_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: boolq_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/utils.py new file mode 100644 index 0000000000..dcbc10d92e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/utils.py @@ -0,0 +1,24 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["question"] + passage = doc["passage"] + instruction = "بناء على المقطع التالي، أجب عن السؤال ب نعم أو لا" + query = f"""{instruction} + المقطع : + {passage} + السؤال: + {question} + الإجابة: + """ + + return { + "query": query, + "choices": ["نعم", "لا"], + "gold": 0 if doc["answer"] else 1, + } + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/arabic_mt_copa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/arabic_mt_copa_light.yaml new file mode 100644 index 0000000000..0ca475e735 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/arabic_mt_copa_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_copa_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: copa_ext_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/arbic_leaderboard_arabic_mt_copa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/arbic_leaderboard_arabic_mt_copa_light.yaml new file mode 100644 index 0000000000..f3ea35bc50 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/arbic_leaderboard_arabic_mt_copa_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_copa_light +task: + - arabic_mt_copa_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/utils.py new file mode 100644 index 0000000000..175ebdadc1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/utils.py @@ -0,0 +1,19 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + premise = doc["premise"] + choices = [doc["choice1"], doc["choice2"]] + question_map = {"cause": "لأن", "effect": "لذلك"} + question = question_map[doc["question"]] + answer = doc["label"] + + query = "{}، {} :\n0) {}\n1) {}\nالإجابة:".format( + premise, question, choices[0], choices[1] + ) + + return {"query": query, "choices": choices, "gold": answer} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/arabic_leaderboard_arabic_mt_hellaswag_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/arabic_leaderboard_arabic_mt_hellaswag_light.yaml new file mode 100644 index 0000000000..0f44bbbc75 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/arabic_leaderboard_arabic_mt_hellaswag_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_hellaswag_light +task: + - arabic_mt_hellaswag_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/arabic_mt_hellaswag_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/arabic_mt_hellaswag_light.yaml new file mode 100644 index 0000000000..56ea04f248 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/arabic_mt_hellaswag_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_hellaswag_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: hellaswag_okapi_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/utils.py new file mode 100644 index 0000000000..6b5a9f1f4f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/utils.py @@ -0,0 +1,30 @@ +import re + +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + ctx = re.sub(r"\[.*?\]", "", doc["ctx"]) # Remove latin words within brackets + endings = [ + re.sub(r"\[.*?\]", "", e) for e in eval(doc["endings"]) + ] # endings is a string representation of a list + answer_index = doc["label"] + instruction = ( + "بناء على السياق التالي، اختر النهاية الصحيحة من الاقتراحات التالية" + ) + + query = f"""{instruction} + السياق: + {ctx} + الاقتراحات: + + """ + for i, ending in enumerate(endings): + query += f"{i}) {ending}\n" + query += "الإجابة:" + + return {"query": query, "choices": endings, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/arabic_leaderboard_arabic_mt_mmlu_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/arabic_leaderboard_arabic_mt_mmlu_light.yaml new file mode 100644 index 0000000000..b95ca1b531 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/arabic_leaderboard_arabic_mt_mmlu_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_mmlu_light +task: + - arabic_mt_mmlu_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/arabic_mt_mmlu_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/arabic_mt_mmlu_light.yaml new file mode 100644 index 0000000000..43084db30b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/arabic_mt_mmlu_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_mmlu_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: mmlu_okapi_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/utils.py new file mode 100644 index 0000000000..62f9874e63 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/arabic_leaderboard_arabic_mt_openbook_qa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/arabic_leaderboard_arabic_mt_openbook_qa_light.yaml new file mode 100644 index 0000000000..3737f621fb --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/arabic_leaderboard_arabic_mt_openbook_qa_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_openbook_qa_light +task: + - arabic_mt_openbook_qa_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/arabic_mt_openbook_qa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/arabic_mt_openbook_qa_light.yaml new file mode 100644 index 0000000000..5e914fbd32 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/arabic_mt_openbook_qa_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_openbook_qa_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: openbook_qa_ext_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/utils.py new file mode 100644 index 0000000000..62f9874e63 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/arabic_leaderboard_arabic_mt_piqa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/arabic_leaderboard_arabic_mt_piqa_light.yaml new file mode 100644 index 0000000000..642b2e0a60 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/arabic_leaderboard_arabic_mt_piqa_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_piqa_light +task: + - arabic_mt_piqa_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/arabic_mt_piqa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/arabic_mt_piqa_light.yaml new file mode 100644 index 0000000000..4dd9e005a9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/arabic_mt_piqa_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_piqa_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: piqa_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/utils.py new file mode 100644 index 0000000000..62f9874e63 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/arabic_leaderboard_arabic_mt_race_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/arabic_leaderboard_arabic_mt_race_light.yaml new file mode 100644 index 0000000000..8f427484d1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/arabic_leaderboard_arabic_mt_race_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_race_light +task: + - arabic_mt_race_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/arabic_mt_race_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/arabic_mt_race_light.yaml new file mode 100644 index 0000000000..fed452cce6 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/arabic_mt_race_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_race_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: race_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/utils.py new file mode 100644 index 0000000000..62f9874e63 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/arabic_leaderboard_arabic_mt_sciq_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/arabic_leaderboard_arabic_mt_sciq_light.yaml new file mode 100644 index 0000000000..13127e9915 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/arabic_leaderboard_arabic_mt_sciq_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_sciq_light +task: + - arabic_mt_sciq_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/arabic_mt_sciq_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/arabic_mt_sciq_light.yaml new file mode 100644 index 0000000000..95976cbb7f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/arabic_mt_sciq_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_sciq_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: sciq_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/utils.py new file mode 100644 index 0000000000..ddb42eeb8c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/utils.py @@ -0,0 +1,41 @@ +import random + +import datasets +import numpy as np + + +def doc_to_text(doc): + instruction = ( + "بناءً على السياق أدناه، اختر الإجابة الصحيحة للسؤال التالي من قائمة الاقتراحات" + ) + support = doc["support"] + question = doc["question"] + query = f"""{instruction} + السياق: + {support} + السؤال: + {question} + الإجابات المحتملة: + + """ + return query + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + correct_answer = doc["correct_answer"] + choices = [ + doc["distractor1"], + doc["distractor2"], + doc["distractor3"], + correct_answer, + ] + + # Shuffle the choices + random.shuffle(choices) + + answer_index = choices.index(correct_answer) + + return {"query": doc_to_text(doc), "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/arabic_leaderboard_arabic_mt_toxigen_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/arabic_leaderboard_arabic_mt_toxigen_light.yaml new file mode 100644 index 0000000000..7e305d5496 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/arabic_leaderboard_arabic_mt_toxigen_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_toxigen_light +task: + - arabic_mt_toxigen_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/arabic_mt_toxigen_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/arabic_mt_toxigen_light.yaml new file mode 100644 index 0000000000..b2bef8abae --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/arabic_mt_toxigen_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_toxigen_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: toxigen_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/utils.py new file mode 100644 index 0000000000..09d311e5a1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + text = doc["text"] + label = 1 if ((doc["toxicity_ai"] + doc["toxicity_human"]) > 5.5) else 0 + instruction = 'هل العبارة التالية تحض على الكراهية؟ قم بالرد بـ"نعم" أو "لا".' + + query = f"""{instruction} + العبارة: + '{text}' + الإجابة: + """ + out_doc = { + "query": query, + "choices": ["لا", "نعم"], + "gold": label, + } + return out_doc + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Algeria_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Algeria_light.yaml new file mode 100644 index 0000000000..4ab4634f60 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Algeria_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Algeria_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Algeria +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Ancient_Egypt_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Ancient_Egypt_light.yaml new file mode 100644 index 0000000000..ab6fffedc1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Ancient_Egypt_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Ancient_Egypt_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Ancient_Egypt +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arab_Empire_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arab_Empire_light.yaml new file mode 100644 index 0000000000..886574ebf2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arab_Empire_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arab_Empire_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arab_Empire +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Architecture_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Architecture_light.yaml new file mode 100644 index 0000000000..e57472ad6e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Architecture_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Architecture_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Architecture +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Art_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Art_light.yaml new file mode 100644 index 0000000000..e94340e755 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Art_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Art_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Art +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Astronomy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Astronomy_light.yaml new file mode 100644 index 0000000000..e8ed990d52 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Astronomy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Astronomy_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Astronomy +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Calligraphy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Calligraphy_light.yaml new file mode 100644 index 0000000000..cd41bdde6a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Calligraphy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Calligraphy_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Calligraphy +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Ceremony_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Ceremony_light.yaml new file mode 100644 index 0000000000..72c6705479 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Ceremony_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Ceremony_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Ceremony +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Clothing_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Clothing_light.yaml new file mode 100644 index 0000000000..9348de07f7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Clothing_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Clothing_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Clothing +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Culture_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Culture_light.yaml new file mode 100644 index 0000000000..4f211064d6 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Culture_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Culture_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Culture +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Food_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Food_light.yaml new file mode 100644 index 0000000000..7ccef6746f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Food_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Food_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Food +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Funeral_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Funeral_light.yaml new file mode 100644 index 0000000000..941154787b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Funeral_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Funeral_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Funeral +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Geography_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Geography_light.yaml new file mode 100644 index 0000000000..36221d8899 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Geography_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Geography_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Geography +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_History_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_History_light.yaml new file mode 100644 index 0000000000..2e12831816 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_History_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_History_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_History +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Language_Origin_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Language_Origin_light.yaml new file mode 100644 index 0000000000..8060604355 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Language_Origin_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Language_Origin_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Language_Origin +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Literature_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Literature_light.yaml new file mode 100644 index 0000000000..3122e39531 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Literature_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Literature_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Literature +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Math_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Math_light.yaml new file mode 100644 index 0000000000..0182aedac7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Math_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Math_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Math +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Medicine_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Medicine_light.yaml new file mode 100644 index 0000000000..aec88febf1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Medicine_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Medicine_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Medicine +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Music_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Music_light.yaml new file mode 100644 index 0000000000..35a771898a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Music_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Music_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Music +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Ornament_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Ornament_light.yaml new file mode 100644 index 0000000000..6b31186cd6 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Ornament_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Ornament_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Ornament +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Philosophy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Philosophy_light.yaml new file mode 100644 index 0000000000..f6b5fa71f1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Philosophy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Philosophy_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Philosophy +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Physics_and_Chemistry_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Physics_and_Chemistry_light.yaml new file mode 100644 index 0000000000..559d729c9b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Physics_and_Chemistry_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Physics_and_Chemistry_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Physics_and_Chemistry +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Wedding_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Wedding_light.yaml new file mode 100644 index 0000000000..9241709c13 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Wedding_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Wedding_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Wedding +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Bahrain_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Bahrain_light.yaml new file mode 100644 index 0000000000..b9c7cef57a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Bahrain_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Bahrain_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Bahrain +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Comoros_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Comoros_light.yaml new file mode 100644 index 0000000000..1f74bd46c5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Comoros_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Comoros_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Comoros +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Egypt_modern_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Egypt_modern_light.yaml new file mode 100644 index 0000000000..e0b19cff58 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Egypt_modern_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Egypt_modern_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Egypt_modern +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromAncientEgypt_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromAncientEgypt_light.yaml new file mode 100644 index 0000000000..6cf755a2a8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromAncientEgypt_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromAncientEgypt_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: InfluenceFromAncientEgypt +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromByzantium_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromByzantium_light.yaml new file mode 100644 index 0000000000..8fe285eb12 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromByzantium_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromByzantium_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: InfluenceFromByzantium +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromChina_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromChina_light.yaml new file mode 100644 index 0000000000..bb028b0892 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromChina_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromChina_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: InfluenceFromChina +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromGreece_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromGreece_light.yaml new file mode 100644 index 0000000000..25060acc1a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromGreece_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromGreece_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: InfluenceFromGreece +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromIslam_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromIslam_light.yaml new file mode 100644 index 0000000000..0a60a2f3f0 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromIslam_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromIslam_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: InfluenceFromIslam +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromPersia_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromPersia_light.yaml new file mode 100644 index 0000000000..7081bec227 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromPersia_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromPersia_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: InfluenceFromPersia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromRome_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromRome_light.yaml new file mode 100644 index 0000000000..8c64cf3bbe --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromRome_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromRome_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: InfluenceFromRome +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Iraq_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Iraq_light.yaml new file mode 100644 index 0000000000..a056a9cf04 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Iraq_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Iraq_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Iraq +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islam_Education_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islam_Education_light.yaml new file mode 100644 index 0000000000..e8f6ad45d9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islam_Education_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Islam_Education_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Islam_Education +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islam_branches_and_schools_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islam_branches_and_schools_light.yaml new file mode 100644 index 0000000000..98137e9a3a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islam_branches_and_schools_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Islam_branches_and_schools_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Islam_branches_and_schools +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islamic_law_system_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islamic_law_system_light.yaml new file mode 100644 index 0000000000..d9aff345da --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islamic_law_system_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Islamic_law_system_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Islamic_law_system +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Jordan_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Jordan_light.yaml new file mode 100644 index 0000000000..674a998e01 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Jordan_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Jordan_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Jordan +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Kuwait_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Kuwait_light.yaml new file mode 100644 index 0000000000..0c3d372d9e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Kuwait_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Kuwait_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Kuwait +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Lebanon_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Lebanon_light.yaml new file mode 100644 index 0000000000..9c3856d698 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Lebanon_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Lebanon_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Lebanon +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Libya_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Libya_light.yaml new file mode 100644 index 0000000000..6070ccbfb8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Libya_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Libya_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Libya +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Mauritania_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Mauritania_light.yaml new file mode 100644 index 0000000000..0b1deda614 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Mauritania_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Mauritania_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Mauritania +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Mesopotamia_civilization_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Mesopotamia_civilization_light.yaml new file mode 100644 index 0000000000..65474b724b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Mesopotamia_civilization_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Mesopotamia_civilization_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Mesopotamia_civilization +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Morocco_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Morocco_light.yaml new file mode 100644 index 0000000000..d752434a5a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Morocco_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Morocco_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Morocco +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Oman_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Oman_light.yaml new file mode 100644 index 0000000000..448498f4a1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Oman_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Oman_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Oman +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Palestine_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Palestine_light.yaml new file mode 100644 index 0000000000..a619c460a1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Palestine_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Palestine_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Palestine +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Qatar_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Qatar_light.yaml new file mode 100644 index 0000000000..967dbc57ef --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Qatar_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Qatar_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Qatar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Saudi_Arabia_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Saudi_Arabia_light.yaml new file mode 100644 index 0000000000..d45558b9ff --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Saudi_Arabia_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Saudi_Arabia_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Saudi_Arabia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Somalia_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Somalia_light.yaml new file mode 100644 index 0000000000..558ea176a3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Somalia_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Somalia_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Somalia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Sudan_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Sudan_light.yaml new file mode 100644 index 0000000000..ce59973306 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Sudan_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Sudan_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Sudan +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Syria_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Syria_light.yaml new file mode 100644 index 0000000000..8b0bd7aebc --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Syria_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Syria_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Syria +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Tunisia_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Tunisia_light.yaml new file mode 100644 index 0000000000..a53c5e0bf9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Tunisia_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Tunisia_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Tunisia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_United_Arab_Emirates_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_United_Arab_Emirates_light.yaml new file mode 100644 index 0000000000..1ce5993a67 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_United_Arab_Emirates_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_United_Arab_Emirates_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: United_Arab_Emirates +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Yemen_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Yemen_light.yaml new file mode 100644 index 0000000000..e480b19d60 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Yemen_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Yemen_light +dataset_path: OALL/ACVA +dataset_name: Yemen +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_communication_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_communication_light.yaml new file mode 100644 index 0000000000..2814278ace --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_communication_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_communication_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: communication +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_computer_and_phone_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_computer_and_phone_light.yaml new file mode 100644 index 0000000000..ddd07e3f50 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_computer_and_phone_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_computer_and_phone_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: computer_and_phone +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_daily_life_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_daily_life_light.yaml new file mode 100644 index 0000000000..2d975e4e85 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_daily_life_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_daily_life_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: daily_life +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_entertainment_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_entertainment_light.yaml new file mode 100644 index 0000000000..721e6cdd3b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_entertainment_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_entertainment_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: entertainment +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_light.yaml new file mode 100644 index 0000000000..ea4a89771f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_light.yaml @@ -0,0 +1,70 @@ +group: arabic_leaderboard_acva_light +task: + - arabic_leaderboard_acva_Algeria_light + - arabic_leaderboard_acva_Ancient_Egypt_light + - arabic_leaderboard_acva_Arab_Empire_light + - arabic_leaderboard_acva_Arabic_Architecture_light + - arabic_leaderboard_acva_Arabic_Art_light + - arabic_leaderboard_acva_Arabic_Astronomy_light + - arabic_leaderboard_acva_Arabic_Calligraphy_light + - arabic_leaderboard_acva_Arabic_Ceremony_light + - arabic_leaderboard_acva_Arabic_Clothing_light + - arabic_leaderboard_acva_Arabic_Culture_light + - arabic_leaderboard_acva_Arabic_Food_light + - arabic_leaderboard_acva_Arabic_Funeral_light + - arabic_leaderboard_acva_Arabic_Geography_light + - arabic_leaderboard_acva_Arabic_History_light + - arabic_leaderboard_acva_Arabic_Language_Origin_light + - arabic_leaderboard_acva_Arabic_Literature_light + - arabic_leaderboard_acva_Arabic_Math_light + - arabic_leaderboard_acva_Arabic_Medicine_light + - arabic_leaderboard_acva_Arabic_Music_light + - arabic_leaderboard_acva_Arabic_Ornament_light + - arabic_leaderboard_acva_Arabic_Philosophy_light + - arabic_leaderboard_acva_Arabic_Physics_and_Chemistry_light + - arabic_leaderboard_acva_Arabic_Wedding_light + - arabic_leaderboard_acva_Bahrain_light + - arabic_leaderboard_acva_Comoros_light + - arabic_leaderboard_acva_Egypt_modern_light + - arabic_leaderboard_acva_InfluenceFromAncientEgypt_light + - arabic_leaderboard_acva_InfluenceFromByzantium_light + - arabic_leaderboard_acva_InfluenceFromChina_light + - arabic_leaderboard_acva_InfluenceFromGreece_light + - arabic_leaderboard_acva_InfluenceFromIslam_light + - arabic_leaderboard_acva_InfluenceFromPersia_light + - arabic_leaderboard_acva_InfluenceFromRome_light + - arabic_leaderboard_acva_Iraq_light + - arabic_leaderboard_acva_Islam_Education_light + - arabic_leaderboard_acva_Islam_branches_and_schools_light + - arabic_leaderboard_acva_Islamic_law_system_light + - arabic_leaderboard_acva_Jordan_light + - arabic_leaderboard_acva_Kuwait_light + - arabic_leaderboard_acva_Lebanon_light + - arabic_leaderboard_acva_Libya_light + - arabic_leaderboard_acva_Mauritania_light + - arabic_leaderboard_acva_Mesopotamia_civilization_light + - arabic_leaderboard_acva_Morocco_light + - arabic_leaderboard_acva_Oman_light + - arabic_leaderboard_acva_Palestine_light + - arabic_leaderboard_acva_Qatar_light + - arabic_leaderboard_acva_Saudi_Arabia_light + - arabic_leaderboard_acva_Somalia_light + - arabic_leaderboard_acva_Sudan_light + - arabic_leaderboard_acva_Syria_light + - arabic_leaderboard_acva_Tunisia_light + - arabic_leaderboard_acva_United_Arab_Emirates_light + - arabic_leaderboard_acva_Yemen_light + - arabic_leaderboard_acva_communication_light + - arabic_leaderboard_acva_computer_and_phone_light + - arabic_leaderboard_acva_daily_life_light + - arabic_leaderboard_acva_entertainment_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/utils.py new file mode 100644 index 0000000000..7e91496f59 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/utils.py @@ -0,0 +1,16 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["question"] + answer = doc["answer"] + + return { + "query": f"السؤال: {question}\nالإجابة:", + "choices": ["صح", "خطأ"], + "gold": ["صح", "خطأ"].index(answer), + } + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_light.yaml new file mode 100644 index 0000000000..d77ebd1eeb --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_light.yaml @@ -0,0 +1,25 @@ +group: arabic_leaderboard_light +task: + - arabic_leaderboard_acva_light + - arabic_leaderboard_alghafa_light + - arabic_leaderboard_arabic_exams_light + - arabic_leaderboard_arabic_mt_arc_challenge_light + - arabic_leaderboard_arabic_mt_arc_easy_light + - arabic_leaderboard_arabic_mt_boolq_light + - arabic_leaderboard_arabic_mt_hellaswag_light + - arabic_leaderboard_arabic_mt_mmlu_light + - arabic_leaderboard_arabic_mt_copa_light + - arabic_leaderboard_arabic_mt_openbook_qa_light + - arabic_leaderboard_arabic_mt_piqa_light + - arabic_leaderboard_arabic_mt_race_light + - arabic_leaderboard_arabic_mt_sciq_light + - arabic_leaderboard_arabic_mt_toxigen_light +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 From fb963f0f0a5b28b69763590bb59676072cf43a01 Mon Sep 17 00:00:00 2001 From: Lintang Sutawika Date: Fri, 13 Sep 2024 13:47:02 -0400 Subject: [PATCH 6/6] Multimodal prototyping (#2243) * add WIP hf vlm class * add doc_to_image * add mmmu tasks * fix merge conflicts * add lintang's changes to hf_vlms.py * fix doc_to_image * added yaml_path for config-loading * revert * add line to process str type v * update * modeling cleanup * add aggregation for mmmu * rewrite MMMU processing code based on only MMMU authors' repo (doc_to_image still WIP) * implemented doc_to_image * update doc_to_image to accept list of features * update functions * readd image processed * update args process * bugfix for repeated images fed to model * push WIP loglikelihood code * commit most recent code (generative ; qwen2-vl testing) * preliminary image_token_id handling * small mmmu update: some qs have >4 mcqa options * push updated modeling code * use processor.apply_chat_template * add mathvista draft * nit * nit * ensure no footguns in text<>multimodal LM<>task incompatibility * add notification to readme regarding launch of prototype! * fix compatibility check * reorganize mmmu configs * chat_template=None * add interleave chat_template * add condition * add max_images; interleave=true * nit * testmini_mcq * nit * pass image string; convert img * add vllm * add init * vlm add multi attr * fixup * pass max images to vllm model init * nit * encoding to device * fix HFMultimodalLM.chat_template ? * add mmmu readme * remove erroneous prints * use HFMultimodalLM.chat_template ; restore tasks/__init__.py * add docstring for replace_placeholders in utils * fix `replace_placeholders`; set image_string=None * fix typo * cleanup + fix merge conflicts * update MMMU readme * del mathvista * add some sample scores * Update README.md * add log msg for image_string value --------- Co-authored-by: haileyschoelkopf Co-authored-by: Baber Abbasi Co-authored-by: Baber Co-authored-by: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com> --- README.md | 1 + lm_eval/api/task.py | 99 ++- lm_eval/evaluator.py | 20 + lm_eval/models/__init__.py | 2 + lm_eval/models/hf_vlms.py | 719 ++++++++++++++++++ lm_eval/models/huggingface.py | 9 + lm_eval/models/utils.py | 34 + lm_eval/models/vllm_vlms.py | 279 +++++++ lm_eval/tasks/mmmu/README.md | 150 ++++ lm_eval/tasks/mmmu/_art_and_design.yaml | 13 + lm_eval/tasks/mmmu/_business.yaml | 14 + lm_eval/tasks/mmmu/_health_and_medicine.yaml | 14 + .../mmmu/_humanities_and_social_sciences.yaml | 13 + lm_eval/tasks/mmmu/_mmmu.yaml | 14 + lm_eval/tasks/mmmu/_science.yaml | 14 + lm_eval/tasks/mmmu/_tech_and_engineering.yaml | 16 + lm_eval/tasks/mmmu/_template_yaml | 19 + lm_eval/tasks/mmmu/mmmu_accounting.yaml | 4 + lm_eval/tasks/mmmu/mmmu_agriculture.yaml | 4 + .../mmmu_architecture_and_engineering.yaml | 4 + lm_eval/tasks/mmmu/mmmu_art.yaml | 4 + lm_eval/tasks/mmmu/mmmu_art_theory.yaml | 4 + .../mmmu/mmmu_basic_medical_science.yaml | 4 + lm_eval/tasks/mmmu/mmmu_biology.yaml | 4 + lm_eval/tasks/mmmu/mmmu_chemistry.yaml | 4 + .../tasks/mmmu/mmmu_clinical_medicine.yaml | 4 + lm_eval/tasks/mmmu/mmmu_computer_science.yaml | 4 + lm_eval/tasks/mmmu/mmmu_design.yaml | 4 + ...u_diagnostics_and_laboratory_medicine.yaml | 4 + lm_eval/tasks/mmmu/mmmu_economics.yaml | 4 + lm_eval/tasks/mmmu/mmmu_electronics.yaml | 4 + lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml | 4 + lm_eval/tasks/mmmu/mmmu_finance.yaml | 4 + lm_eval/tasks/mmmu/mmmu_geography.yaml | 4 + lm_eval/tasks/mmmu/mmmu_history.yaml | 4 + lm_eval/tasks/mmmu/mmmu_literature.yaml | 4 + lm_eval/tasks/mmmu/mmmu_manage.yaml | 4 + lm_eval/tasks/mmmu/mmmu_marketing.yaml | 4 + lm_eval/tasks/mmmu/mmmu_materials.yaml | 4 + lm_eval/tasks/mmmu/mmmu_math.yaml | 4 + .../mmmu/mmmu_mechanical_engineering.yaml | 4 + lm_eval/tasks/mmmu/mmmu_music.yaml | 4 + lm_eval/tasks/mmmu/mmmu_pharmacy.yaml | 4 + lm_eval/tasks/mmmu/mmmu_physics.yaml | 4 + lm_eval/tasks/mmmu/mmmu_psychology.yaml | 4 + lm_eval/tasks/mmmu/mmmu_public_health.yaml | 4 + lm_eval/tasks/mmmu/mmmu_sociology.yaml | 4 + lm_eval/tasks/mmmu/utils.py | 341 +++++++++ 48 files changed, 1865 insertions(+), 26 deletions(-) create mode 100644 lm_eval/models/hf_vlms.py create mode 100644 lm_eval/models/vllm_vlms.py create mode 100644 lm_eval/tasks/mmmu/README.md create mode 100644 lm_eval/tasks/mmmu/_art_and_design.yaml create mode 100644 lm_eval/tasks/mmmu/_business.yaml create mode 100644 lm_eval/tasks/mmmu/_health_and_medicine.yaml create mode 100644 lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml create mode 100644 lm_eval/tasks/mmmu/_mmmu.yaml create mode 100644 lm_eval/tasks/mmmu/_science.yaml create mode 100644 lm_eval/tasks/mmmu/_tech_and_engineering.yaml create mode 100644 lm_eval/tasks/mmmu/_template_yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_accounting.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_agriculture.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_art.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_art_theory.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_biology.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_chemistry.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_computer_science.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_design.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_economics.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_electronics.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_finance.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_geography.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_history.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_literature.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_manage.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_marketing.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_materials.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_math.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_music.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_pharmacy.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_physics.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_psychology.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_public_health.yaml create mode 100644 lm_eval/tasks/mmmu/mmmu_sociology.yaml create mode 100644 lm_eval/tasks/mmmu/utils.py diff --git a/README.md b/README.md index e973cdb7e8..9f26c811d1 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ *Latest News 📣* +- [2024/09] We are prototyping allowing users of LM Evaluation Harness to create and evaluate on text+image multimodal input, text output tasks, and have just added the `hf-multimodal` and `vllm-vlm` model types and `mmmu` task as a prototype feature. We welcome users to try out this in-progress feature and stress-test it for themselves, and suggest they check out [`lmms-eval`](https://github.com/EvolvingLMMs-Lab/lmms-eval), a wonderful project originally forking off of the lm-evaluation-harness, for a broader range of multimodal tasks, models, and features. - [2024/07] [API model](docs/API_guide.md) support has been updated and refactored, introducing support for batched and async requests, and making it significantly easier to customize and use for your own purposes. **To run Llama 405B, we recommend using VLLM's OpenAI-compliant API to host the model, and use the `local-completions` model type to evaluate the model.** - [2024/07] New Open LLM Leaderboard tasks have been added ! You can find them under the [leaderboard](lm_eval/tasks/leaderboard/README.md) task group. diff --git a/lm_eval/api/task.py b/lm_eval/api/task.py index 8a1a3bdbaf..8dadf94857 100644 --- a/lm_eval/api/task.py +++ b/lm_eval/api/task.py @@ -75,6 +75,7 @@ class TaskConfig(dict): process_docs: Optional[Callable] = None doc_to_text: Optional[Union[Callable, str]] = None doc_to_target: Optional[Union[Callable, str]] = None + doc_to_image: Union[Callable, str] = None doc_to_choice: Optional[Union[Callable, str, dict, list]] = None process_results: Optional[Union[Callable, str]] = None use_prompt: Optional[str] = None @@ -377,6 +378,10 @@ def doc_to_text(self, doc): def doc_to_target(self, doc): pass + # not an abstractmethod because not every language-only task has to implement this + def doc_to_image(self, doc): + raise NotImplementedError + def build_all_requests( self, *, @@ -735,6 +740,10 @@ def __init__( ) self.OUTPUT_TYPE = self.config.output_type + if self.config.doc_to_image is not None: + # mark the task as requiring multimodality. + self.MULTIMODAL = True + if self.config.dataset_path is not None: self.DATASET_PATH = self.config.dataset_path @@ -1042,8 +1051,8 @@ def fewshot_context( Whether to apply the chat template to the fewshot context. :param fewshot_as_multiturn: bool Whether to provide the fewshot examples as a multiturn conversation or a single user turn. - :param chat_template: Callable - Chat template to be applied to the fewshot context. + :param chat_template: + callable (from lm.apply_chat_template) that takes in a list[Dict] chat transcript and renders it into a string. :returns: str The fewshot context. """ @@ -1279,9 +1288,34 @@ def doc_to_choice(self, doc: Any, doc_to_choice=None) -> List[str]: else: raise TypeError + def doc_to_image(self, doc: Any, doc_to_image=None) -> Union[int, str, list]: + if doc_to_image is not None: + doc_to_image = doc_to_image + elif self.config.doc_to_image is not None: + doc_to_image = self.config.doc_to_image + else: + return None + + if isinstance(doc_to_image, list): + image_feature = [ + self.doc_to_image(doc, feature) for feature in doc_to_image + ] + return [feature for feature in image_feature if feature is not None] + elif isinstance(doc_to_image, str): + if doc_to_image in self.features: + return doc[doc_to_image] + else: + return ast.literal_eval(utils.apply_template(doc_to_image, doc)) + elif callable(doc_to_image): + return doc_to_image(doc) + else: + return None + def construct_requests( self, doc: dict, ctx: str, **kwargs ) -> Union[List[Instance], Instance]: + aux_arguments = None + if self.OUTPUT_TYPE == "loglikelihood": arguments = (ctx, self.doc_to_target(doc)) elif self.OUTPUT_TYPE == "loglikelihood_rolling": @@ -1299,6 +1333,37 @@ def construct_requests( # Otherwise they are placed in the continuation arguments = [(ctx, f"{target_delimiter}{cont}") for cont in choices] + # TODO: we should raise a warning telling users this will at most ~2x runtime. + if "acc_mutual_info" in self._metric_fn_list.keys(): + # if we are calculating multiple choice accuracy + # using mutual information instead of raw loglikelihood as metric, need unconditional lls. + + # here mutual info refers to calculating + # log(P(choice|ctx) / P(choice)) = log(P(choice|ctx)) - log(P(choice)) + # in other words normalizing by subtracting the unconditional logprob of each choice. + aux_arguments = [("", f"{choice}") for choice in choices] + + arguments.extend(aux_arguments) + + elif self.OUTPUT_TYPE == "generate_until": + arguments = (ctx, deepcopy(self.config.generation_kwargs)) + + multimodal_arg = {} + if ( + self.config.doc_to_image + ): # TODO: ensure that non-multimodal tasks aren't getting visual args + multimodal_arg = { + **multimodal_arg, + **{"visual": self.doc_to_image(doc)}, + } + + if bool(multimodal_arg): + if isinstance(arguments, list): + arguments = [arg + (multimodal_arg,) for arg in arguments] + else: + arguments = arguments + (multimodal_arg,) + + if self.OUTPUT_TYPE == "multiple_choice": request_list = [ Instance( request_type="loglikelihood", @@ -1309,33 +1374,15 @@ def construct_requests( ) for i, arg in enumerate(arguments) ] - # TODO: we should raise a warning telling users this will at most ~2x runtime. - if "acc_mutual_info" in self._metric_fn_list.keys(): - # if we are calculating multiple choice accuracy - # using mutual information instead of raw loglikelihood as metric, need unconditional lls. - # here mutual info refers to calculating - # log(P(choice|ctx) / P(choice)) = log(P(choice|ctx)) - log(P(choice)) - # in other words normalizing by subtracting the unconditional logprob of each choice. - request_list.extend( - [ - Instance( - request_type="loglikelihood", - doc=doc, - arguments=("", "{}".format(choice)), - idx=i, - **kwargs, - ) - for i, choice in enumerate(choices) - ] - ) return request_list - elif self.OUTPUT_TYPE == "generate_until": - arguments = (ctx, deepcopy(self.config.generation_kwargs)) - return Instance( - request_type=self.OUTPUT_TYPE, doc=doc, arguments=arguments, idx=0, **kwargs + request_type=self.OUTPUT_TYPE, + doc=doc, + arguments=arguments, + idx=0, + **kwargs, ) def process_results(self, doc, results): @@ -1547,7 +1594,7 @@ def __repr__(self): f"ConfigurableTask(task_name={getattr(self.config, 'task', None)}," f"output_type={self.OUTPUT_TYPE}," f"num_fewshot={getattr(self.config, 'num_fewshot', None)}," - f"num_samples={len(self.eval_docs)})" + f"num_samples={len(self.eval_docs)})", ) diff --git a/lm_eval/evaluator.py b/lm_eval/evaluator.py index 52a8d00d74..4f4c8f7fba 100644 --- a/lm_eval/evaluator.py +++ b/lm_eval/evaluator.py @@ -414,8 +414,28 @@ def evaluate( for task_output in eval_tasks ): raise ValueError("log_samples must be True for 'bypass' metric-only tasks") + + # validation check: are we running multimodal task <-> non-multimodal model class, or vice-versa. + incompatible_tasks = [] for task_output in eval_tasks: task: Task = task_output.task + + if getattr(lm, "MULTIMODAL", False) != getattr(task, "MULTIMODAL", False): + incompatible_tasks.append(task_output.task_name) + if len(incompatible_tasks) > 0: + if not getattr(lm, "MULTIMODAL", False): + raise ValueError( + f"Attempted to run tasks: {incompatible_tasks} which require multimodal input, but the selected model type does not currently implement this. Multimodal support is currently restricted to the ['hf-multimodal', 'vllm-vlm'] model type." + ) + else: + raise ValueError( + f"Attempted to run tasks: {incompatible_tasks} which are text-only, but used a model type which only currently supports multimodal tasks." + ) + # end multimodality validation check + + for task_output in eval_tasks: + task: Task = task_output.task + limit = get_sample_size(task, limit) task.build_all_requests( limit=limit, diff --git a/lm_eval/models/__init__.py b/lm_eval/models/__init__.py index 0b2441eb87..553be3a5fd 100644 --- a/lm_eval/models/__init__.py +++ b/lm_eval/models/__init__.py @@ -3,6 +3,7 @@ api_models, dummy, gguf, + hf_vlms, huggingface, mamba_lm, nemo_lm, @@ -12,6 +13,7 @@ optimum_lm, textsynth, vllm_causallms, + vllm_vlms, ) diff --git a/lm_eval/models/hf_vlms.py b/lm_eval/models/hf_vlms.py new file mode 100644 index 0000000000..c94c76b26f --- /dev/null +++ b/lm_eval/models/hf_vlms.py @@ -0,0 +1,719 @@ +import copy +from typing import Dict, List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import transformers +from tqdm import tqdm +from transformers import BatchEncoding + +from lm_eval import utils +from lm_eval.api.instance import Instance +from lm_eval.api.registry import register_model +from lm_eval.models.huggingface import HFLM +from lm_eval.models.utils import ( + Collator, + pad_and_concat, + replace_placeholders, + stop_sequences_criteria, +) + + +DEFAULT_IMAGE_PLACEHOLDER = "" + + +eval_logger = utils.eval_logger + + +@register_model("hf-multimodal") +class HFMultimodalLM(HFLM): + """ + An abstracted Hugging Face model class for multimodal LMs like Llava and Idefics. + """ + + AUTO_MODEL_CLASS = transformers.AutoModelForVision2Seq + MULTIMODAL = True # flag to indicate, for now, that this model type can run multimodal requests + + def __init__( + self, + pretrained: Union[str, transformers.PreTrainedModel], + image_token_id: Optional[int] = None, + image_string: Optional[str] = None, + interleave: bool = True, + # TODO: handle whitespace in image placeholder (replacement) + max_images: Optional[int] = 999, + convert_img_format=False, + **kwargs, + ): + # We initialize using HFLM's init. Sub-methods like _create_model and _create_tokenizer + # modify init behavior. + super().__init__(pretrained, **kwargs) + + assert ( + self.batch_size != "auto" + ), "Batch size 'auto' is not yet supported for hf-multimodal models." + self.chat_applied: bool = False + # TODO: phi-3.5 "image placeholders" are , , ... in order. how to handle this case + + # HF AutoModelForVision2Seq models have an `image_token_id` value in their configs + # denoting the token which indicates a location where an image will be substituted in. + # This can take different string values across models, e.g. for Idefics2 and <|image_pad|> for Qwen2-VL + self.interleave = interleave + self.max_images = max_images + self.rgb = convert_img_format + # WARNING: improperly set image_token_id can lead to ignored image input or other (potentially silent) errors! + if not image_string: + self.image_token_id = ( + int(image_token_id) + if image_token_id + else ( + getattr(self.config, "image_token_id", None) + or getattr(self.config, "image_token_index", None) + ) + ) + assert ( + self.image_token_id is not None + ), "Must have a non-None image_token_id to evaluate a Hugging Face AutoModelForVision2Seq model. Please pass `image_token_id` in `--model_args` if model's config does not already specify one." + # get the string this token ID corresponds to + self.image_token = self.tok_decode( + [self.image_token_id], skip_special_tokens=False + ) + if image_token_id is not None: + eval_logger.info( + f"A non-default image_token_id with image_token_id={self.image_token_id} and string value '{self.image_token}' was specified manually. Note that using an improper image_token placeholder may lead to ignored image input or errors!" + ) + else: + eval_logger.info( + f"A non-default image_token string with string value image_string='{image_string}' was specified manually. Note that using an improper image_token placeholder may lead to ignored image input or errors!" + ) + self.image_token = image_string + + def _create_tokenizer( + self, + pretrained: Union[str, transformers.PreTrainedModel], + tokenizer: Optional[ + Union[ + str, + transformers.ProcessorMixin, + ] + ], + revision: Optional[str] = "main", + trust_remote_code: Optional[bool] = False, + **kwargs, + ) -> None: + """ + Helper method during initialization. + + For the multimodal variant, we initialize not just + `self.tokenizer` but also `self.processor`. + """ + + if tokenizer: + if isinstance(tokenizer, str): + return transformers.AutoProcessor.from_pretrained( + tokenizer, + revision=revision, + trust_remote_code=trust_remote_code, + # use_fast=use_fast_tokenizer, + ) + else: + assert isinstance( + tokenizer, transformers.ProcessorMixin + ) # TODO: check this condition + return tokenizer + + # Get tokenizer based on 'pretrained' + if isinstance(pretrained, str): + model_name = pretrained + else: + # get the HF hub name via accessor on model + model_name = self.model.name_or_path + + self.processor = transformers.AutoProcessor.from_pretrained( + model_name, + revision=revision, + trust_remote_code=trust_remote_code, + # use_fast=use_fast_tokenizer, + ) + + self.tokenizer = self.processor.tokenizer + + def tok_multimodal_encode( + self, string, images, left_truncate_len=None, add_special_tokens=None + ): + """Helper function which encodes an image + string combo using AutoProcessor""" + # We inherit special token kwarg setup from HFLM.tok_encode + # special_tokens_kwargs = {} + + # by default for CausalLM - false or self.add_bos_token is set + # if add_special_tokens is None: + # special_tokens_kwargs = {"add_special_tokens": False or self.add_bos_token} + # otherwise the method explicitly defines the value + # else: + # special_tokens_kwargs = {"add_special_tokens": add_special_tokens} + + # encode text+images + # TODO: why does (Qwen2-VL) processor error when attempting to add special tokens to text? + encoding = self.processor( + text=string, images=images, return_tensors=None + ) # , **special_tokens_kwargs) + + # remove (and store) our tokenized text + text_encoding = encoding.pop("input_ids") + encoding.pop("attention_mask") + + # left-truncate the encoded context to be at most `left_truncate_len` tokens long + if left_truncate_len: + text_encoding = text_encoding[-left_truncate_len:] + + return text_encoding, encoding # image_encoding is a dict + + def _encode_multimodal_pair(self, context, continuation, images): + """Helper function to perform the role of TemplateLM._encode_pair + Except allowing for image input to also be processed alongside `context`. + + This method is a bit messy due to the need to defer conversion of image and text token input + into PyTorch tensors until the main inference loop. + """ + + n_spaces = len(context) - len(context.rstrip()) + if n_spaces > 0: + continuation = context[-n_spaces:] + continuation + context = context[:-n_spaces] + + # TODO: replace default placeholder with self.image_token, for contexts + + whole_enc, image_enc = self.tok_multimodal_encode( + context + continuation, images + ) + context_enc, _ = self.tok_multimodal_encode(context, images) + + # tok_multimodal_encode returns List[List[int]] for tokenized text. Get rid of the batch dim + # since we only are encoding a single string. + # TODO: this is a bit hacky, it'd be nice to make this generally cleaner + whole_enc, context_enc = whole_enc[0], context_enc[0] + + context_enc_len = len(context_enc) + continuation_enc = whole_enc[context_enc_len:] + + return context_enc, continuation_enc, image_enc + + def apply_chat_template(self, chat_history: List[Dict[str, str]]) -> str: + self.chat_applied = True + if not self.interleave: + for content in chat_history: + c = [] + text = content["content"] + + # Count and remove image placeholders + image_count = min( + self.max_images, text.count(DEFAULT_IMAGE_PLACEHOLDER) + ) + text = text.replace(DEFAULT_IMAGE_PLACEHOLDER, "") + + # Add image entries + for _ in range(image_count): + c.append({"type": "image", "image": None}) + + # Add single text entry at the end + c.append({"type": "text", "text": text}) + + content["content"] = c + else: + for content in chat_history: + c = [] + text = content["content"] + expected_image_count = min( + self.max_images, text.count(DEFAULT_IMAGE_PLACEHOLDER) + ) + actual_image_count = 0 + + text_parts = text.split(DEFAULT_IMAGE_PLACEHOLDER) + + for i, part in enumerate(text_parts): + # TODO: concatenate text parts (esp. if skipping images)? + if part: # Add non-empty text parts + c.append({"type": "text", "text": part}) + if ( + (i < len(text_parts) - 1) and i < self.max_images + ): # Add image placeholder after each split except the last + c.append({"type": "image"}) + actual_image_count += 1 + + content["content"] = c + + if actual_image_count != expected_image_count: + raise ValueError( + f"Mismatch in image placeholder count. Expected: {expected_image_count}, Actual: {actual_image_count}" + ) + + return self.processor.apply_chat_template( + chat_history, add_generation_prompt=True + ) + + def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]: + if hasattr(self.processor, "apply_chat_template"): + _tokenizer = self.tokenizer + self.tokenizer = self.processor + + selected_template = super().chat_template(chat_template) + + self.tokenizer = _tokenizer + return selected_template + else: + return super().chat_template(chat_template) + + def tok_batch_multimodal_encode( + self, + strings: List[str], # note that input signature of this fn is different + images: List[List], # TODO: images are pil.Image at the moment, update typehint + padding_side: str = "left", + left_truncate_len: int = None, + truncation: bool = False, + ) -> Union[ + BatchEncoding, Dict[str, torch.Tensor] + ]: # note that this return signature differs from HFLM tok_batch_encode. + # NOTE: here, we replace tags with our model's corresponding image_token string value. + if not self.chat_applied: + # TODO: This still keeps the whitespace in the image placeholder, which is not ideal. + strings = [ + replace_placeholders( + string, DEFAULT_IMAGE_PLACEHOLDER, self.image_token, self.max_images + ) + for string in strings + ] + + # encode a batch of strings. converts to tensors and pads automatically, unlike tok_encode. + old_padding_side = self.tokenizer.padding_side + self.tokenizer.padding_side = padding_side + + # add_special_tokens = {"add_special_tokens": False or self.add_bos_token} + + images = [img[: self.max_images] for img in images] + if self.rgb: + images = [[img.convert("RGB") for img in sublist] for sublist in images] + + encoding = self.processor( + images=images, + text=strings, + truncation=truncation, + padding="longest", + return_tensors="pt", + # **add_special_tokens, # TODO: at least some Processors error out when passing this. How do we control whether text gets BOS added? + ) + + encoding.to( # TODO: our other tokenization methods in HFLM don't typically move to device. this breaks convention + self.device, self.model.dtype + ) # TODO: This only casts the pixel values. Should they always be float16? + if left_truncate_len: + encoding["input_ids"] = encoding["input_ids"][:, -left_truncate_len:] + encoding["attention_mask"] = encoding["attention_mask"][ + :, -left_truncate_len: + ] + self.tokenizer.padding_side = old_padding_side + + return encoding + + def _model_multimodal_call(self, inps, imgs, attn_mask=None, labels=None): + """ + TODO: update docstring + """ + # note: imgs is a dict. + with torch.no_grad(): + return self.model(inps, **imgs).logits + + def _model_multimodal_generate(self, inputs, max_length, stop, **generation_kwargs): + generation_kwargs["temperature"] = generation_kwargs.get("temperature", 0.0) + do_sample = generation_kwargs.get("do_sample", None) + + # The temperature has to be a strictly positive float -- if it is 0.0, use greedy decoding strategies + if generation_kwargs.get("temperature") == 0.0 and do_sample is None: + generation_kwargs["do_sample"] = do_sample = False + + if do_sample is False and generation_kwargs.get("temperature") == 0.0: + generation_kwargs.pop("temperature") + + stopping_criteria = stop_sequences_criteria( + self.tokenizer, + stop, + inputs["input_ids"].shape[1], + inputs["input_ids"].shape[0], + ) + return self.model.generate( + **inputs, + max_length=max_length, + stopping_criteria=stopping_criteria, + pad_token_id=self.tokenizer.pad_token_id, + use_cache=True, + **generation_kwargs, + ) + + def _batch_images(self, image_encs): + """ + Helper function: batch together image encodings across examples in a batch. + # TODO: for variable-sized images, this may break down. + """ + batched_imgs = {} + for key in image_encs[0].keys(): + batched_imgs[key] = torch.cat( + [ + torch.tensor( + image_enc[key], device=self.device, dtype=self.model.dtype + ) + for image_enc in image_encs + ], + dim=0, + ) + return batched_imgs + + def loglikelihood_rolling(self, requests: List[Instance]) -> List[float]: + raise NotImplementedError( + "model type `hf-multimodal` does not support loglikelihood_rolling. Use 'hf' model type for text-only loglikelihood_rolling tasks ", + "this is because we do not support measuring the loglikelihood a model assigns to an image.", + ) + + def loglikelihood( + self, requests: List[Instance], disable_tqdm: bool = False + ) -> List[Tuple[float, bool]]: + raise NotImplementedError( + "'loglikelihood' requests for model type `hf-multimodal` are not yet tested. This feature will be enabled when a loglikelihood-based multiple-choice VQA dataset is added!" + ) + + new_reqs = [] + for context, continuation, aux_arguments in [req.args for req in requests]: + if context == "": + raise ValueError( + "Must get non-empty context for multimodal requests! You might be trying to run 'loglikelihood_rolling', which is not supported in the multimodal case." + ) + else: + visuals = aux_arguments["visual"] + + context_enc, continuation_enc, image_enc = self._encode_multimodal_pair( + context, continuation, visuals + ) + # TODO: key to pick for caching images + new_reqs.append( + ( + (context, continuation, visuals), + context_enc, + continuation_enc, + image_enc, + ) + ) + + return self._loglikelihood_tokens(new_reqs, disable_tqdm=disable_tqdm) + + def _loglikelihood_tokens( + self, + requests: List[ + Tuple[Tuple[None, str, str], List[int], List[int], List[int]] + ], # TODO: update typehint to be correct + disable_tqdm: bool = False, + override_bs: int = None, + ) -> List[Tuple[float, bool]]: + res = [] + + # TODO: **improve multimodal collation.** We currently ignore image size when ordering docs. ideally we'd take them into account + def _collate(req: Tuple[Tuple[str, str], List[int], List[int]]): + """Defines the key for the sorted method""" + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + toks = req[1] + req[2] + return -len(toks), tuple(toks) + + def _lookup_one_token_cont(req: Tuple[Tuple[str, str], List[int], List[int]]): + """Defines the key to group and lookup one-token continuations""" + # Use with group_by="contexts" (optional)" + # allows for the creation of a lookup, so we can reuse logits in case of one-token continuations. + # speeds up some multiple-choice tasks proportionally to the number of choices. + # groups requests by context+continuation[:-1] and infer on one request/group. + return req[-1] + req[-3] + req[-2][:-1] + + re_ord = Collator( + requests, + sort_fn=_collate, + group_by="contexts" # TODO: can't group-by just "contexts" any more, need to incorporate imgs + if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM + and self.logits_cache + else None, + group_fn=_lookup_one_token_cont, + ) + + # automatic (variable) batch size detection for vectorization + # pull longest context sample from request + n_reordered_requests = len(re_ord) + batch_size = ( + self.batch_size + if self.batch_size != "auto" + else override_bs + if override_bs is not None + else 0 + ) + batch_fn = ( + self._batch_scheduler + if self.batch_size == "auto" + and n_reordered_requests > 0 + and not override_bs + else None + ) + + chunks = re_ord.get_batched(n=batch_size, batch_fn=batch_fn) + pbar = tqdm( + total=len(requests), + disable=(disable_tqdm or (self.rank != 0)), + desc="Running loglikelihood requests with text+image input", + ) + for chunk in chunks: + imgs = [] + inps = [] + cont_toks_list = [] + inplens = [] + + padding_len_inp = None + # because vectorizing is annoying, we first convert each (context, continuation) pair to padded + # tensors, then we pack them together into a batch, call the model, and then pick it all apart + # again because vectorizing is annoying + + for _, context_enc, continuation_enc, image_enc in chunk: + # sanity check + assert len(image_enc) > 0 + assert len(context_enc) > 0 + assert len(continuation_enc) > 0 + assert len(continuation_enc) <= self.max_length + + # how this all works (illustrated on a causal decoder-only setup): + # CTX CONT + # inp 0 1 2 3|4 5 6 7 8 9 <- last token is deleted by inp[:, :-1] + # model \ \ + # logits 1 2 3|4 5 6 7 8 9 <- the ctx half gets tossed out by the + # cont_toks 4 5 6 7 8 9 [:, -len(continuation_enc):, :self.vocab_size] slice + + # when too long to fit in context, truncate from the left + # TODO: assuming that we won't handle enc-dec Vision2Seq models. Is that a safe assumption? + inp = torch.tensor( + (context_enc + continuation_enc)[-(self.max_length + 1) :][:-1], + dtype=torch.long, + device=self.device, + ) + (inplen,) = inp.shape + + padding_len_inp = ( + max(padding_len_inp, inplen) + if padding_len_inp is not None + else inplen + ) + + inps.append(inp) # [1, inp_length] + cont_toks_list.append(continuation_enc) + inplens.append(inplen) + + imgs.append(image_enc) + + # create encoder attn mask and batched conts, if seq2seq + call_kwargs = {} + batched_inps = pad_and_concat( + padding_len_inp, inps, padding_side="right" + ) # [batch, padding_len_inp] + # batch our examples' image inputs together + batched_imgs = self._batch_images( + imgs + ) # TODO: fix/test for bs>1 case with differently-sized imgs! + + multi_logits = F.log_softmax( + self._model_multimodal_call(batched_inps, batched_imgs, **call_kwargs), + dim=-1, + ) # [batch, padding_length (inp or cont), vocab] + + for ( + request_str, + ctx_tokens, + _, + image_encs, + ), logits, inplen, cont_toks in zip( + chunk, multi_logits, inplens, cont_toks_list + ): + # Slice to original seq length + contlen = len(cont_toks) + # take only logits in the continuation + # (discard context toks if decoder-only ; discard right-padding) + # also discards + checks for "virtual tokens" in the causal LM's input window + # from prompt/prefix tuning tokens, if applicable + ctx_len = ( + inplen + (logits.shape[0] - padding_len_inp) + if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM + else None + ) + logits = self._select_cont_toks(logits, contlen=contlen, inplen=ctx_len) + logits = logits.unsqueeze(0) # [1, seq, vocab] + + # Check if per-token argmax is exactly equal to continuation + greedy_tokens = logits.argmax(dim=-1) + + # check for one-token continuation cache hits. + # noop in case group_by != "contexts" or no cache hit and returns the + # original args. Otherwise, expands the logits batch dimension and yields each + # batch along with matching continuation tokens and prompt strings. + # logits -> [1, seq, vocab] + for request_str, cont_toks, logits in re_ord.get_cache( + req_str=request_str, + cxt_toks=ctx_tokens, + cont_toks=cont_toks, + logits=logits, + ): + cont_toks = torch.tensor( + cont_toks, dtype=torch.long, device=self.device + ).unsqueeze(0) # [1, seq] + max_equal = (greedy_tokens == cont_toks).all() + + # Obtain log-probs at the corresponding continuation token indices + # last_token_slice = logits[:, -1, :].squeeze(0).tolist() + logits = torch.gather(logits, 2, cont_toks.unsqueeze(-1)).squeeze( + -1 + ) # [1, seq] + + # Answer: (log prob, is-exact-match) + answer = (float(logits.sum()), bool(max_equal)) + + res.append(answer) + + self.cache_hook.add_partial( + "loglikelihood", request_str, answer + ) # TODO: choose convention for adding images into the cache key + pbar.update(1) + + pbar.close() + + return re_ord.get_original(res) + + def generate_until( + self, requests: List[Instance], disable_tqdm: bool = False + ) -> List[str]: + # TODO: back out to HFLM.generate_until() for all requests without aux_arguments (text-only reqs) + res = [] + + def _collate(x): + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + toks = self.tok_encode(x[0]) + return -len(toks), x[0] + + pbar = tqdm( + total=len(requests), + disable=(disable_tqdm or (self.rank != 0)), + desc="Running generate_until requests with text+image input", + ) + # TODO: port auto-batch sizing into this. + + # we group requests by their generation_kwargs, + # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling + # in the same batch. + re_ords = Collator( + [reg.args for reg in requests], + _collate, + group_by="gen_kwargs", + group_fn=lambda x: x[1], + ) + chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None) + + ### Up to here: was identical to non-multimodal HFLM generate_until ### + + for chunk in chunks: + contexts, all_gen_kwargs, aux_arguments = zip(*chunk) + + visuals = [arg["visual"] for arg in aux_arguments] + + if not isinstance(contexts, list): + contexts = list( + contexts + ) # for Qwen2-VL, processor is unhappy accepting a tuple of strings instead of a list. + # TODO: could we upstream this workaround to HF? + ### this part onward: same as HFLM ### + + # we assume all gen kwargs in the batch are the same + # this is safe to assume because the `grouper` object ensures it. + gen_kwargs = all_gen_kwargs[0] + # unpack our keyword arguments. + until = None + if isinstance(gen_kwargs, dict): + kwargs = copy.deepcopy(gen_kwargs) # edge case for repeats > 1 + if "until" in kwargs.keys(): + until = kwargs.pop("until") + if isinstance(until, str): + until = [until] + elif not isinstance(until, list): + raise ValueError( + f"Expected `kwargs['until']` to be of type Union[str,list] but got {until}" + ) + else: + raise ValueError( + f"Expected `kwargs` to be of type `dict` but got {type(gen_kwargs)}" + ) + # add EOS token to stop sequences + eos = self.tok_decode(self.eot_token_id, skip_special_tokens=False) + if not until: + until = [eos] + else: + until.append(eos) + if "max_gen_toks" in kwargs.keys(): + max_gen_toks = kwargs.pop("max_gen_toks") + else: + max_gen_toks = self.max_gen_toks + + ### end stuff that's entirely copied verbatim from HFLM ### + + max_ctx_len = self.max_length - max_gen_toks + + inputs = self.tok_batch_multimodal_encode( + contexts, + visuals, + left_truncate_len=max_ctx_len, + truncation=self.truncation, + ) + + context_enc = inputs["input_ids"] + + if "max_length" not in kwargs: + kwargs["max_length"] = context_enc.shape[1] + max_gen_toks + + cont = self._model_multimodal_generate(inputs, stop=until, **kwargs) + + del inputs + torch.cuda.empty_cache() + import gc + + gc.collect() + + ### essentially same as HFLM beyond this line! + + cont_toks_list = cont.tolist() + for cont_toks, context in zip(cont_toks_list, contexts): + # discard context + left-padding toks if using causal decoder-only VLM + cont_toks = cont_toks[context_enc.shape[1] :] + + s = self.tok_decode(cont_toks) + + # use secondary stop seqs to cut off should-have-been-stopped content post-hoc + for term in until: + if len(term) > 0: + # ignore '' separator, + # for seq2seq case where self.tok_decode(self.eot_token_id) = '' + s = s.split(term)[0] + + res.append(s) + self.cache_hook.add_partial( + "generate_until", (context, gen_kwargs), s + ) # TODO: cache key for multimodal input should be what? + pbar.update(1) + # reorder this group of results back to original unsorted form + res = re_ords.get_original(res) + + pbar.close() + return res diff --git a/lm_eval/models/huggingface.py b/lm_eval/models/huggingface.py index 01a8cf6789..63ad1440bd 100644 --- a/lm_eval/models/huggingface.py +++ b/lm_eval/models/huggingface.py @@ -448,7 +448,16 @@ def _get_backend( Helper method during initialization. Determines the backend ("causal" (decoder-only) or "seq2seq" (encoder-decoder)) model type to be used. + sets `self.AUTO_MODEL_CLASS` appropriately if not already set. """ + # escape hatch: if we're using a subclass that shouldn't follow + # the default _get_backend logic, + # then skip over the method. + # TODO: this seems very much undesirable in some cases--our code in HFLM + # references AutoModelForCausalLM at times to check for equality + if self.AUTO_MODEL_CLASS is not None: + return + assert backend in ["default", "causal", "seq2seq"] if backend != "default": diff --git a/lm_eval/models/utils.py b/lm_eval/models/utils.py index 8a81e5deca..72fac03811 100644 --- a/lm_eval/models/utils.py +++ b/lm_eval/models/utils.py @@ -664,3 +664,37 @@ def configure_pad_token( tokenizer.add_special_tokens({"pad_token": "<|pad|>"}) return tokenizer + + +def replace_placeholders( + string: str, default_placeholder: str, image_token: str, max_images: int +): + """ + A utility function used for local multimodal models. It locates all `placeholder` string + occurrences in the given input `string_` and replaces the first `max_count` instances with + `replacement`, and all subsequent occurrences with the empty string. + + This is used to replace placeholder tags by model-specific image tokens like <|image_pad|> + and to allow for only the first `max_count` images to be passed to a model if desired. + + :param string: The original string containing placeholders. + :param default_placeholder: The placeholder text to be replaced. + :param image_token: The token to replace the placeholder with. + :param max_images: The maximum number of replacements to make. + :return: The string with placeholders replaced. + """ + count = 0 + result = [] + + parts = string.split(default_placeholder) + for part in parts[:-1]: # Iterate through all but the last part + result.append(part) + if count < max_images: + result.append(image_token) + count += 1 + elif default_placeholder != image_token: + result.append(default_placeholder) + + # Add the last part of the string + result.append(parts[-1]) + return "".join(result) diff --git a/lm_eval/models/vllm_vlms.py b/lm_eval/models/vllm_vlms.py new file mode 100644 index 0000000000..5a1260901c --- /dev/null +++ b/lm_eval/models/vllm_vlms.py @@ -0,0 +1,279 @@ +import copy +from typing import Dict, List, Optional + +import transformers +from more_itertools import distribute +from tqdm import tqdm + +from lm_eval.api.instance import Instance +from lm_eval.api.registry import register_model +from lm_eval.models.utils import Collator, undistribute +from lm_eval.models.vllm_causallms import VLLM +from lm_eval.utils import simple_parse_args_string + + +try: + import ray + from vllm import LLM, SamplingParams + from vllm.lora.request import LoRARequest # noqa: F401 + from vllm.transformers_utils.tokenizer import get_tokenizer # noqa: F401 +except ModuleNotFoundError: + pass + + +DEFAULT_IMAGE_PLACEHOLDER = "" + + +@register_model("vllm-vlm") +class VLLM_VLM(VLLM): + MULTIMODAL = True + + def __init__( + self, + pretrained: str, + trust_remote_code: Optional[bool] = False, + revision: Optional[str] = None, + interleave: bool = True, + # TODO: handle max_images and limit_mm_per_prompt better + max_images: int = 999, + limit_mm_per_prompt: str = "image=1", + **kwargs, + ): + kwargs["limit_mm_per_prompt"] = simple_parse_args_string(limit_mm_per_prompt) + super().__init__( + pretrained=pretrained, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + self.interleave = interleave + self.max_images = max_images + self.processor = transformers.AutoProcessor.from_pretrained( + pretrained, + revision=revision, + trust_remote_code=trust_remote_code, + ) + self.chat_applied: bool = False + + def tok_batch_multimodal_encode( + self, + strings: List[str], # note that input signature of this fn is different + images, # TODO: typehint on this + left_truncate_len: int = None, + truncation: bool = False, + ): + images = [img[: self.max_images] for img in images] + + outputs = [] + for x, i in zip(strings, images): + inputs = { + "prompt": x, + "multi_modal_data": {"image": i}, + } + outputs.append(inputs) + return outputs + + def _model_generate( + self, + requests: List[List[dict]] = None, + generate: bool = False, + max_tokens: int = None, + stop: Optional[List[str]] = None, + **kwargs, + ): + if generate: + kwargs = self.modify_gen_kwargs(kwargs) + sampling_params = SamplingParams(max_tokens=max_tokens, stop=stop, **kwargs) + else: + sampling_params = SamplingParams( + temperature=0, prompt_logprobs=1, max_tokens=1, detokenize=False + ) + if self.data_parallel_size > 1: + # vLLM hangs if tensor_parallel > 1 and resources are set in ray.remote + # also seems to only work with decorator and not with ray.remote() fn + # see https://github.com/vllm-project/vllm/issues/973 + # note: this has changed on 0.3.3, and it only works now if num_gpus are set. + # but then tensor_parallel breaks + @ray.remote + def run_inference_one_model( + model_args: dict, sampling_params, requests: List[List[dict]] + ): + llm = LLM(**model_args) + return llm.generate(requests, sampling_params=sampling_params) + + # dispatch requests to all self.data_parallel_size workers, in interleaved fashion + # interleaved important to balance context lengths across workers + requests = [list(x) for x in distribute(self.data_parallel_size, requests)] + inputs = ((self.model_args, sampling_params, req) for req in requests) + object_refs = [run_inference_one_model.remote(*x) for x in inputs] + results = ray.get(object_refs) + # Invoke ray.shutdown() to prevent hang-ups if subsequent calls required. + ray.shutdown() + # flatten results + return undistribute(results) + + if self.lora_request is not None: + outputs = self.model.generate( + requests, + sampling_params=sampling_params, + use_tqdm=True if self.batch_size == "auto" else False, + lora_request=self.lora_request, + ) + else: + outputs = self.model.generate( + requests, + sampling_params=sampling_params, + use_tqdm=True if self.batch_size == "auto" else False, + ) + return outputs + + def apply_chat_template(self, chat_history: List[Dict[str, str]]) -> str: + self.chat_applied = True + if not self.interleave: + for content in chat_history: + c = [] + text = content["content"] + + # Count and remove image placeholders + image_count = min( + self.max_images, text.count(DEFAULT_IMAGE_PLACEHOLDER) + ) + text = text.replace(DEFAULT_IMAGE_PLACEHOLDER, "") + + # Add image entries + for _ in range(image_count): + c.append({"type": "image", "image": None}) + + # Add single text entry at the end + c.append({"type": "text", "text": text}) + + content["content"] = c + else: + for content in chat_history: + c = [] + text = content["content"] + expected_image_count = min( + self.max_images, text.count(DEFAULT_IMAGE_PLACEHOLDER) + ) + actual_image_count = 0 + + text_parts = text.split(DEFAULT_IMAGE_PLACEHOLDER) + + for i, part in enumerate(text_parts): + # TODO: concatenate text parts (esp. if skipping images)? + if part: # Add non-empty text parts + c.append({"type": "text", "text": part}) + if ( + (i < len(text_parts) - 1) and i < self.max_images + ): # Add image placeholder after each split except the last + c.append({"type": "image"}) + actual_image_count += 1 + + content["content"] = c + + if actual_image_count != expected_image_count: + raise ValueError( + f"Mismatch in image placeholder count. Expected: {expected_image_count}, Actual: {actual_image_count}" + ) + + return self.processor.apply_chat_template( + chat_history, add_generation_prompt=True + ) + + def generate_until( + self, requests: List[Instance], disable_tqdm: bool = False + ) -> List[str]: + # TODO: support text-only reqs + res = [] + + def _collate(x): + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + toks = self.tok_encode(x[0]) + return -len(toks), x[0] + + pbar = tqdm( + total=len(requests), + disable=(disable_tqdm or (self.rank != 0)), + desc="Running generate_until requests with text+image input", + ) + # TODO: port auto-batch sizing into this. + + # we group requests by their generation_kwargs, + # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling + # in the same batch. + re_ords = Collator( + [reg.args for reg in requests], + _collate, + group_by="gen_kwargs", + group_fn=lambda x: x[1], + ) + chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None) + + for chunk in chunks: + contexts, all_gen_kwargs, aux_arguments = zip(*chunk) + + visuals = [arg["visual"] for arg in aux_arguments] + + if not isinstance(contexts, list): + contexts = list( + contexts + ) # for Qwen2-VL, processor is unhappy accepting a tuple of strings instead of a list. + # TODO: could we upstream this workaround to HF? + + # we assume all gen kwargs in the batch are the same + # this is safe to assume because the `grouper` object ensures it. + gen_kwargs = all_gen_kwargs[0] + # unpack our keyword arguments. + until = None + if isinstance(gen_kwargs, dict): + kwargs = copy.deepcopy(gen_kwargs) # edge case for repeats > 1 + if "until" in kwargs.keys(): + until = kwargs.pop("until") + if isinstance(until, str): + until = [until] + elif not isinstance(until, list): + raise ValueError( + f"Expected `kwargs['until']` to be of type Union[str,list] but got {until}" + ) + else: + raise ValueError( + f"Expected `kwargs` to be of type `dict` but got {type(gen_kwargs)}" + ) + # add EOS token to stop sequences + eos = self.tokenizer.decode(self.eot_token_id) + if not until: + until = [eos] + else: + until.append(eos) + if "max_gen_toks" in kwargs.keys(): + max_gen_toks = kwargs.pop("max_gen_toks") + else: + max_gen_toks = self.max_gen_toks + + max_ctx_len = self.max_length - max_gen_toks + + inputs = self.tok_batch_multimodal_encode( + contexts, + visuals, + left_truncate_len=max_ctx_len, + ) + + cont = self._model_generate(inputs, stop=until, generate=True, **kwargs) + + for output, context in zip(cont, contexts): + generated_text = output.outputs[0].text + res.append(generated_text) + self.cache_hook.add_partial( + "generate_until", (context, gen_kwargs), generated_text + ) + pbar.update(1) + # reorder this group of results back to original unsorted form + res = re_ords.get_original(res) + + pbar.close() + return res diff --git a/lm_eval/tasks/mmmu/README.md b/lm_eval/tasks/mmmu/README.md new file mode 100644 index 0000000000..e9d0da12f6 --- /dev/null +++ b/lm_eval/tasks/mmmu/README.md @@ -0,0 +1,150 @@ +# MMMU Benchmark + +### Paper + +Title: `MMMU: A Massive Multi-discipline MultimodalUnderstanding and Reasoning Benchmark for Expert AGI` + +Abstract: `MMMU is a new benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning.` + +`The benchmark is composed of 30 tasks, for a total of 900 mixed image+text examples (some with multiple images in context)` + +Homepage: `https://github.com/MMMU-Benchmark/MMMU/tree/main/mmmu` + +Note: Some questions have multiple images in context. To control for this use `max_images=N` in model init. + +### Citation + +``` +@inproceedings{yue2023mmmu, + title={MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI}, + author={Xiang Yue and Yuansheng Ni and Kai Zhang and Tianyu Zheng and Ruoqi Liu and Ge Zhang and Samuel Stevens and Dongfu Jiang and Weiming Ren and Yuxuan Sun and Cong Wei and Botao Yu and Ruibin Yuan and Renliang Sun and Ming Yin and Boyuan Zheng and Zhenzhu Yang and Yibo Liu and Wenhao Huang and Huan Sun and Yu Su and Wenhu Chen}, + booktitle={Proceedings of CVPR}, + year={2024}, + } +``` + +### Groups, Tags, and Tasks + +#### Groups + +* `mmmu_val` +* `mmmu_val_art_and_design` +* `mmmu_val_business` +* `mmmu_val_health_and_medicine` +* `mmmu_val_humanities_and_social_science` +* `mmmu_val_science` +* `mmmu_val_tech_and_engineering` + +#### Tags + + +#### Tasks + +* `mmmu_val_accounting` +* `mmmu_val_agriculture` +* `mmmu_val_architecture_and_engineering.yaml` +* `mmmu_val_art` +* `mmmu_val_art_theory` +* `mmmu_val_basic_medical_science` +* `mmmu_val_biology` +* `mmmu_val_chemistry` +* `mmmu_val_computer_science` +* `mmmu_val_clinical_medicine` +* `mmmu_val_design` +* `mmmu_val_diagnostics_and_laboratory_medicine` +* `mmmu_val_electronics` +* `mmmu_val_energy_and_power` +* `mmmu_val_economics` +* `mmmu_val_finance` +* `mmmu_val_geography` +* `mmmu_val_history` +* ... + +### Variants + +The `mmmu_val` group implements MMMU using processing code [from the original MMMU authors](https://github.com/MMMU-Benchmark/MMMU/tree/main/mmmu) and uses the prompt format found in [the MMMU repository for Llava-1.5](https://github.com/MMMU-Benchmark/MMMU/blob/main/mmmu/configs/llava1.5.yaml). This implementation should give scores on par with or slightly higher than those reported by [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval/tree/main/lmms_eval/tasks/mmmu) for `mmmu_val` and the MMMU repository code. + +Scores on several tested models (**all with `--apply_chat_template`**) are: + +Qwen2-VL-2B: +``` +hf-multimodal (pretrained=Qwen/Qwen2-VL-2B-Instruct,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.3778|± |0.0155| +| - Art and Design | 0|none | |acc |↑ |0.5500|± |0.0415| +| - Business | 0|none | |acc |↑ |0.3600|± |0.0389| +| - Health and Medicine | 0|none | |acc |↑ |0.3667|± |0.0394| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5167|± |0.0438| +| - Science | 0|none | |acc |↑ |0.2467|± |0.0352| +| - Tech and Engineering | 0|none | |acc |↑ |0.3143|± |0.0317| +``` +Author-reported score: 41.1% + + +Qwen2-VL-7B: +``` +hf-multimodal (pretrained=Qwen/Qwen2-VL-7B-Instruct,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.5056|± |0.0160| +| - Art and Design | 0|none | |acc |↑ |0.6917|± |0.0398| +| - Business | 0|none | |acc |↑ |0.4333|± |0.0406| +| - Health and Medicine | 0|none | |acc |↑ |0.5667|± |0.0401| +| - Humanities and Social Science| 0|none | |acc |↑ |0.6750|± |0.0426| +| - Science | 0|none | |acc |↑ |0.3800|± |0.0392| +| - Tech and Engineering | 0|none | |acc |↑ |0.4000|± |0.0341| +``` +Author-reported score: 54.1% + +Idefics2-8B: +``` +hf-multimodal (pretrained=HuggingFaceM4/idefics2-8b,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True,max_images=2), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.4011|± |0.0154| +| - Art and Design | 0|none | |acc |↑ |0.6167|± |0.0436| +| - Business | 0|none | |acc |↑ |0.3200|± |0.0373| +| - Health and Medicine | 0|none | |acc |↑ |0.4000|± |0.0401| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5750|± |0.0424| +| - Science | 0|none | |acc |↑ |0.2600|± |0.0358| +| - Tech and Engineering | 0|none | |acc |↑ |0.3381|± |0.0312| +``` +Author-reported score: ~43% + +Llava-v1.6-Mistral-7B: +``` +hf-multimodal (pretrained=llava-hf/llava-v1.6-mistral-7b-hf,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.3522|± |0.0151| +| - Art and Design | 0|none | |acc |↑ |0.5167|± |0.0440| +| - Business | 0|none | |acc |↑ |0.2667|± |0.0362| +| - Health and Medicine | 0|none | |acc |↑ |0.3867|± |0.0397| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5917|± |0.0433| +| - Science | 0|none | |acc |↑ |0.2200|± |0.0342| +| - Tech and Engineering | 0|none | |acc |↑ |0.2524|± |0.0299| +``` +Author-reported score: 35.3% + + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/mmmu/_art_and_design.yaml b/lm_eval/tasks/mmmu/_art_and_design.yaml new file mode 100644 index 0000000000..b0dda1876f --- /dev/null +++ b/lm_eval/tasks/mmmu/_art_and_design.yaml @@ -0,0 +1,13 @@ +group: mmmu_val_art_and_design +group_alias: Art and Design +task: + - mmmu_val_art + - mmmu_val_art_theory + - mmmu_val_design + - mmmu_val_music +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/_business.yaml b/lm_eval/tasks/mmmu/_business.yaml new file mode 100644 index 0000000000..497948e5e5 --- /dev/null +++ b/lm_eval/tasks/mmmu/_business.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_business +group_alias: Business +task: + - mmmu_val_accounting + - mmmu_val_economics + - mmmu_val_finance + - mmmu_val_manage + - mmmu_val_marketing +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/_health_and_medicine.yaml b/lm_eval/tasks/mmmu/_health_and_medicine.yaml new file mode 100644 index 0000000000..04709d3d46 --- /dev/null +++ b/lm_eval/tasks/mmmu/_health_and_medicine.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_health_and_medicine +group_alias: Health and Medicine +task: + - mmmu_val_basic_medical_science + - mmmu_val_clinical_medicine + - mmmu_val_diagnostics_and_laboratory_medicine + - mmmu_val_pharmacy + - mmmu_val_public_health +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml b/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml new file mode 100644 index 0000000000..b8d70a9ea3 --- /dev/null +++ b/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml @@ -0,0 +1,13 @@ +group: mmmu_val_humanities_and_social_science +group_alias: Humanities and Social Science +task: + - mmmu_val_history + - mmmu_val_literature + - mmmu_val_sociology + - mmmu_val_psychology +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/_mmmu.yaml b/lm_eval/tasks/mmmu/_mmmu.yaml new file mode 100644 index 0000000000..6bcd234bbc --- /dev/null +++ b/lm_eval/tasks/mmmu/_mmmu.yaml @@ -0,0 +1,14 @@ +group: mmmu_val +task: + - mmmu_val_art_and_design + - mmmu_val_business + - mmmu_val_health_and_medicine + - mmmu_val_humanities_and_social_science + - mmmu_val_science + - mmmu_val_tech_and_engineering +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/_science.yaml b/lm_eval/tasks/mmmu/_science.yaml new file mode 100644 index 0000000000..9b41585a65 --- /dev/null +++ b/lm_eval/tasks/mmmu/_science.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_science +group_alias: Science +task: + - mmmu_val_biology + - mmmu_val_chemistry + - mmmu_val_geography + - mmmu_val_math + - mmmu_val_physics +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/_tech_and_engineering.yaml b/lm_eval/tasks/mmmu/_tech_and_engineering.yaml new file mode 100644 index 0000000000..956f9906b3 --- /dev/null +++ b/lm_eval/tasks/mmmu/_tech_and_engineering.yaml @@ -0,0 +1,16 @@ +group: mmmu_val_tech_and_engineering +group_alias: Tech and Engineering +task: + - mmmu_val_agriculture + - mmmu_val_architecture_and_engineering + - mmmu_val_computer_science + - mmmu_val_electronics + - mmmu_val_energy_and_power + - mmmu_val_materials + - mmmu_val_mechanical_engineering +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/_template_yaml b/lm_eval/tasks/mmmu/_template_yaml new file mode 100644 index 0000000000..f92ce60d6b --- /dev/null +++ b/lm_eval/tasks/mmmu/_template_yaml @@ -0,0 +1,19 @@ +dataset_path: MMMU/MMMU +validation_split: validation +output_type: generate_until +doc_to_image: !function utils.doc_to_image +doc_to_text: !function utils.doc_to_text +doc_to_target: "answer" +process_results: !function utils.process_results +generation_kwargs: + until: + - "<|endoftext|>" + temperature: 0.0 + do_sample: false + max_gen_toks: 512 +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/mmmu_accounting.yaml b/lm_eval/tasks/mmmu/mmmu_accounting.yaml new file mode 100644 index 0000000000..3c4aa41d9c --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_accounting.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_accounting +include: _template_yaml +task_alias: Accounting +dataset_name: Accounting diff --git a/lm_eval/tasks/mmmu/mmmu_agriculture.yaml b/lm_eval/tasks/mmmu/mmmu_agriculture.yaml new file mode 100644 index 0000000000..387f33f7a9 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_agriculture.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_agriculture +include: _template_yaml +task_alias: Agriculture +dataset_name: Agriculture diff --git a/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml b/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml new file mode 100644 index 0000000000..829ee3a8f8 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_architecture_and_engineering +include: _template_yaml +task_alias: Architecture and Engineering +dataset_name: Architecture_and_Engineering diff --git a/lm_eval/tasks/mmmu/mmmu_art.yaml b/lm_eval/tasks/mmmu/mmmu_art.yaml new file mode 100644 index 0000000000..ff812ab3c1 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_art.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_art +include: _template_yaml +task_alias: Art +dataset_name: Art diff --git a/lm_eval/tasks/mmmu/mmmu_art_theory.yaml b/lm_eval/tasks/mmmu/mmmu_art_theory.yaml new file mode 100644 index 0000000000..d6331871a5 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_art_theory.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_art_theory +include: _template_yaml +task_alias: Art Theory +dataset_name: Art_Theory diff --git a/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml b/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml new file mode 100644 index 0000000000..d7486f1ee3 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_basic_medical_science +include: _template_yaml +task_alias: Basic Medical Science +dataset_name: Basic_Medical_Science diff --git a/lm_eval/tasks/mmmu/mmmu_biology.yaml b/lm_eval/tasks/mmmu/mmmu_biology.yaml new file mode 100644 index 0000000000..49d36f380f --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_biology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_biology +include: _template_yaml +task_alias: Biology +dataset_name: Biology diff --git a/lm_eval/tasks/mmmu/mmmu_chemistry.yaml b/lm_eval/tasks/mmmu/mmmu_chemistry.yaml new file mode 100644 index 0000000000..cb096531c2 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_chemistry.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_chemistry +include: _template_yaml +task_alias: Chemistry +dataset_name: Chemistry diff --git a/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml b/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml new file mode 100644 index 0000000000..a0833d165f --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_clinical_medicine +include: _template_yaml +task_alias: Clinical Medicine +dataset_name: Clinical_Medicine diff --git a/lm_eval/tasks/mmmu/mmmu_computer_science.yaml b/lm_eval/tasks/mmmu/mmmu_computer_science.yaml new file mode 100644 index 0000000000..53c91a61b3 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_computer_science.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_computer_science +include: _template_yaml +task_alias: Computer Science +dataset_name: Computer_Science diff --git a/lm_eval/tasks/mmmu/mmmu_design.yaml b/lm_eval/tasks/mmmu/mmmu_design.yaml new file mode 100644 index 0000000000..0367dd65ee --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_design.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_design +include: _template_yaml +task_alias: Design +dataset_name: Design diff --git a/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml b/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml new file mode 100644 index 0000000000..2ec1b7dca6 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_diagnostics_and_laboratory_medicine +include: _template_yaml +task_alias: Diagnostics and Laboratory Medicine +dataset_name: Diagnostics_and_Laboratory_Medicine diff --git a/lm_eval/tasks/mmmu/mmmu_economics.yaml b/lm_eval/tasks/mmmu/mmmu_economics.yaml new file mode 100644 index 0000000000..3d6465da74 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_economics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_economics +include: _template_yaml +task_alias: Economics +dataset_name: Economics diff --git a/lm_eval/tasks/mmmu/mmmu_electronics.yaml b/lm_eval/tasks/mmmu/mmmu_electronics.yaml new file mode 100644 index 0000000000..02df22f8a3 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_electronics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_electronics +include: _template_yaml +task_alias: Electronics +dataset_name: Electronics diff --git a/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml b/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml new file mode 100644 index 0000000000..add24ebed7 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_energy_and_power +include: _template_yaml +task_alias: Energy and Power +dataset_name: Energy_and_Power diff --git a/lm_eval/tasks/mmmu/mmmu_finance.yaml b/lm_eval/tasks/mmmu/mmmu_finance.yaml new file mode 100644 index 0000000000..c6b6f042c1 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_finance.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_finance +include: _template_yaml +task_alias: Finance +dataset_name: Finance diff --git a/lm_eval/tasks/mmmu/mmmu_geography.yaml b/lm_eval/tasks/mmmu/mmmu_geography.yaml new file mode 100644 index 0000000000..b5f9022136 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_geography.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_geography +include: _template_yaml +task_alias: Geography +dataset_name: Geography diff --git a/lm_eval/tasks/mmmu/mmmu_history.yaml b/lm_eval/tasks/mmmu/mmmu_history.yaml new file mode 100644 index 0000000000..435388fd74 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_history.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_history +include: _template_yaml +task_alias: History +dataset_name: History diff --git a/lm_eval/tasks/mmmu/mmmu_literature.yaml b/lm_eval/tasks/mmmu/mmmu_literature.yaml new file mode 100644 index 0000000000..9e0a1ba503 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_literature.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_literature +include: _template_yaml +task_alias: Literature +dataset_name: Literature diff --git a/lm_eval/tasks/mmmu/mmmu_manage.yaml b/lm_eval/tasks/mmmu/mmmu_manage.yaml new file mode 100644 index 0000000000..bd19924114 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_manage.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_manage +include: _template_yaml +task_alias: Manage +dataset_name: Manage diff --git a/lm_eval/tasks/mmmu/mmmu_marketing.yaml b/lm_eval/tasks/mmmu/mmmu_marketing.yaml new file mode 100644 index 0000000000..53848171be --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_marketing.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_marketing +include: _template_yaml +task_alias: Marketing +dataset_name: Marketing diff --git a/lm_eval/tasks/mmmu/mmmu_materials.yaml b/lm_eval/tasks/mmmu/mmmu_materials.yaml new file mode 100644 index 0000000000..ee46944316 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_materials.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_materials +include: _template_yaml +task_alias: Materials +dataset_name: Materials diff --git a/lm_eval/tasks/mmmu/mmmu_math.yaml b/lm_eval/tasks/mmmu/mmmu_math.yaml new file mode 100644 index 0000000000..e6817689ab --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_math.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_math +include: _template_yaml +task_alias: Math +dataset_name: Math diff --git a/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml b/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml new file mode 100644 index 0000000000..4b9d95be85 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_mechanical_engineering +include: _template_yaml +task_alias: Mechanical Engineering +dataset_name: Mechanical_Engineering diff --git a/lm_eval/tasks/mmmu/mmmu_music.yaml b/lm_eval/tasks/mmmu/mmmu_music.yaml new file mode 100644 index 0000000000..cf4032dea3 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_music.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_music +include: _template_yaml +task_alias: Music +dataset_name: Music diff --git a/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml b/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml new file mode 100644 index 0000000000..04a4f1ff67 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_pharmacy +include: _template_yaml +task_alias: Pharmacy +dataset_name: Pharmacy diff --git a/lm_eval/tasks/mmmu/mmmu_physics.yaml b/lm_eval/tasks/mmmu/mmmu_physics.yaml new file mode 100644 index 0000000000..53371cad73 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_physics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_physics +include: _template_yaml +task_alias: Physics +dataset_name: Physics diff --git a/lm_eval/tasks/mmmu/mmmu_psychology.yaml b/lm_eval/tasks/mmmu/mmmu_psychology.yaml new file mode 100644 index 0000000000..8471d4a18f --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_psychology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_psychology +include: _template_yaml +task_alias: Psychology +dataset_name: Psychology diff --git a/lm_eval/tasks/mmmu/mmmu_public_health.yaml b/lm_eval/tasks/mmmu/mmmu_public_health.yaml new file mode 100644 index 0000000000..aefc0590a3 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_public_health.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_public_health +include: _template_yaml +task_alias: Public Health +dataset_name: Public_Health diff --git a/lm_eval/tasks/mmmu/mmmu_sociology.yaml b/lm_eval/tasks/mmmu/mmmu_sociology.yaml new file mode 100644 index 0000000000..5da664a8e7 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_sociology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_sociology +include: _template_yaml +task_alias: Sociology +dataset_name: Sociology diff --git a/lm_eval/tasks/mmmu/utils.py b/lm_eval/tasks/mmmu/utils.py new file mode 100644 index 0000000000..87a3022098 --- /dev/null +++ b/lm_eval/tasks/mmmu/utils.py @@ -0,0 +1,341 @@ +import ast +import random +import re + +import numpy as np + + +random.seed(42) + + +# source for prompt fstrings: https://github.com/MMMU-Benchmark/MMMU/blob/7787d60648c82a9d40acd656fa541a6c74f58995/eval/configs/llava1.5.yaml#L3 +MULTI_CHOICE_EXAMPLE_FORMAT = """{} + +{} + +Answer with the option's letter from the given choices directly.""" + + +SHORT_ANS_EXAMPLE_FORMAT = """{} + +Answer the question using a single word or phrase.""" + +START_CHR = "A" + + +def doc_to_image(doc): + # get formatted prompt (incl. multi-choice options) pre- reformatting + input_text = _doc_to_text(doc) + # locate instances in input + image_placeholders = [ + img.replace(" ", "_").replace("<", "").replace(">", "") + for img in re.findall("", input_text) + ] + + # collect visuals (can have dupes of a given image or be out of order) + # E.g. validation_Math_19 contains and but seen as [, , , , ] + visuals = [doc[img] for img in image_placeholders] + + return visuals + + +def doc_to_text(doc): + """Get the prompt for a given document.""" + + prompt = _doc_to_text(doc) + + for i in range(1, 8): + # replace with . TODO: check this is always the right decision incl. for non-HF models + prompt = prompt.replace(f"", "") + + return prompt + + +def _doc_to_text(doc): + """Helper--get the prompt for a given document but DO NOT yet replace with .""" + + if doc["question_type"] == "multiple-choice": + choices_str = "" + + for i, choice in enumerate(ast.literal_eval(doc["options"])): + # add (A) {choice1}\n , (B) {choice2}\n , and so on + # to create the list of formatted choices in the prompt + choices_str += f"\n({chr(ord(START_CHR) + i)}) {choice}" + + choices_str = ( + choices_str.lstrip() + ) # remove the extraneous prepended \n that we added + + prompt = MULTI_CHOICE_EXAMPLE_FORMAT.format(doc["question"], choices_str) + else: + prompt = SHORT_ANS_EXAMPLE_FORMAT.format(doc["question"]) + + return prompt + + +def process_results(doc, results): + if doc["question_type"] == "multiple-choice": + # multichoice logic + option_strs = ast.literal_eval(doc["options"]) + option_letters = ["A", "B", "C", "D", "E", "F", "G", "H", "I"] + + all_choices = option_letters[: len(option_strs)] + index2ans = {index: ans for index, ans in zip(option_letters, option_strs)} + + pred = parse_multi_choice_response(results[0], all_choices, index2ans) + # print(pred, all_choices, index2ans) + is_correct = eval_multi_choice(doc["answer"], pred) + else: + # freeform response handling + pred = parse_open_response(results[0]) + is_correct = eval_open(doc["answer"], pred) + + return {"acc": float(is_correct)} + + # TODO: it would be better if we could use a Filter for this logic. + + +### Output parsing and answer selection taken from +### https://github.com/MMMU-Benchmark/MMMU/blob/main/eval/utils/data_utils.py +### and +### https://github.com/MMMU-Benchmark/MMMU/blob/main/eval/utils/eval_utils.py + + +# ----------- Process Multi-choice ------------- +def parse_multi_choice_response(response, all_choices, index2ans): + """ + Parse the prediction from the generated response. + Return the predicted index e.g., A, B, C, D. + """ + for char in [",", ".", "!", "?", ";", ":", "'"]: + response = response.strip(char) + response = " " + response + " " # add space to avoid partial match + + index_ans = True + ans_with_brack = False + candidates = [] + for choice in all_choices: # e.g., (A) (B) (C) (D) + if f"({choice})" in response: + candidates.append(choice) + ans_with_brack = True + + if len(candidates) == 0: + for choice in all_choices: # e.g., A B C D + if f" {choice} " in response: + candidates.append(choice) + + # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example + if len(candidates) == 0 and len(response.split()) > 5: + for index, ans in index2ans.items(): + if ans.lower() in response.lower(): + candidates.append(index) + index_ans = False # it's content ans. + + if len(candidates) == 0: # still not get answer, randomly choose one. + pred_index = random.choice(all_choices) + elif len(candidates) > 1: + start_indexes = [] + if index_ans: + if ans_with_brack: + for can in candidates: + index = response.rfind(f"({can})") + start_indexes.append(index) # -1 will be ignored anyway + # start_indexes = [generated_response.index(f'({can})') for can in candidates] + else: + for can in candidates: + index = response.rfind(f" {can} ") + start_indexes.append(index) + else: + for can in candidates: + index = response.lower().rfind(index2ans[can].lower()) + start_indexes.append(index) + # get the last one + pred_index = candidates[np.argmax(start_indexes)] + else: # if only one candidate, use it. + pred_index = candidates[0] + + # print(response, all_choices, index2ans, pred_index) + + return pred_index + + +# ----------- Process Open ------------- +def check_is_number(string): + """ + Check if the given string a number. + """ + try: + float(string.replace(",", "")) + return True + except ValueError: + # check if there's comma inside + return False + + +def normalize_str(string): + """ + Normalize the str to lower case and make them float numbers if possible. + """ + # check if characters in the string + + # if number, numerize it. + string = string.strip() + + is_number = check_is_number(string) + + if is_number: + string = string.replace(",", "") + string = float(string) + # leave 2 decimal + string = round(string, 2) + return [string] + else: # it's likely to be a string + # lower it + string = string.lower() + if len(string) == 1: + return [" " + string, string + " "] # avoid trivial matches + return [string] + + +def extract_numbers(string): + """ + Exact all forms of numbers from a string with regex. + """ + # Pattern for numbers with commas + pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b" + # Pattern for scientific notation + pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+" + # Pattern for simple numbers without commas + pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])" + + # Extract numbers with commas + numbers_with_commas = re.findall(pattern_commas, string) + # Extract numbers in scientific notation + numbers_scientific = re.findall(pattern_scientific, string) + # Extract simple numbers without commas + numbers_simple = re.findall(pattern_simple, string) + + # Combine all extracted numbers + all_numbers = numbers_with_commas + numbers_scientific + numbers_simple + return all_numbers + + +def parse_open_response(response): + """ + Parse the prediction from the generated response. + Return a list of predicted strings or numbers. + """ + + # content = content.strip("\n").strip(".").strip(" ") + def get_key_subresponses(response): + key_responses = [] + response = response.strip().strip(".").lower() + sub_responses = re.split(r"\.\s(?=[A-Z])|\n", response) + indicators_of_keys = [ + "could be ", + "so ", + "is ", + "thus ", + "therefore ", + "final ", + "answer ", + "result ", + ] + key_responses = [] + for index, resp in enumerate(sub_responses): + # if last one, accept it's an equation (the entire response can be just one sentence with equation) + if index == len(sub_responses) - 1: + indicators_of_keys.extend(["="]) + shortest_key_response = None # the shortest response that may contain the answer (tail part of the response) + for indicator in indicators_of_keys: + if indicator in resp: + if not shortest_key_response: + shortest_key_response = resp.split(indicator)[-1].strip() + else: + if len(resp.split(indicator)[-1].strip()) < len( + shortest_key_response + ): + shortest_key_response = resp.split(indicator)[-1].strip() + # key_responses.append(resp.split(indicator)[1].strip()) + + if shortest_key_response: + # and it's not trivial + if shortest_key_response.strip() not in [ + ":", + ",", + ".", + "!", + "?", + ";", + ":", + "'", + ]: + key_responses.append(shortest_key_response) + if len(key_responses) == 0: # did not found any + return [response] + return key_responses + + # pdb.set_trace() + key_responses = get_key_subresponses(response) + + pred_list = key_responses.copy() # keep the original string response + for resp in key_responses: + pred_list.extend(extract_numbers(resp)) + + tmp_pred_list = [] + for i in range(len(pred_list)): + tmp_pred_list.extend(normalize_str(pred_list[i])) + pred_list = tmp_pred_list + + # remove duplicates + pred_list = list(set(pred_list)) + + return pred_list + + +# ----------- Evaluation ------------- + + +def eval_multi_choice(gold_i, pred_i): + """ + Evaluate a multiple choice instance. + """ + correct = False + # only they are exactly the same, we consider it as correct + if isinstance(gold_i, list): + for answer in gold_i: + if answer == pred_i: + correct = True + break + else: # gold_i is a string + if gold_i == pred_i: + correct = True + return correct + + +def eval_open(gold_i, pred_i): + """ + Evaluate an open question instance + """ + correct = False + if isinstance(gold_i, list): + # use float to avoid trivial matches + norm_answers = [] + for answer in gold_i: + norm_answers.extend(normalize_str(answer)) + else: + norm_answers = normalize_str(gold_i) + for pred in pred_i: # pred is already normalized in parse response phase + if isinstance(pred, str): # if it's a string, then find if ans in the pred_i + for norm_ans in norm_answers: + # only see if the string answer in the string pred + if isinstance(norm_ans, str) and norm_ans in pred: + if not correct: + correct = True + break + else: # it's a float number + if pred in norm_answers: + if not correct: + correct = True + break + return correct