[Refactor] Verbosity rework

lm_eval/__init__.py

@@ -1 +1,2 @@
-from .evaluator import evaluate, simple_evaluate
+# from .evaluator import evaluate, simple_evaluate
+# from .logger import eval_logger, SPACING

lm_eval/__main__.py

@@ -1,18 +1,17 @@
 import os
 import re
+import sys
 import json
-import fnmatch
-import argparse
 import logging
-from pathlib import Path
+import argparse
 import numpy as np
-from lm_eval import evaluator, utils
-from lm_eval.api.registry import ALL_TASKS
-from lm_eval.logger import eval_logger, SPACING
-from lm_eval.tasks import include_path
 
+from pathlib import Path
 from typing import Union
 
+from lm_eval import evaluator, utils
+from lm_eval.tasks import initialize_tasks, include_path
+from lm_eval.api.registry import ALL_TASKS
 
 def _handle_non_serializable(o):
     if isinstance(o, np.int64) or isinstance(o, np.int32):
@@ -25,11 +24,11 @@ def _handle_non_serializable(o):
 
 def parse_eval_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
-    parser.add_argument("--model", required=True, help="Name of model e.g. `hf`")
+    parser.add_argument("--model", default="hf", help="Name of model e.g. `hf`")
     parser.add_argument(
         "--tasks",
         default=None,
-        help="Available Tasks:\n - {}".format("\n - ".join(sorted(ALL_TASKS))),
+        help="To get full list of tasks, use the command lm-eval --tasks list",
     )
     parser.add_argument(
         "--model_args",
@@ -119,9 +118,13 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
         # we allow for args to be passed externally, else we parse them ourselves
         args = parse_eval_args()
 
+    eval_logger = utils.eval_logger
     eval_logger.setLevel(getattr(logging, f"{args.verbosity}"))
+    eval_logger.info(f"Verbosity set to {args.verbosity}")
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
+    initialize_tasks(args.verbosity)
+
     if args.limit:
         eval_logger.warning(
             " --limit SHOULD ONLY BE USED FOR TESTING."
@@ -133,6 +136,11 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
 
     if args.tasks is None:
         task_names = ALL_TASKS
+    elif args.tasks == "list":
+        eval_logger.info(
+            "Available Tasks:\n - {}".format(f"\n - ".join(sorted(ALL_TASKS)))
+        )
+        sys.exit()
     else:
         if os.path.isdir(args.tasks):
             import glob
@@ -155,10 +163,10 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
                 missing = ", ".join(task_missing)
                 eval_logger.error(
                     f"Tasks were not found: {missing}\n"
-                    f"{SPACING}Try `lm-eval -h` for list of available tasks",
+                    f"{' ' * 47}Try `lm-eval --tasks list` for list of available tasks",
                 )
                 raise ValueError(
-                    f"Tasks {missing} were not found. Try `lm-eval -h` for list of available tasks."
+                    f"Tasks {missing} were not found. Try `lm-eval --tasks list` for list of available tasks."
                 )
 
     if args.output_path:

lm_eval/api/metrics.py

@@ -9,6 +9,9 @@
 
 from lm_eval.api.registry import register_metric, register_aggregation
 
+import logging
+
+eval_logger = logging.getLogger("lm-eval")
 
 # Register Aggregations First
 @register_aggregation("mean")

lm_eval/api/model.py

@@ -10,7 +10,10 @@
 from tqdm import tqdm
 
 from lm_eval import utils
-from lm_eval.logger import eval_logger
+
+import logging
+
+eval_logger = logging.getLogger("lm-eval")
 
 T = TypeVar("T", bound="LM")
 

lm_eval/api/registry.py

@@ -1,7 +1,10 @@
 import os
 import evaluate
 from lm_eval.api.model import LM
-from lm_eval.logger import eval_logger
+
+import logging
+
+eval_logger = logging.getLogger("lm-eval")
 
 MODEL_REGISTRY = {}
 

lm_eval/api/task.py

@@ -4,6 +4,7 @@
 import re
 import ast
 import yaml
+import logging
 import evaluate
 import random
 import itertools
@@ -21,7 +22,6 @@
 from lm_eval.api.instance import Instance
 from lm_eval.api.filter import FilterEnsemble
 
-from lm_eval.logger import eval_logger
 from lm_eval.prompts import get_prompt
 from lm_eval.filters import build_filter_ensemble
 from lm_eval.api.metrics import (
@@ -48,6 +48,9 @@
 ]
 
 
+eval_logger = logging.getLogger("lm-eval")
+
+
 @dataclass
 class TaskConfig(dict):
     # task naming/registry

lm_eval/evaluator.py

@@ -20,10 +20,9 @@
     make_table,
     create_iterator,
     get_git_commit_hash,
+    eval_logger
 )
 
-from lm_eval.logger import eval_logger
-
 
 @positional_deprecated
 def simple_evaluate(
@@ -41,6 +40,7 @@ def simple_evaluate(
     decontamination_ngrams_path=None,
     write_out: bool = False,
     log_samples: bool = True,
+    verbosity: str = "INFO",
 ):
     """Instantiate and evaluate a model on a list of tasks.
 
@@ -253,7 +253,7 @@ def evaluate(
 
         task.build_all_requests(limit=limit, rank=lm.rank, world_size=lm.world_size)
 
-        eval_logger.info(
+        eval_logger.debug(
             f"Task: {task_name}; number of requests on this rank: {len(task.instances)}"
         )
 

lm_eval/models/anthropic_llms.py

@@ -2,9 +2,10 @@
 from lm_eval.api.registry import register_model
 from tqdm import tqdm
 import time
-from lm_eval.logger import eval_logger
+from lm_eval import utils
 from typing import List, Any, Tuple
 
+eval_logger = utils.eval_logger
 
 def anthropic_completion(
     client,  #: anthropic.Anthropic,

lm_eval/models/huggingface.py

@@ -16,7 +16,6 @@
 import torch.nn.functional as F
 
 from lm_eval import utils
-from lm_eval.logger import eval_logger
 from lm_eval.api.model import LM
 from lm_eval.api.registry import register_model
 
@@ -25,6 +24,7 @@
 from accelerate import Accelerator, find_executable_batch_size, DistributedType
 from typing import List, Optional, Union
 
+eval_logger = utils.eval_logger
 
 def _get_accelerate_args(
     device_map_option: Optional[str] = "auto",

lm_eval/prompts/__init__.py

@@ -3,7 +3,7 @@
 
 from typing import Dict
 from lm_eval import utils
-from lm_eval.logger import eval_logger
+from lm_eval.utils import eval_logger
 
 # Prompt library.
 # Stores prompts in a dictionary indexed by 2 levels:

lm_eval/tasks/__init__.py

@@ -14,8 +14,7 @@
 )
 
 import logging
-
-eval_logger = logging.getLogger("lm-eval")
+eval_logger = utils.eval_logger
 
 
 def register_configurable_task(config: Dict[str, str]) -> int:
@@ -141,8 +140,11 @@ def include_task_folder(task_dir: str, register_task: bool = True) -> None:
                         else:
                             if type(config["task"]) == list:
                                 register_configurable_group(config, yaml_path)
+
+                # Log this silently and show it only when
+                # the user defines the appropriate verbosity.
                 except ModuleNotFoundError as e:
-                    eval_logger.warning(
+                    eval_logger.debug(
                         f"{yaml_path}: {e}. Config will not be added to registry."
                     )
                 except Exception as error:
@@ -165,8 +167,12 @@ def include_path(task_dir):
     return 0
 
 
-task_dir = os.path.dirname(os.path.abspath(__file__)) + "/"
-include_path(task_dir)
+def initialize_tasks(verbosity="INFO"):
+
+    eval_logger.setLevel(getattr(logging, f"{verbosity}"))
+
+    task_dir = os.path.dirname(os.path.abspath(__file__)) + "/"
+    include_path(task_dir)
 
 
 def get_task(task_name, config):

lm_eval/tasks/minerva_math/utils.py

@@ -1,7 +1,7 @@
 import datasets
 import re
 import signal
-from lm_eval.logger import eval_logger
+from lm_eval.utils import eval_logger
 from typing import Optional, List, Dict
 
 try:

lm_eval/tasks/realtoxicityprompts/metric.py

@@ -3,7 +3,7 @@
 import requests
 import numpy as np
 
-from lm_eval.logger import eval_logger
+from lm_eval.utils import eval_logger
 
 
 def toxicity_perspective_api(references, predictions, **kwargs):

lm_eval/utils.py

@@ -19,8 +19,16 @@
 from jinja2 import BaseLoader, Environment, StrictUndefined
 from itertools import islice
 
-from lm_eval.logger import eval_logger
+import logging
 
+logging.basicConfig(
+    format="%(asctime)s,%(msecs)03d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s",
+    datefmt="%Y-%m-%d:%H:%M:%S",
+    level=logging.INFO,
+)
+eval_logger = logging.getLogger("lm-eval")
+
+SPACING = " " * 47
 
 def escaped_split(text, sep_char, maxsplit=-1):
     """Split text into a list on occurrences of the given separation