All tests should now pass

src/helm/benchmark/adaptation/adapters/test_adapter.py

@@ -4,17 +4,13 @@
 from helm.common.authentication import Authentication
 from helm.proxy.services.server_service import ServerService
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
-from helm.benchmark.model_deployment_registry import maybe_register_helm
 
 
 class TestAdapter:
     """
     Has setup and teardown methods downstream Adapter tests need.
     """
 
-    def setup_class(self):
-        maybe_register_helm()
-
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
         service = ServerService(base_path=self.path, root_mode=True)

src/helm/benchmark/model_deployment_registry.py

@@ -10,12 +10,15 @@
 from helm.benchmark.model_metadata_registry import (
     ModelMetadata,
     get_model_metadata,
-    maybe_register_model_metadata_from_base_path,
+    register_metadatas_if_not_already_registered,
+    CONFIG_PATH,
 )
-from helm.benchmark.tokenizer_config_registry import maybe_register_tokenizer_configs_from_base_path
+from helm.benchmark.tokenizer_config_registry import register_tokenizers_if_not_already_registered
 
 
-MODEL_DEPLOYMENTS_FILE = "model_deployments.yaml"
+MODEL_DEPLOYMENTS_FILE: str = "model_deployments.yaml"
+DEPLOYMENTS_REGISTERED: bool = False
+HELM_REGISTERED: bool = False
 
 
 class ClientSpec(ObjectSpec):
@@ -133,6 +136,7 @@ def maybe_register_model_deployments_from_base_path(base_path: str) -> None:
 
 # ===================== UTIL FUNCTIONS ==================== #
 def get_model_deployment(name: str) -> ModelDeployment:
+    register_deployments_if_not_already_registered()
     if name not in DEPLOYMENT_NAME_TO_MODEL_DEPLOYMENT:
         raise ValueError(f"Model deployment {name} not found")
     deployment: ModelDeployment = DEPLOYMENT_NAME_TO_MODEL_DEPLOYMENT[name]
@@ -144,6 +148,7 @@ def get_model_deployment(name: str) -> ModelDeployment:
 # TODO: Remove when we no longer want to offer backwards compatibility for model names
 # that are now moved to model deployments (PR #1903).
 def get_deployment_name_from_model_arg(model_arg: str) -> str:
+    register_deployments_if_not_already_registered()
     if model_arg in DEPLOYMENT_NAME_TO_MODEL_DEPLOYMENT:
         return model_arg
 
@@ -166,6 +171,7 @@ def get_model_deployments_by_host_group(host_group: str) -> List[str]:
     Gets models by host group.
     Example:   together   =>   TODO(PR)
     """
+    register_deployments_if_not_already_registered()
     return [deployment.name for deployment in ALL_MODEL_DEPLOYMENTS if deployment.host_group == host_group]
 
 
@@ -188,6 +194,7 @@ def get_default_deployment_for_model(model_metadata: ModelMetadata) -> ModelDepl
     For example if several deplyments are available but only some can be used
     given the API keys present, then we should choose the one that can be used.
     """
+    register_deployments_if_not_already_registered()
     if model_metadata.name in DEPLOYMENT_NAME_TO_MODEL_DEPLOYMENT:
         return DEPLOYMENT_NAME_TO_MODEL_DEPLOYMENT[model_metadata.name]
     elif model_metadata.deployment_names is not None and len(model_metadata.deployment_names) > 0:
@@ -208,21 +215,24 @@ def get_metadata_for_deployment(deployment_name: str) -> ModelMetadata:
 
 def get_model_names_with_tokenizer(tokenizer_name: str) -> List[str]:
     """Get all the name of the models with tokenizer `tokenizer_name`."""
+    register_deployments_if_not_already_registered()
     deployments: List[ModelDeployment] = [
         deployment for deployment in ALL_MODEL_DEPLOYMENTS if deployment.tokenizer_name == tokenizer_name
     ]
     return [deployment.model_name or deployment.name for deployment in deployments]
 
 
-HELM_REGISTERED: bool = False
+def register_deployments_if_not_already_registered() -> None:
+    global DEPLOYMENTS_REGISTERED
+    if not DEPLOYMENTS_REGISTERED:
+        maybe_register_model_deployments_from_base_path(CONFIG_PATH)
+        DEPLOYMENTS_REGISTERED = True
 
 
 def maybe_register_helm():
     global HELM_REGISTERED
-    if HELM_REGISTERED:
-        return
-    config_path: str = "src/helm/config"
-    maybe_register_model_metadata_from_base_path(config_path)
-    maybe_register_model_deployments_from_base_path(config_path)
-    maybe_register_tokenizer_configs_from_base_path(config_path)
-    HELM_REGISTERED = True
+    if not HELM_REGISTERED:
+        register_metadatas_if_not_already_registered()
+        register_tokenizers_if_not_already_registered()
+        register_deployments_if_not_already_registered()
+        HELM_REGISTERED = True

src/helm/benchmark/model_metadata_registry.py

@@ -49,7 +49,9 @@
 VISION_LANGUAGE_MODEL_TAG: str = "VISION_LANGUAGE_MODEL_TAG"
 
 
-MODEL_METADATA_FILE = "model_metadatas.yaml"
+MODEL_METADATA_FILE: str = "model_metadatas.yaml"
+CONFIG_PATH: str = "src/helm/config"
+METADATAS_REGISTERED: bool = False
 
 
 # Frozen is set to false as the model_deployment_registry.py file
@@ -153,6 +155,7 @@ def maybe_register_model_metadata_from_base_path(base_path: str) -> None:
 # ===================== UTIL FUNCTIONS ==================== #
 def get_model_metadata(model_name: str) -> ModelMetadata:
     """Get the `Model` given the name."""
+    register_metadatas_if_not_already_registered()
     if model_name not in MODEL_NAME_TO_MODEL_METADATA:
         raise ValueError(f"No model with name: {model_name}")
 
@@ -167,6 +170,7 @@ def get_model_creator_organization(model_name: str) -> str:
 
 def get_all_models() -> List[str]:
     """Get all model names."""
+    register_metadatas_if_not_already_registered()
     return list(MODEL_NAME_TO_MODEL_METADATA.keys())
 
 
@@ -175,11 +179,13 @@ def get_models_by_creator_organization(organization: str) -> List[str]:
     Gets models by creator organization.
     Example:   ai21   =>   ai21/j1-jumbo, ai21/j1-grande, ai21-large.
     """
+    register_metadatas_if_not_already_registered()
     return [model.name for model in ALL_MODELS_METADATA if model.creator_organization == organization]
 
 
 def get_model_names_with_tag(tag: str) -> List[str]:
     """Get all the name of the models with tag `tag`."""
+    register_metadatas_if_not_already_registered()
     return [model.name for model in ALL_MODELS_METADATA if tag in model.tags]
 
 
@@ -196,3 +202,10 @@ def get_all_code_models() -> List[str]:
 def get_all_instruction_following_models() -> List[str]:
     """Get all instruction-following model names."""
     return get_model_names_with_tag(INSTRUCTION_FOLLOWING_MODEL_TAG)
+
+
+def register_metadatas_if_not_already_registered() -> None:
+    global METADATAS_REGISTERED
+    if not METADATAS_REGISTERED:
+        maybe_register_model_metadata_from_base_path(CONFIG_PATH)
+        METADATAS_REGISTERED = True

src/helm/benchmark/presentation/test_run_entry.py

@@ -4,13 +4,9 @@
 from helm.benchmark.presentation.run_entry import read_run_entries
 from helm.benchmark.run_specs import construct_run_specs
 from helm.benchmark import vlm_run_specs  # noqa
-from helm.benchmark.model_deployment_registry import maybe_register_helm
 
 
 class TestRunEntry:
-    def setup_class(self):
-        maybe_register_helm()
-
     def test_read_all_specs(self):
         """Read all the run entries and make sure they parse and we can instantiate them."""
         base_path = os.path.dirname(__file__)

src/helm/benchmark/tokenizer_config_registry.py

@@ -7,9 +7,11 @@
 
 from helm.common.hierarchical_logger import hlog
 from helm.common.object_spec import ObjectSpec
+from helm.benchmark.model_metadata_registry import CONFIG_PATH
 
 
-TOKENIZER_CONFIGS_FILE = "tokenizer_configs.yaml"
+TOKENIZER_CONFIGS_FILE: str = "tokenizer_configs.yaml"
+TOKENIZERS_REGISTERED: bool = False
 
 
 class TokenizerSpec(ObjectSpec):
@@ -38,11 +40,13 @@ class TokenizerConfigs:
     tokenizer_configs: List[TokenizerConfig]
 
 
-_name_to_tokenizer_config: Dict[str, TokenizerConfig] = {}
+ALL_TOKENIZER_CONFIGS: List[TokenizerConfig] = []
+TOKENIZER_NAME_TO_CONFIG: Dict[str, TokenizerConfig] = {config.name: config for config in ALL_TOKENIZER_CONFIGS}
 
 
 def register_tokenizer_config(tokenizer_config: TokenizerConfig) -> None:
-    _name_to_tokenizer_config[tokenizer_config.name] = tokenizer_config
+    ALL_TOKENIZER_CONFIGS.append(tokenizer_config)
+    TOKENIZER_NAME_TO_CONFIG[tokenizer_config.name] = tokenizer_config
 
 
 def register_tokenizer_configs_from_path(path: str) -> None:
@@ -61,4 +65,12 @@ def maybe_register_tokenizer_configs_from_base_path(base_path: str) -> None:
 
 
 def get_tokenizer_config(name: str) -> Optional[TokenizerConfig]:
-    return _name_to_tokenizer_config.get(name)
+    register_tokenizers_if_not_already_registered()
+    return TOKENIZER_NAME_TO_CONFIG.get(name)
+
+
+def register_tokenizers_if_not_already_registered() -> None:
+    global TOKENIZERS_REGISTERED
+    if not TOKENIZERS_REGISTERED:
+        maybe_register_tokenizer_configs_from_base_path(CONFIG_PATH)
+        TOKENIZERS_REGISTERED = True

src/helm/benchmark/window_services/test_ai21_window_service.py

@@ -10,7 +10,6 @@
 from .test_utils import TEST_PROMPT
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
-from helm.benchmark.model_deployment_registry import maybe_register_helm
 
 # TODO(#1522): Remove "▁" from the tokens.
 TEST_TOKEN_REPRESENTATIONS: List[TokenizationToken] = [
@@ -106,9 +105,6 @@
 
 @unittest.skip("The requests and results cannot be unpicked after the modules moved")
 class TestAI21WindowService:
-    def setup_class(self):
-        maybe_register_helm()
-
     def setup_method(self):
         # We use mocking for tokenization calls so no valid api_keys are required.
         auth = Authentication(api_key="DUMMY_API_KEY")

src/helm/benchmark/window_services/test_anthropic_window_service.py

@@ -5,7 +5,6 @@
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
-from helm.benchmark.model_deployment_registry import maybe_register_helm
 
 
 class TestAnthropicWindowService:
@@ -119,9 +118,6 @@ class TestAnthropicWindowService:
         ".",
     ]
 
-    def setup_class(self):
-        maybe_register_helm()
-
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
         service: TokenizerService = get_tokenizer_service(self.path)

src/helm/benchmark/window_services/test_bloom_window_service.py

@@ -5,7 +5,6 @@
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
-from helm.benchmark.model_deployment_registry import maybe_register_helm
 
 
 class TestBloomWindowService:
@@ -63,9 +62,6 @@ class TestBloomWindowService:
         17,
     ]
 
-    def setup_class(self):
-        maybe_register_helm()
-
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
         service: TokenizerService = get_tokenizer_service(self.path)

src/helm/benchmark/window_services/test_cohere_window_service.py

@@ -11,13 +11,11 @@
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service
-from helm.benchmark.model_deployment_registry import maybe_register_helm
 
 
 class TestCohereWindowService:
     @classmethod
     def setup_class(cls):
-        maybe_register_helm()
         cls.path: str = tempfile.mkdtemp()
         cache_path: str = os.path.join(cls.path, "cache")
         ensure_directory_exists(cache_path)

src/helm/benchmark/window_services/test_flan_t5_window_service.py

@@ -3,13 +3,9 @@
 from helm.benchmark.window_services.test_t511b_window_service import TestT511bWindowService
 from helm.benchmark.window_services.window_service_factory import TokenizerService, WindowServiceFactory
 from helm.benchmark.window_services.test_utils import get_tokenizer_service
-from helm.benchmark.model_deployment_registry import maybe_register_helm
 
 
 class TestFlanT5WindowService(TestT511bWindowService):
-    def setup_class(self):
-        maybe_register_helm()
-
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
         service: TokenizerService = get_tokenizer_service(self.path)

src/helm/benchmark/window_services/test_gpt2_window_service.py

@@ -5,13 +5,9 @@
 
 from .test_utils import get_tokenizer_service, TEST_PROMPT, GPT2_TEST_TOKENS, GPT2_TEST_TOKEN_IDS
 from .window_service_factory import WindowServiceFactory
-from helm.benchmark.model_deployment_registry import maybe_register_helm
 
 
 class TestGPT2WindowService:
-    def setup_class(self):
-        maybe_register_helm()
-
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
         service: TokenizerService = get_tokenizer_service(self.path)

src/helm/benchmark/window_services/test_gpt4_window_service.py

@@ -4,13 +4,9 @@
 from .test_utils import get_tokenizer_service, TEST_PROMPT, GPT4_TEST_TOKEN_IDS, GPT4_TEST_TOKENS
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
-from helm.benchmark.model_deployment_registry import maybe_register_helm
 
 
 class TestOpenAIWindowService:
-    def setup_class(self):
-        maybe_register_helm()
-
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
         service: TokenizerService = get_tokenizer_service(self.path)

src/helm/benchmark/window_services/test_gptj_window_service.py

@@ -4,13 +4,9 @@
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, GPT2_TEST_TOKENS, GPT2_TEST_TOKEN_IDS, TEST_PROMPT
-from helm.benchmark.model_deployment_registry import maybe_register_helm
 
 
 class TestGPTJWindowService:
-    def setup_class(self):
-        maybe_register_helm()
-
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
         service: TokenizerService = get_tokenizer_service(self.path)

src/helm/benchmark/window_services/test_gptneox_window_service.py

@@ -5,7 +5,6 @@
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
-from helm.benchmark.model_deployment_registry import maybe_register_helm
 
 
 class TestGPTNeoXWindowService:
@@ -64,9 +63,6 @@ class TestGPTNeoXWindowService:
         15,
     ]
 
-    def setup_class(self):
-        maybe_register_helm()
-
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
         service: TokenizerService = get_tokenizer_service(self.path)

src/helm/benchmark/window_services/test_ice_window_service.py

@@ -5,7 +5,6 @@
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
-from helm.benchmark.model_deployment_registry import maybe_register_helm
 
 
 class TestICEWindowService:
@@ -63,9 +62,6 @@ class TestICEWindowService:
         20007,
     ]
 
-    def setup_class(self):
-        maybe_register_helm()
-
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
         service: TokenizerService = get_tokenizer_service(self.path)

src/helm/benchmark/window_services/test_mt_nlg_window_service.py

@@ -4,13 +4,9 @@
 from .test_utils import get_tokenizer_service, TEST_PROMPT, GPT2_TEST_TOKENS, GPT2_TEST_TOKEN_IDS
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
-from helm.benchmark.model_deployment_registry import maybe_register_helm
 
 
 class TestMTNLGWindowService:
-    def setup_class(self):
-        maybe_register_helm()
-
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
         service: TokenizerService = get_tokenizer_service(self.path)

src/helm/benchmark/window_services/test_openai_window_service.py

@@ -4,13 +4,9 @@
 from .test_utils import get_tokenizer_service, TEST_PROMPT, GPT2_TEST_TOKENS, GPT2_TEST_TOKEN_IDS
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
-from helm.benchmark.model_deployment_registry import maybe_register_helm
 
 
 class TestOpenAIWindowService:
-    def setup_class(self):
-        maybe_register_helm()
-
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
         service: TokenizerService = get_tokenizer_service(self.path)