Skip to content

Commit

Permalink
Cleaning up
Browse files Browse the repository at this point in the history
  • Loading branch information
@JosselinSomervilleRoberts
JosselinSomervilleRoberts committed Nov 4, 2023
1 parent 1a71401 commit 3603549
Show file tree
Hide file tree
Showing 7 changed files with 0 additions and 1,136 deletions.
7 changes: 0 additions & 7 deletions src/helm/benchmark/model_deployment_registry.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@
FULL_FUNCTIONALITY_TEXT_MODEL_TAG,
)

from toolbox.printing import print_visible, debug # TODO(PR): Remove this


MODEL_DEPLOYMENTS_FILE = "model_deployments.yaml"

Expand Down Expand Up @@ -107,8 +105,6 @@ def register_model_deployment(model_deployment: ModelDeployment) -> None:
ALL_MODEL_DEPLOYMENTS.append(model_deployment)

model_name: str = model_deployment.model_name or model_deployment.name
print_visible("register_model_deployment")
debug(model_name, visible=True)

try:
model_metadata: ModelMetadata = get_model_metadata(model_name)
Expand Down Expand Up @@ -137,18 +133,15 @@ def register_model_deployment(model_deployment: ModelDeployment) -> None:

def register_model_deployments_from_path(path: str) -> None:
hlog(f"Reading model deployments from {path}...")
print_visible("register_model_deployments_from_path")
with open(path, "r") as f:
raw = yaml.safe_load(f)
model_deployments: ModelDeployments = cattrs.structure(raw, ModelDeployments)
for model_deployment in model_deployments.model_deployments:
debug(model_deployment, visible=True)
register_model_deployment(model_deployment)


def maybe_register_model_deployments_from_base_path(base_path: str) -> None:
"""Register model deployments from prod_env/model_deployments.yaml"""
print_visible("maybe_register_model_deployments_from_base_path")
path = os.path.join(base_path, MODEL_DEPLOYMENTS_FILE)
if os.path.exists(path):
register_model_deployments_from_path(path)
Expand Down
14 changes: 0 additions & 14 deletions src/helm/benchmark/model_metadata_registry.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@

from helm.common.hierarchical_logger import hlog

from toolbox.printing import debug, print_visible # TODO(PR): Remove this


# Different modalities
TEXT_MODEL_TAG: str = "TEXT_MODEL_TAG"
Expand Down Expand Up @@ -131,19 +129,12 @@ class ModelMetadataList:
# ===================== REGISTRATION FUNCTIONS ==================== #
def register_model_metadata_from_path(path: str) -> None:
"""Register model configurations from the given path."""
print_visible("register_model_metadata_from_path")
with open(path, "r") as f:
raw = yaml.safe_load(f)
debug(raw, visible=True)
# Using dacite instead of cattrs because cattrs doesn't have a default
# serialization format for dates
model_metadata_list = dacite.from_dict(ModelMetadataList, raw)
debug(model_metadata_list, visible=True)
for model_metadata in model_metadata_list.models:
debug(model_metadata, visible=True)
debug(model_metadata.tags, visible=True)
debug(ANTHROPIC_CLAUDE_1_MODEL_TAG, visible=True)
debug(ANTHROPIC_CLAUDE_1_MODEL_TAG in model_metadata.tags, visible=True)
register_model_metadata(model_metadata)


Expand All @@ -156,19 +147,14 @@ def register_model_metadata(model_metadata: ModelMetadata) -> None:

def maybe_register_model_metadata_from_base_path(base_path: str) -> None:
"""Register model metadata from prod_env/model_metadata.yaml"""
print_visible("maybe_register_model_metadata_from_base_path")
debug(base_path)
path = os.path.join(base_path, MODEL_METADATA_FILE)
debug(path)
if os.path.exists(path):
register_model_metadata_from_path(path)


# ===================== UTIL FUNCTIONS ==================== #
def get_model_metadata(model_name: str) -> ModelMetadata:
"""Get the `Model` given the name."""
debug(model_name, visible=True)
# debug(MODEL_NAME_TO_MODEL_METADATA, visible=True) # TODO(PR): Remove this
if model_name not in MODEL_NAME_TO_MODEL_METADATA:
raise ValueError(f"No model with name: {model_name}")

Expand Down
10 changes: 0 additions & 10 deletions src/helm/benchmark/run_specs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2568,20 +2568,11 @@ def construct_run_specs(spec: ObjectSpec) -> List[RunSpec]:
]

def alter_run_spec(run_spec: RunSpec) -> RunSpec:
from toolbox.printing import print_visible, debug

print_visible("ALTER RUN SPEC") # TODO(PR): Remove
try:
deployment: ModelDeployment = get_model_deployment(run_spec.adapter_spec.model_deployment)
debug(deployment, visible=True)
model_name: str = deployment.model_name or deployment.name
model: ModelMetadata = get_model_metadata(model_name)
debug(model, visible=True)
debug(model.tags, visible=True)
debug(ANTHROPIC_CLAUDE_1_MODEL_TAG in model.tags, visible=True)
except ValueError as e:
print_visible("EXCEPTION") # TODO(PR): Remove
print_visible("Error: " + str(e)) # TODO(PR): Remove
# Models registered from configs cannot have expanders applied to them,
# because the models will not have been registered yet at this point.
# TODO: Figure out a cleaner way to deal with this.
Expand Down Expand Up @@ -2611,7 +2602,6 @@ def alter_run_spec(run_spec: RunSpec) -> RunSpec:

# Special handling for Anthropic Claude
if ANTHROPIC_CLAUDE_1_MODEL_TAG in model.tags or ANTHROPIC_CLAUDE_2_MODEL_TAG in model.tags:
print_visible("ANTHROPIC MODEL TAG") # TODO(PR): Remove
try:
import anthropic
from helm.proxy.clients.anthropic_client import AnthropicClient
Expand Down
133 changes: 0 additions & 133 deletions src/helm/proxy/clients/auto_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@
from helm.proxy.tokenizers.tokenizer import Tokenizer
from helm.proxy.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer

# TODO(PR): Remove this
# from .http_model_client import HTTPModelClient

if TYPE_CHECKING:
import helm.proxy.clients.huggingface_client
Expand Down Expand Up @@ -90,8 +88,6 @@ def _get_client(self, model: str) -> Client:
host_group: str = model.split("/")[0]
cache_config: CacheConfig = self._build_cache_config(host_group)

# TODO: Migrate all clients to use model deployments
# TODO(PR): Remove the TODO above.
model_deployment = get_model_deployment(model)
if model_deployment:
# Get tokenizer from model deployment
Expand Down Expand Up @@ -122,135 +118,6 @@ def _get_client(self, model: str) -> Client:
provider_bindings={"api_key": self.get_provide_api_key_callable(host_group, model)},
)
client = create_object(client_spec)

# TODO(PR): Remove all of this and set it up in the model deployment registry.
# elif host_group == "neurips":
# client = HTTPModelClient(tokenizer=tokenizer, cache_config=cache_config)
# elif host_group == "openai":
# from helm.proxy.clients.openai_client import OpenAIClient

# org_id = self.credentials.get("openaiOrgId", None)
# api_key = self.credentials.get("openaiApiKey", None)
# client = OpenAIClient(
# tokenizer=tokenizer,
# cache_config=cache_config,
# api_key=api_key,
# org_id=org_id,
# )
# elif host_group == "AlephAlpha":
# from helm.proxy.clients.aleph_alpha_client import AlephAlphaClient

# client = AlephAlphaClient(
# tokenizer=tokenizer,
# api_key=self.credentials["alephAlphaKey"],
# cache_config=cache_config,
# )
# elif host_group == "ai21":
# from helm.proxy.clients.ai21_client import AI21Client

# client = AI21Client(
# tokenizer=tokenizer,
# api_key=self.credentials["ai21ApiKey"],
# cache_config=cache_config,
# )
# elif host_group == "cohere":
# from helm.proxy.clients.cohere_client import CohereClient

# client = CohereClient(
# tokenizer=tokenizer,
# api_key=self.credentials["cohereApiKey"],
# cache_config=cache_config,
# )
# elif host_group == "gooseai":
# from helm.proxy.clients.goose_ai_client import GooseAIClient

# org_id = self.credentials.get("gooseaiOrgId", None)
# client = GooseAIClient(
# tokenizer=tokenizer,
# api_key=self.credentials["gooseaiApiKey"],
# cache_config=cache_config,
# org_id=org_id,
# )
# elif host_group == "huggingface":
# from helm.proxy.clients.huggingface_client import HuggingFaceClient

# client = HuggingFaceClient(tokenizer=tokenizer, cache_config=cache_config)
# elif host_group == "anthropic":
# from helm.proxy.clients.anthropic_client import AnthropicClient

# client = AnthropicClient(
# api_key=self.credentials.get("anthropicApiKey", None),
# tokenizer=tokenizer,
# cache_config=cache_config,
# )
# elif host_group == "microsoft":
# from helm.proxy.clients.microsoft_client import MicrosoftClient

# org_id = self.credentials.get("microsoftOrgId", None)
# lock_file_path: str = os.path.join(self.cache_path, f"{host_group}.lock")
# client = MicrosoftClient(
# api_key=self.credentials.get("microsoftApiKey", None),
# tokenizer=tokenizer,
# lock_file_path=lock_file_path,
# cache_config=cache_config,
# org_id=org_id,
# )
# elif host_group == "google":
# from helm.proxy.clients.google_client import GoogleClient

# client = GoogleClient(
# tokenizer=tokenizer,
# cache_config=cache_config,
# )
# elif host_group in [
# "together",
# "databricks",
# "eleutherai",
# "lmsys",
# "meta",
# "mistralai",
# "mosaicml",
# "stabilityai",
# "stanford",
# "tiiuae",
# ]:
# from helm.proxy.clients.together_client import TogetherClient

# client = TogetherClient(
# api_key=self.credentials.get("togetherApiKey", None),
# tokenizer=tokenizer,
# cache_config=cache_config,
# )
# elif host_group == "simple":
# from helm.proxy.clients.simple_client import SimpleClient

# client = SimpleClient(tokenizer=tokenizer, cache_config=cache_config)
# elif host_group == "writer":
# from helm.proxy.clients.palmyra_client import PalmyraClient

# client = PalmyraClient(
# api_key=self.credentials["writerApiKey"],
# tokenizer=tokenizer,
# cache_config=cache_config,
# )
# elif host_group == "nvidia":
# from helm.proxy.clients.megatron_client import MegatronClient

# client = MegatronClient(tokenizer=tokenizer, cache_config=cache_config)

# elif host_group == "lightningai":
# from helm.proxy.clients.lit_gpt_client import LitGPTClient

# client = LitGPTClient(
# tokenizer=tokenizer,
# cache_config=cache_config,
# checkpoint_dir=Path(os.environ.get("LIT_GPT_CHECKPOINT_DIR", "")),
# precision=os.environ.get("LIT_GPT_PRECISION", "bf16-true"),
# )
# elif organization == "HuggingFaceM4":
# from helm.proxy.clients.vision_language.idefics_client import IDEFICSClient

# client = IDEFICSClient(tokenizer=tokenizer, cache_config=cache_config)
else:
raise ValueError(f"Could not find client for model: {model}")
self.clients[model] = client
Expand Down
1 change: 0 additions & 1 deletion src/helm/proxy/clients/cohere_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ def make_request(self, request: Request) -> RequestResult:
assert request.max_tokens > 0, "max_tokens can only be 0 if echo_prompt=True"

# model: "Currently available models are small, medium, large, xlarge"
# TODO(PR): Check if we want deployments or metadata here
assert request.model in get_model_deployments_by_host_group("cohere")
# temperature: "min value of 0.0, max value of 5.0"
assert 0.0 <= request.temperature <= 5.0, f"Invalid temperature: {request.temperature}. Valid range: [0,5]"
Expand Down
Loading

0 comments on commit 3603549

Please sign in to comment.