Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Deprecated use_auth_token #1837

Merged
merged 7 commits into from
Jun 25, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 26 additions & 4 deletions optimum/configuration_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
import json
import os
import re
import warnings
from typing import Any, Dict, List, Tuple, Union

from huggingface_hub import HfFolder
from packaging import version
from transformers import PretrainedConfig
from transformers import __version__ as transformers_version_str
Expand Down Expand Up @@ -93,7 +93,19 @@ def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub:
repo_id = self._create_repo(repo_id, **kwargs)

use_auth_token = kwargs.get("use_auth_token", None)
token = HfFolder.get_token() if use_auth_token is True else use_auth_token
token = kwargs.get("token", None)

if use_auth_token is not None:
warnings.warn(
"The `use_auth_token` argument is deprecated and will be removed soon. Please use the `token` argument instead.",
FutureWarning,
)
if token is not None:
raise ValueError(
"You cannot use both `use_auth_token` and `token` arguments at the same time."
)
kwargs["token"] = use_auth_token
token = use_auth_token

files_timestamps = self._get_files_timestamps(save_directory)

Expand Down Expand Up @@ -197,6 +209,7 @@ def _get_config_dict(
resume_download = kwargs.pop("resume_download", False)
proxies = kwargs.pop("proxies", None)
use_auth_token = kwargs.pop("use_auth_token", None)
token = kwargs.pop("token", None)
local_files_only = kwargs.pop("local_files_only", False)
revision = kwargs.pop("revision", None)
trust_remote_code = kwargs.pop("trust_remote_code", None)
Expand All @@ -205,6 +218,15 @@ def _get_config_dict(
from_auto_class = kwargs.pop("_from_auto", False)
commit_hash = kwargs.pop("_commit_hash", None)

if use_auth_token is not None:
warnings.warn(
IlyasMoutawwakil marked this conversation as resolved.
Show resolved Hide resolved
"The `use_auth_token` argument is deprecated and will be removed soon. Please use the `token` argument instead.",
FutureWarning,
)
if token is not None:
raise ValueError("You cannot use both `use_auth_token` and `token` arguments at the same time.")
token = use_auth_token

if trust_remote_code is True:
logger.warning(
"The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is"
Expand Down Expand Up @@ -255,7 +277,7 @@ def _get_config_dict(
proxies=proxies,
resume_download=resume_download,
local_files_only=local_files_only,
use_auth_token=use_auth_token,
token=token,
user_agent=user_agent,
)
else:
Expand All @@ -268,7 +290,7 @@ def _get_config_dict(
proxies=proxies,
resume_download=resume_download,
local_files_only=local_files_only,
use_auth_token=use_auth_token,
token=token,
user_agent=user_agent,
revision=revision,
subfolder=subfolder,
Expand Down
21 changes: 17 additions & 4 deletions optimum/exporters/onnx/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
"""Entry point to the optimum.exporters.onnx command line."""

import argparse
import warnings
from pathlib import Path

from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE
Expand Down Expand Up @@ -66,6 +67,7 @@ def main_export(
force_download: bool = False,
local_files_only: bool = False,
use_auth_token: Optional[Union[bool, str]] = None,
token: Optional[Union[bool, str]] = None,
for_ort: bool = False,
do_validation: bool = True,
model_kwargs: Optional[Dict[str, Any]] = None,
Expand Down Expand Up @@ -135,9 +137,11 @@ def main_export(
cached versions if they exist.
local_files_only (`Optional[bool]`, defaults to `False`):
Whether or not to only look at local files (i.e., do not try to download the model).
use_auth_token (`Optional[str]`, defaults to `None`):
use_auth_token (`Optional[Union[bool,str]]`, defaults to `None`):
Deprecated. Please use the `token` argument instead.
token (`Optional[Union[bool,str]]`, defaults to `None`):
The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
when running `transformers-cli login` (stored in `~/.huggingface`).
when running `huggingface-cli login` (stored in `huggingface_hub.constants.HF_TOKEN_PATH`).
model_kwargs (`Optional[Dict[str, Any]]`, defaults to `None`):
Experimental usage: keyword arguments to pass to the model during
the export. This argument should be used along the `custom_onnx_configs` argument
Expand Down Expand Up @@ -174,6 +178,15 @@ def main_export(
```
"""

if use_auth_token is not None:
warnings.warn(
"The `use_auth_token` argument is deprecated and will be removed soon. Please use the `token` argument instead.",
FutureWarning,
)
if token is not None:
raise ValueError("You cannot use both `use_auth_token` and `token` arguments at the same time.")
token = use_auth_token

if fp16:
if dtype is not None:
raise ValueError(
Expand Down Expand Up @@ -250,7 +263,7 @@ def main_export(
subfolder=subfolder,
revision=revision,
cache_dir=cache_dir,
use_auth_token=use_auth_token,
token=token,
local_files_only=local_files_only,
force_download=force_download,
trust_remote_code=trust_remote_code,
Expand Down Expand Up @@ -283,7 +296,7 @@ def main_export(
subfolder=subfolder,
revision=revision,
cache_dir=cache_dir,
use_auth_token=use_auth_token,
token=token,
local_files_only=local_files_only,
force_download=force_download,
trust_remote_code=trust_remote_code,
Expand Down
58 changes: 48 additions & 10 deletions optimum/exporters/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@
import inspect
import itertools
import os
import warnings
from functools import partial
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union

import huggingface_hub
from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE
from huggingface_hub.utils import OfflineModeIsEnabled
IlyasMoutawwakil marked this conversation as resolved.
Show resolved Hide resolved
from packaging import version
from requests.exceptions import ConnectionError as RequestsConnectionError
from transformers import AutoConfig, PretrainedConfig, is_tf_available, is_torch_available
Expand Down Expand Up @@ -1379,9 +1381,19 @@ def get_model_files(
model_name_or_path: Union[str, Path],
subfolder: str = "",
cache_dir: str = HUGGINGFACE_HUB_CACHE,
use_auth_token: Optional[str] = None,
use_auth_token: Optional[Union[bool, str]] = None,
token: Optional[Union[bool, str]] = None,
revision: Optional[str] = None,
):
if use_auth_token is not None:
warnings.warn(
"The `use_auth_token` argument is deprecated and will be removed soon. Please use the `token` argument instead.",
FutureWarning,
)
if token is not None:
raise ValueError("You cannot use both `use_auth_token` and `token` arguments at the same time.")
token = use_auth_token

request_exception = None
full_model_path = Path(model_name_or_path) / subfolder
if full_model_path.is_dir():
Expand All @@ -1397,12 +1409,12 @@ def get_model_files(
all_files = huggingface_hub.list_repo_files(
model_name_or_path,
repo_type="model",
token=use_auth_token,
token=token,
revision=revision,
)
if subfolder != "":
all_files = [file[len(subfolder) + 1 :] for file in all_files if file.startswith(subfolder)]
except (RequestsConnectionError, huggingface_hub.utils._http.OfflineModeIsEnabled) as e:
except (RequestsConnectionError, OfflineModeIsEnabled) as e:
request_exception = e
object_id = model_name_or_path.replace("/", "--")
full_model_path = Path(cache_dir, f"models--{object_id}")
Expand Down Expand Up @@ -1576,7 +1588,7 @@ def _infer_task_from_model_name_or_path(
)
try:
model_info = huggingface_hub.model_info(model_name_or_path, revision=revision)
except (RequestsConnectionError, huggingface_hub.utils._http.OfflineModeIsEnabled):
except (RequestsConnectionError, OfflineModeIsEnabled):
raise RuntimeError(
f"Hugging Face Hub is not reachable and we cannot infer the task from a cached model. Make sure you are not offline, or otherwise please specify the `task` (or `--task` in command-line) argument ({', '.join(TasksManager.get_all_tasks())})."
)
Expand Down Expand Up @@ -1693,7 +1705,8 @@ def infer_library_from_model(
revision: Optional[str] = None,
cache_dir: str = HUGGINGFACE_HUB_CACHE,
library_name: Optional[str] = None,
use_auth_token: Optional[str] = None,
use_auth_token: Optional[Union[bool, str]] = None,
token: Optional[Union[bool, str]] = None,
):
"""
Infers the library from the model repo.
Expand All @@ -1711,16 +1724,30 @@ def infer_library_from_model(
Path to a directory in which a downloaded pretrained model weights have been cached if the standard cache should not be used.
library_name (`Optional[str]`, *optional*):
The library name of the model. Can be any of "transformers", "timm", "diffusers", "sentence_transformers".
use_auth_token (`Optional[str]`, defaults to `None`):
The token to use as HTTP bearer authorization for remote files.
use_auth_token (`Optional[Union[bool,str]]`, defaults to `None`):
Deprecated. Please use the `token` argument instead.
token (`Optional[Union[bool,str]]`, defaults to `None`):
The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
when running `huggingface-cli login` (stored in `huggingface_hub.constants.HF_TOKEN_PATH`).

Returns:
`str`: The library name automatically detected from the model repo.
"""

if use_auth_token is not None:
warnings.warn(
"The `use_auth_token` argument is deprecated and will be removed soon. Please use the `token` argument instead.",
FutureWarning,
)
if token is not None:
raise ValueError("You cannot use both `use_auth_token` and `token` arguments at the same time.")
token = use_auth_token

if library_name is not None:
return library_name

all_files, _ = TasksManager.get_model_files(
model_name_or_path, subfolder, cache_dir, use_auth_token=use_auth_token
model_name_or_path, subfolder, cache_dir, token=token, revision=revision
)

if "model_index.json" in all_files:
Expand All @@ -1736,7 +1763,7 @@ def infer_library_from_model(
"subfolder": subfolder,
"revision": revision,
"cache_dir": cache_dir,
"use_auth_token": use_auth_token,
"token": token,
}
config_dict, kwargs = PretrainedConfig.get_config_dict(model_name_or_path, **kwargs)
model_config = PretrainedConfig.from_dict(config_dict, **kwargs)
Expand Down Expand Up @@ -1912,12 +1939,23 @@ def get_model_from_task(
elif library_name == "sentence_transformers":
cache_folder = model_kwargs.pop("cache_folder", None)
use_auth_token = model_kwargs.pop("use_auth_token", None)
token = model_kwargs.pop("token", None)
trust_remote_code = model_kwargs.pop("trust_remote_code", False)

if use_auth_token is not None:
warnings.warn(
"The `use_auth_token` argument is deprecated and will be removed soon. Please use the `token` argument instead.",
FutureWarning,
)
if token is not None:
raise ValueError("You cannot use both `use_auth_token` and `token` arguments at the same time.")
token = use_auth_token

model = model_class(
model_name_or_path,
device=device,
cache_folder=cache_folder,
use_auth_token=use_auth_token,
token=token,
trust_remote_code=trust_remote_code,
)
else:
Expand Down
Loading
Loading