From 6130334e97d4fdc89caae44748a121befce7417c Mon Sep 17 00:00:00 2001 From: Vitaliy Chiley <6439018+vchiley@users.noreply.github.com> Date: Tue, 28 Nov 2023 15:32:25 -0800 Subject: [PATCH] Apply suggestions from code review Co-authored-by: Daniel King <43149077+dakinggg@users.noreply.github.com> --- llmfoundry/utils/builders.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llmfoundry/utils/builders.py b/llmfoundry/utils/builders.py index 8665fd7ff7..d7a7531d35 100644 --- a/llmfoundry/utils/builders.py +++ b/llmfoundry/utils/builders.py @@ -189,7 +189,7 @@ def _extract_param_groups( } ``` - To create modify the optimizer parameters for all parameters that contain the + To create and modify the optimizer parameters for all parameters that contain the string "norm" and "bias" separately: ``` optimizer_config: { @@ -219,7 +219,7 @@ def _extract_param_groups( Returns: Union[Iterable[torch.Tensor], Iterable[Dict[str, Any]]]: an iterable of - torch.Tensor's or dict's. Specifies what Tensors should be optimized. + torch.Tensor's or dict's. Specifies what Tensors should be optimized and their param groupings. """ if 'disable_grad' in optimizer_config.keys(): str_matches = optimizer_config.pop('disable_grad')