From 17bfa7e4051b6d5e4e0b211340335fc9857f8ffd Mon Sep 17 00:00:00 2001 From: Mihir Patel Date: Tue, 13 Feb 2024 12:01:33 -0500 Subject: [PATCH 1/2] Revert "Add bias argument to LPLN (#2999)" (#3003) This reverts commit 94b63d74b8a54ccaae97c26b43b5da4dec50f1ce. --- .../low_precision_layernorm.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/composer/algorithms/low_precision_layernorm/low_precision_layernorm.py b/composer/algorithms/low_precision_layernorm/low_precision_layernorm.py index 6baffd43e3..64ffaebb11 100644 --- a/composer/algorithms/low_precision_layernorm/low_precision_layernorm.py +++ b/composer/algorithms/low_precision_layernorm/low_precision_layernorm.py @@ -75,21 +75,11 @@ def apply(self, event: Event, state: State, logger: Logger) -> Optional[int]: class LPLayerNorm(torch.nn.LayerNorm): - def __init__( - self, - normalized_shape, - eps: float = 1e-05, - elementwise_affine: bool = True, - *, - bias: bool = True, - device=None, - dtype=None, - ): + def __init__(self, normalized_shape, eps=1e-05, elementwise_affine=True, device=None, dtype=None): super().__init__( normalized_shape=normalized_shape, eps=eps, elementwise_affine=elementwise_affine, - bias=bias, device=device, dtype=dtype, ) @@ -121,7 +111,7 @@ def _to_LPLayerNorm(layer: torch.nn.Module, module_index: int) -> LPLayerNorm: """Defines a replacement policy from a `torch.nn.LayerNorm` to a `LPLayerNorm`""" if not isinstance(layer, torch.nn.LayerNorm): raise TypeError(f'Expected torch.nn.LayerNorm, got {type(layer)}') - lp_layernorm = LPLayerNorm(layer.normalized_shape, layer.eps, layer.elementwise_affine) # type: ignore + lp_layernorm = LPLayerNorm(layer.normalized_shape, layer.eps, layer.elementwise_affine) with torch.no_grad(): if layer.weight is None: # pyright: ignore[reportUnnecessaryComparison] From 0d041acb370f46395cac721b9e92fac277ac527b Mon Sep 17 00:00:00 2001 From: Mihir Patel Date: Tue, 13 Feb 2024 12:01:41 -0500 Subject: [PATCH 2/2] Revert "fix (#3000)" (#3004) This reverts commit b1e7db4f5655a338df508da4a02a67d46991214a. --- composer/cli/launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer/cli/launcher.py b/composer/cli/launcher.py index f26030c745..c72e1b01db 100755 --- a/composer/cli/launcher.py +++ b/composer/cli/launcher.py @@ -291,7 +291,7 @@ def _launch_processes( MASTER_ADDR=master_addr, MASTER_PORT=str(master_port), PYTHONUNBUFFERED='1', - TORCH_NCCL_ASYNC_ERROR_HANDLING='1', + NCCL_ASYNC_ERROR_HANDLING='1', ): # Populate the distributed variables in all launcher args for arg in training_script_args: