You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
已经改掉了NCCL,
接着出现:
Traceback (most recent call last):
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 504, in
train(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 323, in train
encoder_hidden_states = train_util.get_hidden_states(
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\library\train_util.py", line 4215, in get_hidden_states
encoder_hidden_states = text_encoder.text_model.final_layer_norm(encoder_hidden_states)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\nn\modules\module.py", line 1688, in getattr
raise AttributeError(f"'{type(self).name}' object has no attribute '{name}'")
AttributeError: 'DistributedDataParallel' object has no attribute 'text_model'
Traceback (most recent call last):
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 504, in
train(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 323, in train
encoder_hidden_states = train_util.get_hidden_states(
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\library\train_util.py", line 4215, in get_hidden_states
encoder_hidden_states = text_encoder.text_model.final_layer_norm(encoder_hidden_states)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\nn\modules\module.py", line 1688, in getattr
raise AttributeError(f"'{type(self).name}' object has no attribute '{name}'")
AttributeError: 'DistributedDataParallel' object has no attribute 'text_model'
steps: 0%| | 0/1140 [00:50<?, ?it/s]
[2024-05-12 03:28:35,348] torch.distributed.elastic.multiprocessing.api: [ERROR] failed (exitcode: 1) local_rank: 0 (pid: 34304) of binary: E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\python.exe
Traceback (most recent call last):
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\runpy.py", line 86, in _run_code
exec(code, run_globals)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 1027, in
main()
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 1023, in main
launch_command(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 1008, in launch_command
multi_gpu_launcher(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 666, in multi_gpu_launcher
distrib_run.run(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\distributed\run.py", line 803, in run
elastic_launch(
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\distributed\launcher\api.py", line 135, in call
return launch_agent(self._config, self._entrypoint, list(args))
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\distributed\launcher\api.py", line 268, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
已经改掉了NCCL,
接着出现:
Traceback (most recent call last):
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 504, in
train(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 323, in train
encoder_hidden_states = train_util.get_hidden_states(
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\library\train_util.py", line 4215, in get_hidden_states
encoder_hidden_states = text_encoder.text_model.final_layer_norm(encoder_hidden_states)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\nn\modules\module.py", line 1688, in getattr
raise AttributeError(f"'{type(self).name}' object has no attribute '{name}'")
AttributeError: 'DistributedDataParallel' object has no attribute 'text_model'
Traceback (most recent call last):
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 504, in
train(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 323, in train
encoder_hidden_states = train_util.get_hidden_states(
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\library\train_util.py", line 4215, in get_hidden_states
encoder_hidden_states = text_encoder.text_model.final_layer_norm(encoder_hidden_states)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\nn\modules\module.py", line 1688, in getattr
raise AttributeError(f"'{type(self).name}' object has no attribute '{name}'")
AttributeError: 'DistributedDataParallel' object has no attribute 'text_model'
steps: 0%| | 0/1140 [00:50<?, ?it/s]
[2024-05-12 03:28:35,348] torch.distributed.elastic.multiprocessing.api: [ERROR] failed (exitcode: 1) local_rank: 0 (pid: 34304) of binary: E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\python.exe
Traceback (most recent call last):
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\runpy.py", line 86, in _run_code
exec(code, run_globals)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 1027, in
main()
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 1023, in main
launch_command(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 1008, in launch_command
multi_gpu_launcher(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 666, in multi_gpu_launcher
distrib_run.run(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\distributed\run.py", line 803, in run
elastic_launch(
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\distributed\launcher\api.py", line 135, in call
return launch_agent(self._config, self._entrypoint, list(args))
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\distributed\launcher\api.py", line 268, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
./sd-scripts/train_db.py FAILED
Failures:
[1]:
time : 2024-05-12_03:28:35
host : lsy.ft.com
rank : 1 (local_rank: 1)
exitcode : 1 (pid: 26340)
error_file: <N/A>
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
Root Cause (first observed failure):
[0]:
time : 2024-05-12_03:28:35
host : lsy.ft.com
rank : 0 (local_rank: 0)
exitcode : 1 (pid: 34304)
error_file: <N/A>
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
在kohya-ss中看到有一个Issues提到有个分支解决了这方面的问题
kohya-ss/sd-scripts#1019
The text was updated successfully, but these errors were encountered: