diff --git a/optimum/neuron/utils/peft_utils.py b/optimum/neuron/utils/peft_utils.py
index 4855f2ef2..8c753ec38 100644
--- a/optimum/neuron/utils/peft_utils.py
+++ b/optimum/neuron/utils/peft_utils.py
@@ -176,7 +176,7 @@ def state_dict(self):
 
                 adapter_shards_dir_model = os.path.join(output_dir, "adapter_shards", "model")
                 if not os.path.isdir(adapter_shards_dir_model):
-                    os.makedirs(adapter_shards_dir_model)
+                    os.makedirs(adapter_shards_dir_model, exist_ok=True)
 
                 dummy_mod = DummyModule()
                 neuronx_distributed.trainer.save_checkpoint(