diff --git a/mcli/mcli-llama2-finetune.yaml b/mcli/mcli-llama2-finetune.yaml
index 1cacda4e99..1fbeb2d48d 100644
--- a/mcli/mcli-llama2-finetune.yaml
+++ b/mcli/mcli-llama2-finetune.yaml
@@ -140,6 +140,8 @@ parameters:
     memory_monitor: {}
     runtime_estimator: {}
 
+  load_weights_only: true # Only load the weights, not the optimizer state, LR schedule, etc
+
 #   loggers:
 #     wandb: {}
 
diff --git a/scripts/train/yamls/finetune/1b_local_data_sft.yaml b/scripts/train/yamls/finetune/1b_local_data_sft.yaml
index 3ee3f9d5cd..aa4484e62d 100644
--- a/scripts/train/yamls/finetune/1b_local_data_sft.yaml
+++ b/scripts/train/yamls/finetune/1b_local_data_sft.yaml
@@ -136,3 +136,4 @@ callbacks:
 # Load from remote object store
 # REPLACE THE BELOW with you own checkpoint!
 load_path: oci://my-bucket/my-folder/mpt-1b/checkpoints/some_checkpoint.pt
+load_weights_only: true # Only load the weights, not the optimizer state, LR schedule, etc
diff --git a/scripts/train/yamls/finetune/7b_dolly_sft.yaml b/scripts/train/yamls/finetune/7b_dolly_sft.yaml
index ffe9fd6c10..53d8eecbca 100644
--- a/scripts/train/yamls/finetune/7b_dolly_sft.yaml
+++ b/scripts/train/yamls/finetune/7b_dolly_sft.yaml
@@ -124,3 +124,4 @@ callbacks:
 # Load from remote object store
 # REPLACE THE BELOW with you own checkpoint!
 load_path: oci://my-bucket/my-folder/mpt-7b/checkpoints/some_checkpoint.pt
+load_weights_only: true # Only load the weights, not the optimizer state, LR schedule, etc
diff --git a/scripts/train/yamls/finetune/mpt-30b-instruct.yaml b/scripts/train/yamls/finetune/mpt-30b-instruct.yaml
index fab2594847..7966b88471 100644
--- a/scripts/train/yamls/finetune/mpt-30b-instruct.yaml
+++ b/scripts/train/yamls/finetune/mpt-30b-instruct.yaml
@@ -120,13 +120,6 @@ callbacks:
 # save_interval: 3ep
 # save_num_checkpoints_to_keep: 1
 
-# need to use converted checkpoint with llm-foundry code
-# load_path:
-autoresume: false
-load_weights_only: false
-python_log_level: debug
-
-
 icl_max_seq_len: 2048
 
 # YOU MUST ADD YOUR OWN DATASET URIs
diff --git a/scripts/train/yamls/finetune/mpt-7b_domain_adapt.yaml b/scripts/train/yamls/finetune/mpt-7b_domain_adapt.yaml
index 845b24b7e4..e533b9118c 100644
--- a/scripts/train/yamls/finetune/mpt-7b_domain_adapt.yaml
+++ b/scripts/train/yamls/finetune/mpt-7b_domain_adapt.yaml
@@ -115,3 +115,4 @@ save_folder: ./{run_name}/checkpoints
 # Load from local filesystem or remote object store
 # load_path: ./gpt-7b/checkpoints/latest-rank{rank}.pt
 # load_path: s3://my-bucket/my-folder/gpt-7b/checkpoints/latest-rank{rank}.pt
+load_weights_only: true # Only load the weights, not the optimizer state, LR schedule, etc
diff --git a/scripts/train/yamls/finetune/t5-small_dolly_sft.yaml b/scripts/train/yamls/finetune/t5-small_dolly_sft.yaml
index c54f5b9db3..b4d24ef0ff 100644
--- a/scripts/train/yamls/finetune/t5-small_dolly_sft.yaml
+++ b/scripts/train/yamls/finetune/t5-small_dolly_sft.yaml
@@ -101,3 +101,4 @@ callbacks:
 # Load from remote object store
 # REPLACE THE BELOW with you own checkpoint!
 # load_path: oci://my-bucket/my-folder/checkpoints/some_checkpoint.pt
+load_weights_only: true # Only load the weights, not the optimizer state, LR schedule, etc