From 9cf375e3a6aa4a49a5119170691f69870c80dba1 Mon Sep 17 00:00:00 2001
From: Irene Dea <deaairene@gmail.com>
Date: Tue, 19 Sep 2023 19:41:32 -0700
Subject: [PATCH] Fix eval yamls (#609)

---
 scripts/eval/yamls/hf_8bit_eval.yaml | 2 +-
 scripts/eval/yamls/hf_eval.yaml      | 2 +-
 scripts/eval/yamls/hf_lora_eval.yml  | 6 ++++--
 scripts/eval/yamls/mpt_eval.yaml     | 4 ++--
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/scripts/eval/yamls/hf_8bit_eval.yaml b/scripts/eval/yamls/hf_8bit_eval.yaml
index 4f195cdc54..4e1e9c8dc7 100644
--- a/scripts/eval/yamls/hf_8bit_eval.yaml
+++ b/scripts/eval/yamls/hf_8bit_eval.yaml
@@ -10,7 +10,7 @@ models:
   model:
     name: hf_causal_lm
     pretrained_model_name_or_path: ${model_name_or_path}
-    init_device: cpu
+    init_device: mixed
     pretrained: true
     load_in_8bit: true
   tokenizer:
diff --git a/scripts/eval/yamls/hf_eval.yaml b/scripts/eval/yamls/hf_eval.yaml
index 8eecf57c30..05169818d9 100644
--- a/scripts/eval/yamls/hf_eval.yaml
+++ b/scripts/eval/yamls/hf_eval.yaml
@@ -12,7 +12,7 @@ models:
   model:
     name: hf_causal_lm
     pretrained_model_name_or_path: ${model_name_or_path}
-    init_device: cpu
+    init_device: mixed
     pretrained: true
   tokenizer:
     name: ${model_name_or_path}
diff --git a/scripts/eval/yamls/hf_lora_eval.yml b/scripts/eval/yamls/hf_lora_eval.yml
index 80300b7668..fc7074c18b 100644
--- a/scripts/eval/yamls/hf_lora_eval.yml
+++ b/scripts/eval/yamls/hf_lora_eval.yml
@@ -5,7 +5,9 @@ precision: amp_fp16
 # If you are using one model, put it here:
 model_name_or_path: EleutherAI/gpt-neo-125m
 # If you are using a seperated lora weight, put it here:
-lora_id_or_path: nathan0/lora-gpt-neo-125m-alpaca
+# lora weights must be compatible with the specified model
+lora_id_or_path: edbeeching/gpt-neo-125M-imdb-lora # Example lora weights for gpt-neo-125m
+
 # otherwise, write a block for each model you want to test in the `models` section
 
 models:
@@ -14,7 +16,7 @@ models:
   model:
     name: hf_causal_lm
     pretrained_model_name_or_path: ${model_name_or_path}
-    init_device: cpu
+    init_device: mixed
     pretrained: true
     pretrained_lora_id_or_path: ${lora_id_or_path}
   tokenizer:
diff --git a/scripts/eval/yamls/mpt_eval.yaml b/scripts/eval/yamls/mpt_eval.yaml
index 42b071899a..c8d359fc76 100644
--- a/scripts/eval/yamls/mpt_eval.yaml
+++ b/scripts/eval/yamls/mpt_eval.yaml
@@ -3,7 +3,6 @@ tokenizer_name: EleutherAI/gpt-neox-20b
 seed: 1
 precision: amp_fp16
 
-
 models:
 -
   model_name: mpt_test
@@ -14,7 +13,8 @@ models:
       model_max_length: ${max_seq_len}
   model:
     name: mpt_causal_lm
-    init_device: meta
+    init_device: mixed
+    # Set the below model parameters to match the checkpoint specified with load_path
     d_model: 768
     n_heads: 12
     n_layers: 12