Merge pull request #73 from RWKV/rwkv-x-eagle-notebooks

Rwkv x eagle notebooks
RWKV · Feb 2, 2024 · 00274ed · 00274ed
2 parents a7b090d + 786889d
commit 00274ed
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 106 deletions.
diff --git a/notebook/finetune-example/Eagle-x-ALMA-prompt-completion.yaml b/notebook/finetune-example/Eagle-x-ALMA-prompt-completion.yaml
@@ -74,8 +74,8 @@ model:
   load_model: ../model/L6-D512-neox-init.pth
 
   # Starting and ending learning rate
-  lr_init: 5e-5
-  lr_final: 5e-5
+  lr_init: 1e-5
+  lr_final: 1e-5
 
   # Training context length, note that the dataset can be
   # larger then the context size, in which the trainer

diff --git a/notebook/finetune-example/Eagle-x-capybara-chat.yaml b/notebook/finetune-example/Eagle-x-capybara-chat.yaml
@@ -79,8 +79,8 @@ model:
   load_model: ../model/L6-D512-neox-init.pth
 
   # Starting and ending learning rate
-  lr_init: 5e-5
-  lr_final: 5e-5
+  lr_init: 1e-5
+  lr_final: 1e-5
 
   # Training context length, note that the dataset can be
   # larger then the context size, in which the trainer

diff --git a/notebook/finetune-example/Eagle-x-openhermes1-instruct.yaml b/notebook/finetune-example/Eagle-x-openhermes1-instruct.yaml
@@ -74,8 +74,8 @@ model:
   load_model: ../model/L6-D512-neox-init.pth
 
   # Starting and ending learning rate
-  lr_init: 5e-5
-  lr_final: 5e-5
+  lr_init: 1e-5
+  lr_final: 1e-5
 
   # Training context length, note that the dataset can be
   # larger then the context size, in which the trainer

diff --git a/notebook/finetune-example/Eagle-x-textbooks.yaml b/notebook/finetune-example/Eagle-x-textbooks.yaml
@@ -79,8 +79,8 @@ model:
   load_model: ../model/L6-D512-neox-init.pth
 
   # Starting and ending learning rate
-  lr_init: 5e-5
-  lr_final: 5e-5
+  lr_init: 1e-5
+  lr_final: 1e-5
 
   # Training context length, note that the dataset can be
   # larger then the context size, in which the trainer