From bc8f648a91a2f07b061b3ae1ee1944131c61cda2 Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <mudler@users.noreply.github.com>
Date: Mon, 11 Mar 2024 19:49:03 +0100
Subject: [PATCH] fix(doc/examples): set defaults to mirostat (#1820)

The default sampler on some models don't return enough candidates which
leads to a false sense of randomness. Tracing back the code it looks
that with the temperature sampler there might not be enough
candidates to pick from, and since the seed and "randomness" take effect
while picking a good candidate this yields to the same results over and
over.

Fixes https://github.com/mudler/LocalAI/issues/1723 by updating the
examples and documentation to use mirostat instead.
---
 docs/content/docs/advanced/advanced-usage.md  | 7 ++++---
 embedded/models/codellama-7b-gguf.yaml        | 6 +++++-
 embedded/models/dolphin-2.5-mixtral-8x7b.yaml | 5 ++++-
 embedded/models/llava.yaml                    | 3 +++
 embedded/models/mistral-openorca.yaml         | 4 ++++
 embedded/models/mixtral-instruct.yaml         | 4 ++++
 examples/configurations/phi-2.yaml            | 4 ++++
 7 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/docs/content/docs/advanced/advanced-usage.md b/docs/content/docs/advanced/advanced-usage.md
index a892cc3690a..c9926bab276 100644
--- a/docs/content/docs/advanced/advanced-usage.md
+++ b/docs/content/docs/advanced/advanced-usage.md
@@ -130,13 +130,14 @@ parameters:
   typical_p:
   tfz:
   frequency_penalty:
-  mirostat_eta:
-  mirostat_tau:
-  mirostat: 
+
   rope_freq_base:
   rope_freq_scale:
   negative_prompt_scale:
 
+mirostat_eta:
+mirostat_tau:
+mirostat: 
 # Default context size
 context_size: 512
 # Default number of threads
diff --git a/embedded/models/codellama-7b-gguf.yaml b/embedded/models/codellama-7b-gguf.yaml
index d2a6b5184c8..413c838b7b0 100644
--- a/embedded/models/codellama-7b-gguf.yaml
+++ b/embedded/models/codellama-7b-gguf.yaml
@@ -2,10 +2,14 @@ name: codellama-7b-gguf
 backend: transformers
 parameters:
   model: huggingface://TheBloke/CodeLlama-7B-GGUF/codellama-7b.Q4_K_M.gguf
-  temperature: 0.2
+  temperature: 0.5
   top_k: 40
   seed: -1
   top_p: 0.95
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
+
 context_size: 4096
 f16: true
 gpu_layers: 90
diff --git a/embedded/models/dolphin-2.5-mixtral-8x7b.yaml b/embedded/models/dolphin-2.5-mixtral-8x7b.yaml
index ba6020c0856..12ee1efc260 100644
--- a/embedded/models/dolphin-2.5-mixtral-8x7b.yaml
+++ b/embedded/models/dolphin-2.5-mixtral-8x7b.yaml
@@ -2,10 +2,13 @@ name: dolphin-mixtral-8x7b
 mmap: true
 parameters:
   model: huggingface://TheBloke/dolphin-2.5-mixtral-8x7b-GGUF/dolphin-2.5-mixtral-8x7b.Q2_K.gguf
-  temperature: 0.2
+  temperature: 0.5
   top_k: 40
   top_p: 0.95
   seed: -1
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
 template:
   chat_message: |
     <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}}
diff --git a/embedded/models/llava.yaml b/embedded/models/llava.yaml
index 2e571f212e2..3d240681f17 100644
--- a/embedded/models/llava.yaml
+++ b/embedded/models/llava.yaml
@@ -18,6 +18,9 @@ parameters:
   top_k: 40
   top_p: 0.95
   seed: -1
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
 
 template:
   chat: |
diff --git a/embedded/models/mistral-openorca.yaml b/embedded/models/mistral-openorca.yaml
index f40d854f72f..0794a69b0a2 100644
--- a/embedded/models/mistral-openorca.yaml
+++ b/embedded/models/mistral-openorca.yaml
@@ -6,6 +6,10 @@ parameters:
   top_k: 40
   top_p: 0.95
   seed: -1
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
+
 template:
   chat_message: |
     <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}}
diff --git a/embedded/models/mixtral-instruct.yaml b/embedded/models/mixtral-instruct.yaml
index 3272557a717..246b23240f4 100644
--- a/embedded/models/mixtral-instruct.yaml
+++ b/embedded/models/mixtral-instruct.yaml
@@ -6,6 +6,10 @@ parameters:
   top_k: 40
   seed: -1
   top_p: 0.95
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
+
 template:
   chat: &chat |
     [INST] {{.Input}} [/INST]    
diff --git a/examples/configurations/phi-2.yaml b/examples/configurations/phi-2.yaml
index cac1e9da9f5..e5a13442d18 100644
--- a/examples/configurations/phi-2.yaml
+++ b/examples/configurations/phi-2.yaml
@@ -11,6 +11,10 @@ parameters:
   top_k: 40
   top_p: 0.95
   seed: -1
+  
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
 template:
   chat: &template |-
     Instruct: {{.Input}}