fix(doc/examples): set defaults to mirostat (#1820)

The default sampler on some models don't return enough candidates which
leads to a false sense of randomness. Tracing back the code it looks
that with the temperature sampler there might not be enough
candidates to pick from, and since the seed and "randomness" take effect
while picking a good candidate this yields to the same results over and
over.

Fixes #1723 by updating the
examples and documentation to use mirostat instead.

docs/content/docs/advanced/advanced-usage.md

@@ -130,13 +130,14 @@ parameters:
   typical_p:
   tfz:
   frequency_penalty:
-  mirostat_eta:
-  mirostat_tau:
-  mirostat: 
+
   rope_freq_base:
   rope_freq_scale:
   negative_prompt_scale:
 
+mirostat_eta:
+mirostat_tau:
+mirostat: 
 # Default context size
 context_size: 512
 # Default number of threads

embedded/models/codellama-7b-gguf.yaml

@@ -2,10 +2,14 @@ name: codellama-7b-gguf
 backend: transformers
 parameters:
   model: huggingface://TheBloke/CodeLlama-7B-GGUF/codellama-7b.Q4_K_M.gguf
-  temperature: 0.2
+  temperature: 0.5
   top_k: 40
   seed: -1
   top_p: 0.95
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
+
 context_size: 4096
 f16: true
 gpu_layers: 90

embedded/models/dolphin-2.5-mixtral-8x7b.yaml

@@ -2,10 +2,13 @@ name: dolphin-mixtral-8x7b
 mmap: true
 parameters:
   model: huggingface://TheBloke/dolphin-2.5-mixtral-8x7b-GGUF/dolphin-2.5-mixtral-8x7b.Q2_K.gguf
-  temperature: 0.2
+  temperature: 0.5
   top_k: 40
   top_p: 0.95
   seed: -1
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
 template:
   chat_message: |
     <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}}

embedded/models/llava.yaml

@@ -18,6 +18,9 @@ parameters:
   top_k: 40
   top_p: 0.95
   seed: -1
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
 
 template:
   chat: |

embedded/models/mistral-openorca.yaml

@@ -6,6 +6,10 @@ parameters:
   top_k: 40
   top_p: 0.95
   seed: -1
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
+
 template:
   chat_message: |
     <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}}

embedded/models/mixtral-instruct.yaml

@@ -6,6 +6,10 @@ parameters:
   top_k: 40
   seed: -1
   top_p: 0.95
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
+
 template:
   chat: &chat |
     [INST] {{.Input}} [/INST]    

examples/configurations/phi-2.yaml

@@ -11,6 +11,10 @@ parameters:
   top_k: 40
   top_p: 0.95
   seed: -1
+
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
 template:
   chat: &template |-
     Instruct: {{.Input}}