Merge pull request #184 from SkanderHellal/master

Fix issue #183: OpenChat API Non-Deterministic Behavior
imoneoi · Mar 4, 2024 · 7fbad42 · 7fbad42
2 parents 30da91b + 88686d2
commit 7fbad42
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/ochat/serving/openai_api_protocol.py b/ochat/serving/openai_api_protocol.py
@@ -60,6 +60,7 @@ class ChatCompletionRequest(BaseModel):
     top_p: Optional[float] = 1.0
     n: Optional[int] = 1
     max_tokens: Optional[int] = None
+    seed: Optional[int] = None
     stop: Optional[Union[str, List[str]]] = None
     stream: Optional[bool] = False
     presence_penalty: Optional[float] = 0.0

diff --git a/ochat/serving/openai_api_server.py b/ochat/serving/openai_api_server.py
@@ -177,6 +177,7 @@ async def create_chat_completion(raw_request: Request, background_tasks: Backgro
             temperature=request.temperature,
             top_p=request.top_p,
             max_tokens=request.max_tokens,
+            seed=request.seed,
             # Override stop tokens
             stop_token_ids=model.eot_tokens,
             ignore_eos=True