From c454452c37d9fcebea34359354a927cf4b7ab11d Mon Sep 17 00:00:00 2001 From: Christopher Creutzig <89011131+ccreutzi@users.noreply.github.com> Date: Fri, 19 Jul 2024 08:47:03 +0200 Subject: [PATCH 1/2] Add gpt-4o-mini and make it the default --- +llms/+openai/models.m | 1 + doc/OpenAI.md | 1 + openAIChat.m | 4 ++-- tests/topenAIChat.m | 28 +++++++++++++++------------- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/+llms/+openai/models.m b/+llms/+openai/models.m index 96f9c17..df170da 100644 --- a/+llms/+openai/models.m +++ b/+llms/+openai/models.m @@ -4,6 +4,7 @@ % Copyright 2024 The MathWorks, Inc. models = [... "gpt-4o","gpt-4o-2024-05-13",... + "gpt-4o-mini","gpt-4o-mini-2024-07-18",... "gpt-4-turbo","gpt-4-turbo-2024-04-09",... "gpt-4","gpt-4-0613", ... "gpt-3.5-turbo","gpt-3.5-turbo-0125", ... diff --git a/doc/OpenAI.md b/doc/OpenAI.md index 76bd834..51783ae 100644 --- a/doc/OpenAI.md +++ b/doc/OpenAI.md @@ -5,6 +5,7 @@ Several functions in this repository connect MATLAB to the [OpenAI™ Chat Compl To start using the OpenAI APIs, you first need to obtain OpenAI API keys. You are responsible for any fees OpenAI may charge for the use of their APIs. You should be familiar with the limitations and risks associated with using this technology, and you agree that you shall be solely responsible for full compliance with any terms that may apply to your use of the OpenAI APIs. Some of the current LLMs supported on OpenAI are: +- gpt-4o-mini, gpt-4o-mini-2024-07-18 - gpt-3.5-turbo, gpt-3.5-turbo-1106, gpt-3.5-turbo-0125 - gpt-4o, gpt-4o-2024-05-13 (GPT-4 Omni) - gpt-4-turbo, gpt-4-turbo-2024-04-09 (GPT-4 Turbo with Vision) diff --git a/openAIChat.m b/openAIChat.m index 6a46cce..7d73c75 100644 --- a/openAIChat.m +++ b/openAIChat.m @@ -11,7 +11,7 @@ % using one or more name-value arguments: % % ModelName - Name of the model to use for chat completions. -% The default value is "gpt-3.5-turbo". +% The default value is "gpt-4o-mini". % % Temperature - Temperature value for controlling the randomness % of the output. Default value is 1; higher values @@ -92,7 +92,7 @@ arguments systemPrompt {llms.utils.mustBeTextOrEmpty} = [] nvp.Tools (1,:) {mustBeA(nvp.Tools, "openAIFunction")} = openAIFunction.empty - nvp.ModelName (1,1) string {mustBeModel} = "gpt-3.5-turbo" + nvp.ModelName (1,1) string {mustBeModel} = "gpt-4o-mini" nvp.Temperature {llms.utils.mustBeValidTemperature} = 1 nvp.TopP {llms.utils.mustBeValidTopP} = 1 nvp.StopSequences {llms.utils.mustBeValidStop} = {} diff --git a/tests/topenAIChat.m b/tests/topenAIChat.m index 7112b50..2b2b22c 100644 --- a/tests/topenAIChat.m +++ b/tests/topenAIChat.m @@ -34,7 +34,7 @@ function generateAcceptsMessagesAsInput(testCase) function constructChatWithAllNVP(testCase) functions = openAIFunction("funName"); - modelName = "gpt-3.5-turbo"; + modelName = "gpt-4o-mini"; temperature = 0; topP = 1; stop = ["[END]", "."]; @@ -84,7 +84,9 @@ function settingToolChoiceWithNone(testCase) end function fixedSeedFixesResult(testCase) - chat = openAIChat; + % Seed is "beta" in OpenAI documentation + % and not reliable in gpt-4o-mini at this time. + chat = openAIChat(ModelName="gpt-3.5-turbo"); result1 = generate(chat,"This is okay", "Seed", 2); result2 = generate(chat,"This is okay", "Seed", 2); @@ -227,7 +229,7 @@ function warningJSONResponseFormatGPT35(testCase) chat = @() openAIChat("You are a useful assistant", ... APIKey="this-is-not-a-real-key", ... ResponseFormat="json", ... - ModelName="gpt-3.5-turbo"); + ModelName="gpt-4o-mini"); testCase.verifyWarning(@()chat(), "llms:warningJsonInstruction"); end @@ -378,7 +380,7 @@ function keyNotFound(testCase) "FrequencyPenalty", {0}, ... "TimeOut", {10}, ... "FunctionNames", {[]}, ... - "ModelName", {"gpt-3.5-turbo"}, ... + "ModelName", {"gpt-4o-mini"}, ... "SystemPrompt", {[]}, ... "ResponseFormat", {"text"} ... ) ... @@ -394,7 +396,7 @@ function keyNotFound(testCase) "FrequencyPenalty", {0}, ... "TimeOut", {10}, ... "FunctionNames", {[]}, ... - "ModelName", {"gpt-3.5-turbo"}, ... + "ModelName", {"gpt-4o-mini"}, ... "SystemPrompt", {{struct("role","system","content","system prompt")}}, ... "ResponseFormat", {"text"} ... ) ... @@ -410,7 +412,7 @@ function keyNotFound(testCase) "FrequencyPenalty", {0}, ... "TimeOut", {10}, ... "FunctionNames", {[]}, ... - "ModelName", {"gpt-3.5-turbo"}, ... + "ModelName", {"gpt-4o-mini"}, ... "SystemPrompt", {[]}, ... "ResponseFormat", {"text"} ... ) ... @@ -426,7 +428,7 @@ function keyNotFound(testCase) "FrequencyPenalty", {0}, ... "TimeOut", {10}, ... "FunctionNames", {[]}, ... - "ModelName", {"gpt-3.5-turbo"}, ... + "ModelName", {"gpt-4o-mini"}, ... "SystemPrompt", {[]}, ... "ResponseFormat", {"text"} ... ) ... @@ -442,7 +444,7 @@ function keyNotFound(testCase) "FrequencyPenalty", {0}, ... "TimeOut", {10}, ... "FunctionNames", {[]}, ... - "ModelName", {"gpt-3.5-turbo"}, ... + "ModelName", {"gpt-4o-mini"}, ... "SystemPrompt", {[]}, ... "ResponseFormat", {"text"} ... ) ... @@ -458,7 +460,7 @@ function keyNotFound(testCase) "FrequencyPenalty", {0}, ... "TimeOut", {10}, ... "FunctionNames", {[]}, ... - "ModelName", {"gpt-3.5-turbo"}, ... + "ModelName", {"gpt-4o-mini"}, ... "SystemPrompt", {[]}, ... "ResponseFormat", {"text"} ... ) ... @@ -474,7 +476,7 @@ function keyNotFound(testCase) "FrequencyPenalty", {0.1}, ... "TimeOut", {10}, ... "FunctionNames", {[]}, ... - "ModelName", {"gpt-3.5-turbo"}, ... + "ModelName", {"gpt-4o-mini"}, ... "SystemPrompt", {[]}, ... "ResponseFormat", {"text"} ... ) ... @@ -490,7 +492,7 @@ function keyNotFound(testCase) "FrequencyPenalty", {0}, ... "TimeOut", {0.1}, ... "FunctionNames", {[]}, ... - "ModelName", {"gpt-3.5-turbo"}, ... + "ModelName", {"gpt-4o-mini"}, ... "SystemPrompt", {[]}, ... "ResponseFormat", {"text"} ... ) ... @@ -506,7 +508,7 @@ function keyNotFound(testCase) "FrequencyPenalty", {0}, ... "TimeOut", {10}, ... "FunctionNames", {[]}, ... - "ModelName", {"gpt-3.5-turbo"}, ... + "ModelName", {"gpt-4o-mini"}, ... "SystemPrompt", {[]}, ... "ResponseFormat", {"json"} ... ) ... @@ -566,7 +568,7 @@ function keyNotFound(testCase) "Error","MATLAB:validators:mustBeMember"),... ... "InvalidModelNameSize",struct( ... - "Input",{{ "ModelName", ["gpt-3.5-turbo", "gpt-3.5-turbo"] }},... + "Input",{{ "ModelName", ["gpt-4o-mini", "gpt-4o-mini"] }},... "Error","MATLAB:validation:IncompatibleSize"),... ... "InvalidModelNameOption",struct( ... From fc1798cd0f95a59d6bae6c15b032ca08ef871736 Mon Sep 17 00:00:00 2001 From: Christopher Creutzig <89011131+ccreutzi@users.noreply.github.com> Date: Fri, 19 Jul 2024 09:59:52 +0200 Subject: [PATCH 2/2] Disable yet another flaky Ollama test point It is unclear at this time why this test point is unreliable, but it just started failing in the GitHub CI, possibly following some Ollama update. We are not explicitly promising this behaviour, and the change was not on our side. --- tests/tollamaChat.m | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/tollamaChat.m b/tests/tollamaChat.m index 6927797..9ae230b 100644 --- a/tests/tollamaChat.m +++ b/tests/tollamaChat.m @@ -47,6 +47,11 @@ function doGenerateUsingSystemPrompt(testCase) end function extremeTopK(testCase) + %% This should work, and it does on some computers. On others, Ollama + %% receives the parameter, but either Ollama or llama.cpp fails to + %% honor it correctly. + testCase.assumeTrue(false,"disabled due to Ollama/llama.cpp not honoring parameter reliably"); + % setting top-k to k=1 leaves no random choice, % so we expect to get a fixed response. chat = ollamaChat("mistral",TopK=1);