From 4dd9527bfdccc22ae1c85b379ea3276b4475d1c4 Mon Sep 17 00:00:00 2001 From: Demis Bellot Date: Fri, 10 May 2024 13:57:06 +0800 Subject: [PATCH] Add MissingGradedAnswersTop1K and CreateRankingTasks APIs --- MyApp.ServiceInterface/AiServerServices.cs | 49 ++++++++++++++++++---- MyApp.ServiceInterface/StatServices.cs | 32 +++++++++++++- MyApp.ServiceModel/OpenAi.cs | 21 +++++++++- MyApp.ServiceModel/Stats.cs | 12 ++++++ MyApp.Tests/TestUtils.cs | 27 ++++++++++++ MyApp.Tests/Top1KQuestionTasks.cs | 45 +++++++++++++------- 6 files changed, 160 insertions(+), 26 deletions(-) diff --git a/MyApp.ServiceInterface/AiServerServices.cs b/MyApp.ServiceInterface/AiServerServices.cs index 507feb3..2a3a075 100644 --- a/MyApp.ServiceInterface/AiServerServices.cs +++ b/MyApp.ServiceInterface/AiServerServices.cs @@ -20,31 +20,62 @@ public class AiServerServices(ILogger log, public async Task Any(CreateAnswersForModel request) { var command = executor.Command(); + var to = new CreateAnswersForModelResponse(); - int completed = 0; - var sb = StringBuilderCache.Local(); foreach (var postId in request.PostIds) { var post = await questions.GetQuestionFileAsPostAsync(postId); if (post == null) { - sb.Append("Missing: ").Append(postId).Append('\n'); + to.Errors[postId] = "Missing QuestionFile"; continue; } - - completed++; await command.ExecuteAsync(new CreateAnswerTasks { Post = post, ModelUsers = [request.Model], }); + to.Results.Add(post.Id); } + return to; + } + + public async Task Any(CreateRankingTasks request) + { + var command = executor.Command(); + + var to = new CreateRankingTasksResponse(); + + var uniqueUserNames = request.AnswerIds.Select(x => x.RightPart('-')).ToSet(); + var userIdMap = await Db.DictionaryAsync( + Db.From().Where(x => uniqueUserNames.Contains(x.UserName!)) + .Select(x => new { x.UserName, x.Id })); - sb.Append("Completed: ").Append(completed).Append('\n'); - - return new StringResponse { Result = sb.ToString() }; + foreach (var id in request.AnswerIds) + { + try + { + var postId = id.LeftPart('-').ToInt(); + var userName = id.RightPart('-'); + if (!userIdMap.TryGetValue(userName, out var userId)) + { + to.Errors[id] = "Unknown User"; + continue; + } + await command.ExecuteAsync(new CreateRankAnswerTask + { + UserId = userId, + AnswerId = id, + }); + } + catch (Exception e) + { + to.Errors[id] = e.Message; + } + } + return to; } - + public async Task Any(CreateAnswerCallback request) { var modelUser = appConfig.GetModelUserById(request.UserId); diff --git a/MyApp.ServiceInterface/StatServices.cs b/MyApp.ServiceInterface/StatServices.cs index ed7fdf5..e597b4b 100644 --- a/MyApp.ServiceInterface/StatServices.cs +++ b/MyApp.ServiceInterface/StatServices.cs @@ -16,7 +16,7 @@ public class QuestionGroup public PostGroup Group { get; set; } } -public class StatServices(AppConfig appConfig) : Service +public class StatServices(AppConfig appConfig, QuestionsProvider questions) : Service { public async Task Any(MissingTop1K request) { @@ -36,4 +36,34 @@ SELECT PostId from StatTotals S WHERE Id LIKE '%-{model.UserName}' return new MissingTop1KResponse { Results = missingIds }; } + + public async Task Any(MissingGradedAnswersTop1K request) + { + var top1kIds = await Db.ColumnAsync(Db.From() + .Where(x => x.Group == PostGroup.Top1K)); + + var to = new MissingGradedAnswersTop1KResponse(); + foreach (var postId in top1kIds) + { + var id = $"{postId}"; + try + { + var meta = await questions.GetMetaAsync(postId); + foreach (var entry in meta.ModelVotes.Safe()) + { + if (meta.GradedBy == null || !meta.GradedBy.ContainsKey(entry.Key)) + { + var answerId = $"{postId}-{entry.Key}"; + to.Results.Add(answerId); + } + } + } + catch (Exception e) + { + to.Errors[id] = e.Message; + } + } + + return to; + } } \ No newline at end of file diff --git a/MyApp.ServiceModel/OpenAi.cs b/MyApp.ServiceModel/OpenAi.cs index efa0ba4..645b38f 100644 --- a/MyApp.ServiceModel/OpenAi.cs +++ b/MyApp.ServiceModel/OpenAi.cs @@ -90,7 +90,7 @@ public class CreateOpenAiChatResponse } [ValidateHasRole(Roles.Moderator)] -public class CreateAnswersForModel : IPost, IReturn +public class CreateAnswersForModel : IPost, IReturn { [ValidateNotEmpty] public string Model { get; set; } @@ -98,3 +98,22 @@ public class CreateAnswersForModel : IPost, IReturn [Input(Type = "tag"), FieldCss(Field = "col-span-12")] public List PostIds { get; set; } } +public class CreateAnswersForModelResponse +{ + public Dictionary Errors { get; set; } = new(); + public List Results { get; set; } = []; + public ResponseStatus? ResponseStatus { get; set; } +} + +[ValidateHasRole(Roles.Moderator)] +public class CreateRankingTasks : IPost, IReturn +{ + [Input(Type = "tag"), FieldCss(Field = "col-span-12")] + public List AnswerIds { get; set; } +} +public class CreateRankingTasksResponse +{ + public Dictionary Errors { get; set; } = new(); + public List Results { get; set; } = []; + public ResponseStatus? ResponseStatus { get; set; } +} diff --git a/MyApp.ServiceModel/Stats.cs b/MyApp.ServiceModel/Stats.cs index c7d207d..17b2be1 100644 --- a/MyApp.ServiceModel/Stats.cs +++ b/MyApp.ServiceModel/Stats.cs @@ -133,4 +133,16 @@ public class MissingTop1K : IGet, IReturn public class MissingTop1KResponse { public List Results { get; set; } + public ResponseStatus? ResponseStatus { get; set; } +} + +[ValidateHasRole(Roles.Moderator)] +public class MissingGradedAnswersTop1K : IGet, IReturn +{ +} +public class MissingGradedAnswersTop1KResponse +{ + public List Results { get; set; } = []; + public Dictionary Errors { get; set; } = new(); + public ResponseStatus? ResponseStatus { get; set; } } diff --git a/MyApp.Tests/TestUtils.cs b/MyApp.Tests/TestUtils.cs index 94254a1..13bcac0 100644 --- a/MyApp.Tests/TestUtils.cs +++ b/MyApp.Tests/TestUtils.cs @@ -12,4 +12,31 @@ public static string GetHostDir() // var appSettings = JSON.parse(File.ReadAllText(Path.GetFullPath("appsettings.json"))); // return appSettings.ToObjectDictionary()["HostDir"].ToString()!; } + + public static JsonApiClient CreateDevClient() => new("https://localhost:5001"); + public static async Task CreateAuthenticatedDevClientAsync() + { + var client = CreateDevClient(); + await client.ApiAsync(new Authenticate + { + provider = "credentials", + UserName = "mythz", + Password = Environment.GetEnvironmentVariable("AUTH_SECRET") + }); + return client; + } + + public static JsonApiClient CreateProdClient() => new("https://pvq.app"); + public static async Task CreateAuthenticatedProdClientAsync() + { + var client = CreateProdClient(); + await client.ApiAsync(new Authenticate + { + provider = "credentials", + UserName = "mythz", + Password = Environment.GetEnvironmentVariable("AUTH_SECRET") + }); + return client; + } + } \ No newline at end of file diff --git a/MyApp.Tests/Top1KQuestionTasks.cs b/MyApp.Tests/Top1KQuestionTasks.cs index 0545d19..9314bf0 100644 --- a/MyApp.Tests/Top1KQuestionTasks.cs +++ b/MyApp.Tests/Top1KQuestionTasks.cs @@ -20,15 +20,13 @@ public class Top1KQuestionTasks "deepseek-coder", "gemma-2b", ]; - - JsonApiClient CreateLiveClient() => new("https://pvq.app"); - + [Test] public async Task Find_Missing_Top1K_Questions_For_Model() { var model = "mixtral"; - var client = CreateLiveClient(); + var client = TestUtils.CreateProdClient(); var api = await client.ApiAsync(new MissingTop1K { @@ -43,7 +41,7 @@ public async Task Create_missing_Top1K_Answers_for_Models() { // var model = "mixtral"; - var client = CreateLiveClient(); + var client = TestUtils.CreateProdClient(); await client.ApiAsync(new Authenticate { provider = "credentials", @@ -62,14 +60,7 @@ public async Task Create_missing_Top1K_Answers_for_Adhoc_Model() { var model = "command-r-plus"; - var client = CreateLiveClient(); - await client.ApiAsync(new Authenticate - { - provider = "credentials", - UserName = "mythz", - Password = Environment.GetEnvironmentVariable("AUTH_SECRET") - }); - + var client = await TestUtils.CreateAuthenticatedProdClientAsync(); await CreateMissing1KModelsForModelAsync(client, model); } @@ -98,12 +89,36 @@ private static async Task CreateMissing1KModelsForModelAsync(JsonApiClient clien apiCreate.Error.PrintDump(); apiCreate.ThrowIfError(); - apiCreate.Response!.Result.Print(); + apiCreate.Response!.Errors.PrintDump(); + apiCreate.Response!.Results.PrintDump(); } [Test] - public void Find_answers_that_have_not_been_individually_graded() + public async Task Find_answers_that_have_not_been_individually_graded() { + var client = await TestUtils.CreateAuthenticatedProdClientAsync(); + // var client = await TestUtils.CreateAuthenticatedDevClientAsync(); + var api = await client.ApiAsync(new MissingGradedAnswersTop1K()); + + api.Error.PrintDump(); + api.ThrowIfError(); + api.Response.PrintDump(); + + if (api.Response!.Results.Count == 0) + { + $"No more ungraded answers".Print(); + return; + } + + var apiCreate = await client.ApiAsync(new CreateRankingTasks + { + AnswerIds = api.Response!.Results, + }); + + apiCreate.Error.PrintDump(); + apiCreate.ThrowIfError(); + apiCreate.Response!.Errors.PrintDump(); + apiCreate.Response!.Results.PrintDump(); } } \ No newline at end of file