Merge pull request #767 from 0xff-dev/main

add a new column latency in rag eval dataset row

apiserver/pkg/rag/report.go

@@ -48,7 +48,7 @@ const (
 	summarySuggestionTemplate = `通过此次评估，您的智能体得分偏低，主要体现在 <strong>%s</strong> 这 %d 项指标得分偏低。
 <br>
 <strong>建议您对特定场景应用的模型进行模型精调；%s。</strong>`
-	noSuggestionTempalte = `通过此次评估，您的 RAG 方案得分 <span style="color:orange">%.2f</span>`
+	noSuggestionTempalte = `通过此次评估，您的 RAG 方案得分 <span style="color:green">%.2f</span>`
 )
 
 var (
@@ -202,7 +202,7 @@ func ParseSummary(
 		report.TotalScore.Color = orange
 		report.Summary = fmt.Sprintf(summarySuggestionTemplate, strings.Join(metrics, "、"), len(metrics), strings.Join(metricSuggesstion, "、"))
 	} else {
-		report.Summary = fmt.Sprintf(noSuggestionTempalte, report.TotalScore.Score)
+		report.Summary = fmt.Sprintf(noSuggestionTempalte, report.TotalScore.Score*100.0)
 	}
 	return report, nil
 }
@@ -250,13 +250,16 @@ func ParseResult(
 			Contexts:     []string{line[4]},
 			Data:         make(map[string]float64),
 		}
+		item.CostTime, _ = strconv.ParseFloat(line[5], 64)
+
 		sum := float64(0)
-		for i := 5; i < len(line); i++ {
+		// TODO: Avoid direct hardcode. Mapping index via map
+		for i := 6; i < len(line); i++ {
 			f, _ := strconv.ParseFloat(line[i], 64)
 			item.Data[header[i]] = f
 			sum += f
 		}
-		item.TotalScore = sum / float64(len(line)-5)
+		item.TotalScore = sum / float64(len(line)-6)
 		result[i] = item
 	}
 

pkg/evaluation/evaluation.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"time"
 
 	"github.com/tmc/langchaingo/memory"
 	"k8s.io/klog/v2"
@@ -41,6 +42,8 @@ type RagasDataRow struct {
 	Contexts []string `json:"contexts"`
 	// Answer by Application
 	Answer string `json:"answer"`
+	// Latency Q&A generation duration
+	Latency string `json:"latency"`
 }
 
 // RagasDatasetGenerator generates datasets which adapts to the ragas framework
@@ -66,6 +69,7 @@ func NewRagasDatasetGenerator(ctx context.Context, cli client.Client, app *v1alp
 			GroundTruths: []string{"ground_truths"},
 			Contexts:     []string{"contexts"},
 			Answer:       "answer",
+			Latency:      "latency",
 		})
 		if err != nil {
 			return nil, err
@@ -158,12 +162,14 @@ func (eval *RagasDatasetGenerator) Generate(ctx context.Context, csvData io.Read
 			GroundTruths: []string{groundTruths},
 		}
 
+		start := time.Now()
 		// chat with application
 		out, err := eval.app.Run(ctx, eval.cli, nil, appruntime.Input{Question: ragasRow.Question, NeedStream: false, History: memory.NewChatMessageHistory()})
 		if err != nil {
 			klog.V(1).ErrorS(err, "failed to get the answer", "app", eval.app.Name, "namespace", eval.app.Namespace, "question", ragasRow.Question)
 			return err
 		}
+		ragasRow.Latency = fmt.Sprintf("%.1f", time.Since(start).Seconds())
 		ragasRow.Answer = out.Answer
 
 		// handle context

pkg/evaluation/output.go

@@ -31,7 +31,7 @@ type PrintOutput struct{}
 
 // Output this row to standard output
 func (print *PrintOutput) Output(row RagasDataRow) error {
-	fmt.Printf("question:%s \nground_truths:%s \n answer:%s \n contexts:%v \n", row.Question, row.GroundTruths, row.Answer, row.Contexts)
+	fmt.Printf("question:%s \n ground_truths:%s \n answer:%s \n contexts:%v \n latency:%s \n", row.Question, row.GroundTruths, row.Answer, row.Contexts, row.Latency)
 	return nil
 }
 
@@ -42,5 +42,5 @@ type CSVOutput struct {
 
 // Output a row to csv
 func (csv *CSVOutput) Output(row RagasDataRow) error {
-	return csv.W.Write([]string{row.Question, strings.Join(row.GroundTruths, ";"), row.Answer, strings.Join(row.Contexts, ";")})
+	return csv.W.Write([]string{row.Question, strings.Join(row.GroundTruths, ";"), row.Answer, strings.Join(row.Contexts, ";"), row.Latency})
 }