Helicone · ScottMktn · May 16, 2024 · May 13, 2024 · May 14, 2024 · May 14, 2024
diff --git a/docs/swagger.json b/docs/swagger.json
@@ -1806,47 +1806,50 @@
 				"type": "object",
 				"additionalProperties": false
 			},
+			"ScoreValue": {
+				"anyOf": [
+					{
+						"type": "string"
+					},
+					{
+						"type": "number",
+						"format": "double"
+					},
+					{
+						"type": "string",
+						"format": "date-time"
+					}
+				]
+			},
+			"Record_string.ScoreValue_": {
+				"properties": {},
+				"additionalProperties": {
+					"$ref": "#/components/schemas/ScoreValue"
+				},
+				"type": "object",
+				"description": "Construct a type with a set of properties K of type T"
+			},
 			"ExperimentScores": {
 				"properties": {
 					"dataset": {
 						"properties": {
-							"cost": {
-								"type": "number",
-								"format": "double"
-							},
-							"model": {
-								"type": "string"
-							},
-							"dateCreated": {
-								"type": "string",
-								"format": "date-time"
+							"scores": {
+								"$ref": "#/components/schemas/Record_string.ScoreValue_"
 							}
 						},
 						"required": [
-							"cost",
-							"model",
-							"dateCreated"
+							"scores"
 						],
 						"type": "object"
 					},
 					"hypothesis": {
 						"properties": {
-							"cost": {
-								"type": "number",
-								"format": "double"
-							},
-							"model": {
-								"type": "string"
-							},
-							"dateCreated": {
-								"type": "string",
-								"format": "date-time"
+							"scores": {
+								"$ref": "#/components/schemas/Record_string.ScoreValue_"
 							}
 						},
 						"required": [
-							"cost",
-							"model",
-							"dateCreated"
+							"scores"
 						],
 						"type": "object"
 					}

diff --git a/valhalla/jawn/src/lib/stores/ScoreStore.ts b/valhalla/jawn/src/lib/stores/ScoreStore.ts
@@ -17,7 +17,7 @@ export class ScoreStore extends BaseStore {
   ) {
     try {
       const scoreKeys = Object.keys(scores);
-      const scoreValues = Object.values(scores);
+      const scoreValues = Object.values(scores).map((val) => BigInt(val));
 
       const { data: requestData, error: requestError } = await dbExecute(
         `SELECT id FROM request WHERE id = $1 AND helicone_org_id = $2`,
@@ -32,27 +32,38 @@ export class ScoreStore extends BaseStore {
 
       const organizationIds = Array(scoreKeys.length).fill(this.organizationId);
 
-      const query = `
+      const upsertQuery = `
         WITH upserted_attributes AS (
             INSERT INTO score_attribute (score_key, organization)
             SELECT unnest($1::text[]), unnest($2::uuid[])
             ON CONFLICT (score_key, organization) DO UPDATE SET
                 score_key = EXCLUDED.score_key
-            RETURNING id
-        ),
-        inserted_values AS (
-            INSERT INTO score_value (score_attribute, request_id, int_value)
-            SELECT id, $3, unnest($4::bigint[])
-            FROM upserted_attributes
-            ON CONFLICT (score_attribute, request_id) DO NOTHING
-            RETURNING *
+            RETURNING id, score_key
         )
-        SELECT * FROM inserted_values;
-    `;
+        SELECT id, score_key
+        FROM upserted_attributes;
+      `;
+
+      const { data: upsertedAttributes, error: upsertError } = await dbExecute(
+        upsertQuery,
+        [scoreKeys, organizationIds]
+      );
+
+      if (!upsertedAttributes || upsertError) {
+        return err(`Error upserting attributes: ${upsertError}`);
+      }
+
+      const attributeIds = upsertedAttributes.map((attr: any) => attr.id);
+
+      const insertValuesQuery = `
+        INSERT INTO score_value (score_attribute, request_id, int_value)
+        SELECT unnest($1::uuid[]), $2, unnest($3::bigint[])
+        ON CONFLICT (score_attribute, request_id) DO NOTHING
+        RETURNING *;
+      `;
 
-      const { data, error } = await dbExecute(query, [
-        scoreKeys,
-        organizationIds,
+      const { data, error } = await dbExecute(insertValuesQuery, [
+        attributeIds,
         requestId,
         scoreValues,
       ]);

diff --git a/valhalla/jawn/src/lib/stores/experimentStore.ts b/valhalla/jawn/src/lib/stores/experimentStore.ts
@@ -66,18 +66,14 @@ export interface Experiment {
   scores: ExperimentScores | null;
 }
 
+type ScoreValue = string | number | Date;
+
 export interface ExperimentScores {
   dataset: {
-    dateCreated: Date;
-    model: string;
-    cost: number;
-    //customScores: Record<string, number>;
+    scores: Record<string, ScoreValue>;
   };
   hypothesis: {
-    dateCreated: Date;
-    model: string;
-    cost: number;
-    //customScores: Record<string, number>;
+    scores: Record<string, ScoreValue>;
   };
 }
 
@@ -456,9 +452,12 @@ function getExperimentHypothesisScores(
     );
 
     return ok({
-      dateCreated: new Date(hypothesis.createdAt),
-      model: hypothesis.model,
-      cost: hypothesisCost,
+      scores: {
+        dateCreated: new Date(hypothesis.createdAt),
+        model: hypothesis.model,
+        cost: hypothesisCost,
+        ...getCustomScores(hypothesis.runs.map((run) => run.scores)),
+      },
     });
   } catch (error) {
     console.error("Error calculating hypothesis cost", error);
@@ -508,16 +507,43 @@ function getExperimentDatasetScores(
     const averageCost = validRows.length > 0 ? totalCost / validRows.length : 0;
 
     return ok({
-      dateCreated: new Date(latest.createdAt),
-      model: latest.model,
-      cost: averageCost,
+      scores: {
+        dateCreated: new Date(latest.createdAt),
+        model: latest.model,
+        cost: averageCost,
+        ...getCustomScores(validRows.map((row) => row.scores)),
+      },
     });
   } catch (error) {
     console.error("Error calculating dataset cost", error);
     return err("Error calculating dataset cost");
   }
 }
 
+function getCustomScores(
+  scores: Record<string, number>[]
+): Record<string, number> {
+  const scoresValues = scores.reduce((acc, record) => {
+    for (const key in record) {
+      if (record.hasOwnProperty(key)) {
+        if (!acc[key]) {
+          acc[key] = { sum: 0, count: 0 };
+        }
+        acc[key].sum += record[key];
+        acc[key].count += 1;
+      }
+    }
+    return acc;
+  }, {} as Record<string, { sum: number; count: number }>);
+
+  return Object.fromEntries(
+    Object.entries(scoresValues).map(([key, { sum, count }]) => [
+      key,
+      sum / count,
+    ])
+  );
+}
+
 function modelCost(modelRow: {
   model: string;
   provider: string;

diff --git a/valhalla/jawn/src/tsoa-build/public/routes.ts b/valhalla/jawn/src/tsoa-build/public/routes.ts
@@ -677,11 +677,21 @@ const models: TsoaRoute.Models = {
         "additionalProperties": false,
     },
     // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa
+    "ScoreValue": {
+        "dataType": "refAlias",
+        "type": {"dataType":"union","subSchemas":[{"dataType":"string"},{"dataType":"double"},{"dataType":"datetime"}],"validators":{}},
+    },
+    // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa
+    "Record_string.ScoreValue_": {
+        "dataType": "refAlias",
+        "type": {"dataType":"nestedObjectLiteral","nestedProperties":{},"additionalProperties":{"ref":"ScoreValue"},"validators":{}},
+    },
+    // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa
     "ExperimentScores": {
         "dataType": "refObject",
         "properties": {
-            "dataset": {"dataType":"nestedObjectLiteral","nestedProperties":{"cost":{"dataType":"double","required":true},"model":{"dataType":"string","required":true},"dateCreated":{"dataType":"datetime","required":true}},"required":true},
-            "hypothesis": {"dataType":"nestedObjectLiteral","nestedProperties":{"cost":{"dataType":"double","required":true},"model":{"dataType":"string","required":true},"dateCreated":{"dataType":"datetime","required":true}},"required":true},
+            "dataset": {"dataType":"nestedObjectLiteral","nestedProperties":{"scores":{"ref":"Record_string.ScoreValue_","required":true}},"required":true},
+            "hypothesis": {"dataType":"nestedObjectLiteral","nestedProperties":{"scores":{"ref":"Record_string.ScoreValue_","required":true}},"required":true},
         },
         "additionalProperties": false,
     },

diff --git a/valhalla/jawn/src/tsoa-build/public/swagger.json b/valhalla/jawn/src/tsoa-build/public/swagger.json
@@ -1806,47 +1806,50 @@
 				"type": "object",
 				"additionalProperties": false
 			},
+			"ScoreValue": {
+				"anyOf": [
+					{
+						"type": "string"
+					},
+					{
+						"type": "number",
+						"format": "double"
+					},
+					{
+						"type": "string",
+						"format": "date-time"
+					}
+				]
+			},
+			"Record_string.ScoreValue_": {
+				"properties": {},
+				"additionalProperties": {
+					"$ref": "#/components/schemas/ScoreValue"
+				},
+				"type": "object",
+				"description": "Construct a type with a set of properties K of type T"
+			},
 			"ExperimentScores": {
 				"properties": {
 					"dataset": {
 						"properties": {
-							"cost": {
-								"type": "number",
-								"format": "double"
-							},
-							"model": {
-								"type": "string"
-							},
-							"dateCreated": {
-								"type": "string",
-								"format": "date-time"
+							"scores": {
+								"$ref": "#/components/schemas/Record_string.ScoreValue_"
 							}
 						},
 						"required": [
-							"cost",
-							"model",
-							"dateCreated"
+							"scores"
 						],
 						"type": "object"
 					},
 					"hypothesis": {
 						"properties": {
-							"cost": {
-								"type": "number",
-								"format": "double"
-							},
-							"model": {
-								"type": "string"
-							},
-							"dateCreated": {
-								"type": "string",
-								"format": "date-time"
+							"scores": {
+								"$ref": "#/components/schemas/Record_string.ScoreValue_"
 							}
 						},
 						"required": [
-							"cost",
-							"model",
-							"dateCreated"
+							"scores"
 						],
 						"type": "object"
 					}

diff --git a/web/components/templates/prompts/experiments/id/experimentIdPage.tsx b/web/components/templates/prompts/experiments/id/experimentIdPage.tsx
@@ -16,6 +16,7 @@ import ModelPill from "../../../requestsV2/modelPill";
 import HcBreadcrumb from "../../../../ui/hcBreadcrumb";
 import { usePrompt } from "../../../../../services/hooks/prompts/prompts";
 import ArrayDiffViewer from "../../id/arrayDiffViewer";
+import ScoresTable from "../scoresTable";
 
 interface PromptIdPageProps {
   id: string;
@@ -132,6 +133,7 @@ const ExperimentIdPage = (props: PromptIdPageProps) => {
             <h1 className="font-semibold text-3xl text-black dark:text-white">
               {experiment?.id}
             </h1>
+            {experiment?.scores && <ScoresTable scores={experiment?.scores} />}
             <div className="h-full w-full border border-gray-300 dark:border-gray-700 rounded-lg bg-white dark:bg-black">
               <div className="w-full flex justify-between items-center p-4 border-b border-gray-300 dark:border-gray-700 rounded-t-lg">
                 <div className="flex items-center space-x-2">