+
+
+
+
+
+
+
+
+ );
+}
diff --git a/src/helm-frontend/src/components/GroupsTables.tsx b/src/helm-frontend/src/components/GroupsTables.tsx
new file mode 100644
index 0000000000..181081999f
--- /dev/null
+++ b/src/helm-frontend/src/components/GroupsTables.tsx
@@ -0,0 +1,115 @@
+import { useEffect, useState } from "react";
+import { ChevronUpDownIcon } from "@heroicons/react/24/solid";
+import type GroupsTable from "@/types/GroupsTable";
+import RowValue from "@/components/RowValue";
+
+interface Props {
+ groupsTables: GroupsTable[];
+ activeGroup: number;
+ ignoreHref?: boolean;
+ sortable?: boolean;
+}
+
+export default function GroupsTables({
+ groupsTables,
+ activeGroup,
+ ignoreHref = false,
+ sortable = true,
+}: Props) {
+ const [activeSortColumn, setActiveSortColumn] = useState<
+ number | undefined
+ >();
+ const [activeGroupsTable, setActiveGroupsTable] = useState
+
+
+
+
+ );
+}
diff --git a/src/helm-frontend/src/components/HeadTitle.tsx b/src/helm-frontend/src/components/HeadTitle.tsx
new file mode 100644
index 0000000000..897cef8566
--- /dev/null
+++ b/src/helm-frontend/src/components/HeadTitle.tsx
@@ -0,0 +1,9 @@
+interface Props {
+ title: string;
+}
+
+export default function Title({ title }: Props) {
+ return (
+ |
+
+ {headerValue.value}
+ {sortable ? (
+
+ ) : null}
+
+ |
+ ))}
+
|---|
|
+ |
+ ))}
+
+
+ );
+}
diff --git a/src/helm-frontend/src/components/Link.tsx b/src/helm-frontend/src/components/Link.tsx
new file mode 100644
index 0000000000..0ac2b3998c
--- /dev/null
+++ b/src/helm-frontend/src/components/Link.tsx
@@ -0,0 +1,13 @@
+import { Link as ReactRouterLink } from "react-router-dom";
+
+interface Props {
+ to: string;
+}
+
+export default function Link({ to, children }: React.PropsWithChildren+ {`Instance id: ${instance.id} [split: ${instance.split}]`} +
+ Input +
+
+
+ );
+}
diff --git a/src/helm-frontend/src/components/MarkdownValue.test.tsx b/src/helm-frontend/src/components/MarkdownValue.test.tsx
new file mode 100644
index 0000000000..1c68a65008
--- /dev/null
+++ b/src/helm-frontend/src/components/MarkdownValue.test.tsx
@@ -0,0 +1,15 @@
+import { render, screen } from "@testing-library/react";
+import "@testing-library/jest-dom";
+import MarkdownValue from "./MarkdownValue";
+
+it("renders correctly", () => {
+ render({metrics.length} Metrics
+-
+ {metricGroups.map((metricGroup, idx) => (
+
-
+ {metrics.filter((metric) =>
+ metricGroup.metrics.some((m) => m.name === metric.name),
+ ).length > 0 ? (
+
+ {metricGroup.display_name} ({metricGroup.name}) +
+ ) : null} +-
+ {metrics
+ .filter((metric) =>
+ metricGroup.metrics.some((m) => m.name === metric.name),
+ )
+ .map((metric, idx) => {
+ return (
+
- + {metric.display_name} + + ); + })} +
+ ))}
+
{models.length} Models
+-
+ {models.map((model, idx) =>
+ model.todo ? (
+
- + {model.creator_organization} / {model.display_name} + + ) : ( +
- + {model.creator_organization} / {model.display_name} + + ), + )} +
{title}
+ {markdown && subtitle !== undefined ? ( +
+
+ ) : (
+ subtitle !== undefined && (
+ {subtitle}
+ ) + )} +
+
+
+
+
+ );
+}
diff --git a/src/helm-frontend/src/components/Predictions.tsx b/src/helm-frontend/src/components/Predictions.tsx
new file mode 100644
index 0000000000..4110738679
--- /dev/null
+++ b/src/helm-frontend/src/components/Predictions.tsx
@@ -0,0 +1,75 @@
+import { useState } from "react";
+import type DisplayPrediction from "@/types/DisplayPrediction";
+import type DisplayRequest from "@/types/DisplayRequest";
+import Indicator from "@/components/Indicator";
+import Request from "@/components/Request";
+import { List, ListItem } from "@tremor/react";
+import Tab from "@/components/Tab";
+import Tabs from "@/components/Tabs";
+import Preview from "@/components/Preview";
+
+type Props = {
+ predictions: DisplayPrediction[];
+ requests: DisplayRequest[];
+};
+
+/**
+ * @SEE https://github.com/stanford-crfm/helm/blob/cffe38eb2c814d054c778064859b6e1551e5e106/src/helm/benchmark/static/benchmarking.js#L583-L679
+ */
+export default function Predictions({ predictions, requests }: Props) {
+ const [openRequests, setOpenRequests] = useState(false);
+ const [openDetails, setOpenDetails] = useState(true);
+ const handleToggle = () => {
+ setOpenRequests(!openRequests);
+ setOpenDetails(!openDetails);
+ };
+
+ if (predictions.length < 1) {
+ return null;
+ }
+
+ return (
+
+
+ );
+}
diff --git a/src/helm-frontend/src/components/Preview.tsx b/src/helm-frontend/src/components/Preview.tsx
new file mode 100644
index 0000000000..1a1df541cf
--- /dev/null
+++ b/src/helm-frontend/src/components/Preview.tsx
@@ -0,0 +1,49 @@
+import { useState } from "react";
+import { ArrowsPointingOutIcon } from "@heroicons/react/24/solid";
+
+interface Props {
+ value: string;
+}
+
+export default function Preview({ value }: Props) {
+ const [showButton, setShowButton] = useState
+ {predictions.map((prediction, idx) => (
+
+
+
+ ))}
+
+
+ {`Prediction [trial ${prediction.train_trial_index}]`}
+
+
+
+ {`Prediction [trial ${prediction.train_trial_index}]`}
+
+
+
+
+ Details
+
+
+ Requests
+
+
+
+ {openDetails ? (
+ -
+ {(
+ Object.keys(
+ prediction.stats,
+ ) as (keyof typeof prediction.stats)[]
+ ).map((statKey, idx) => (
+
+ {openRequests ?
+ setShowButton(true)}
+ onMouseOut={() => setShowButton(false)}
+ className="relative"
+ >
+
+
+
+ );
+}
diff --git a/src/helm-frontend/src/components/Request.test.tsx b/src/helm-frontend/src/components/Request.test.tsx
new file mode 100644
index 0000000000..193d5fcfd7
--- /dev/null
+++ b/src/helm-frontend/src/components/Request.test.tsx
@@ -0,0 +1,32 @@
+import { render, screen } from "@testing-library/react";
+import "@testing-library/jest-dom";
+import DisplayRequest from "@/types/DisplayRequest";
+import Request from "./Request";
+
+it("renders correctly", () => {
+ const request: DisplayRequest = {
+ instance_id: "id1662",
+ train_trial_index: 0,
+ request: {
+ echo_prompt: false,
+ embedding: false,
+ frequency_penalty: 0,
+ max_tokens: 5,
+ model: "AlephAlpha/luminous-base",
+ num_completions: 1,
+ presence_penalty: 0,
+ prompt: "Passage: Mice are afraid of cats.",
+ stop_sequences: ["\n"],
+ temperature: 0,
+ top_k_per_token: 1,
+ top_p: 1,
+ },
+ };
+
+ render(
+
+
+ {showButton ? (
+
+ ) : null}
+ {value}
+
+
+ );
+}
diff --git a/src/helm-frontend/src/components/RowValue.tsx b/src/helm-frontend/src/components/RowValue.tsx
new file mode 100644
index 0000000000..f1253370e6
--- /dev/null
+++ b/src/helm-frontend/src/components/RowValue.tsx
@@ -0,0 +1,40 @@
+import MarkdownValue from "@/components/MarkdownValue";
+import type RowValueType from "@/types/RowValue";
+import Link from "@/components/Link";
+
+interface Props {
+ value: RowValueType;
+ ignoreHref?: boolean;
+}
+
+function formatNumber(value: string | number): string {
+ if (Number.isNaN(Number(value))) {
+ return String(value);
+ }
+
+ return String(Math.round(Number(value) * 1000) / 1000);
+}
+
+export default function RowValue({ value, ignoreHref = false }: Props) {
+ if (value.value === undefined) {
+ return "-";
+ }
+
+ if (value.href !== undefined && ignoreHref === false) {
+ const href = (() => {
+ const matches = value.href.match(/group=([^&]+)/);
+ if (matches === null) {
+ return value.href;
+ }
+
+ return `/groups/${matches[1]}`;
+ })();
+ return {formatNumber(value.value)};
+ }
+
+ if (value.markdown) {
+ return Prompt
+-
+ {(Object.keys(request.request) as (keyof typeof request.request)[])
+ .filter((key) => key !== "prompt")
+ .map((key, idx) => (
+
{runGroups.length} Scenarios
+-
+ {topLevelGroups
+ .filter((topLevelGroup) =>
+ subGroups.some((subGroup) =>
+ (topLevelGroup.subgroups || []).includes(subGroup.name),
+ ),
+ )
+ .map((topLevelGroup, idx) => (
+
-
+
{topLevelGroup.display_name}
+-
+ {subGroups
+ .filter((subGroup) =>
+ (topLevelGroup.subgroups || []).includes(subGroup.name),
+ )
+ .map((subGroup, idx) => (
+
- + {subGroup.display_name} + + ))} +
+ ))}
+
+ {children}
+
+ );
+}
diff --git a/src/helm-frontend/src/components/Tabs.test.tsx b/src/helm-frontend/src/components/Tabs.test.tsx
new file mode 100644
index 0000000000..3e42383b80
--- /dev/null
+++ b/src/helm-frontend/src/components/Tabs.test.tsx
@@ -0,0 +1,21 @@
+import { render, screen } from "@testing-library/react";
+import "@testing-library/jest-dom";
+import Tabs from "./Tabs";
+import Tab from "./Tab";
+
+test("display tabs", () => {
+ render(
+ Group currently has no results.
+ + ); + } + + return ( + <> +
+
+
+ {groupsTables.length > 1 ? (
+
+ {groupsTables.map((groupsTable, idx) => (
+ setActiveGroup(idx)}
+ >
+ {groupsTable.title}
+
+ ))}
+
+ ) : null}
+
+
+ {showChart ? (
+
+
+
+
+ {tabs.map((tab, idx) => (
+ setActiveGroup(idx)}
+ key={idx}
+ active={activeGroup === idx}
+ size="lg"
+ >
+ {tab}
+
+ ))}
+
+
+
+
+
+ );
+}
diff --git a/src/helm-frontend/src/routes/Home.tsx b/src/helm-frontend/src/routes/Home.tsx
new file mode 100644
index 0000000000..67e2d343f5
--- /dev/null
+++ b/src/helm-frontend/src/routes/Home.tsx
@@ -0,0 +1,147 @@
+import { useEffect, useState } from "react";
+import getSchema from "@/services/getSchema";
+import type Schema from "@/types/Schema";
+import ModelsList from "@/components/ModelsList";
+import MetricsList from "@/components/MetricsList";
+import ScenariosList from "@/components/ScenariosList";
+
+import helmLogo from "@/assets/helm-logo.png";
+import languageModelHelm from "@/assets/language-model-helm.png";
+import scenariosByMetrics from "@/assets/scenarios-by-metrics.png";
+import taxonomyScenarios from "@/assets/taxonomy-scenarios.png";
+import ai21 from "@/assets/logos/ai21.png";
+import anthropic from "@/assets/logos/anthropic.png";
+import bigscience from "@/assets/logos/bigscience.png";
+import cohere from "@/assets/logos/cohere.png";
+import eleutherai from "@/assets/logos/eleutherai.png";
+import google from "@/assets/logos/google.png";
+import meta from "@/assets/logos/meta.png";
+import microsoft from "@/assets/logos/microsoft.png";
+import nvidia from "@/assets/logos/nvidia.png";
+import openai from "@/assets/logos/openai.png";
+import together from "@/assets/logos/together.png";
+import tsinghuaKeg from "@/assets/logos/tsinghua-keg.png";
+import yandex from "@/assets/logos/yandex.png";
+
+const logos = [
+ ai21,
+ anthropic,
+ bigscience,
+ cohere,
+ eleutherai,
+ google,
+ meta,
+ microsoft,
+ nvidia,
+ openai,
+ together,
+ tsinghuaKeg,
+ yandex,
+];
+
+export default function Home() {
+ const [schema, setSchema] = useState
+
+
+
+
+
+
+
+ A language model takes in text and produces text:
+
+
+ Despite their simplicity, language models are increasingly functioning + as the foundation for almost all language technologies from question + answering to summarization. But their immense capabilities and risks + are not well understood. Holistic Evaluation of Language Models (HELM) + is a living benchmark that aims to improve the transparency of + language models. +
+ +-
+
-
+ Broad coverage and recognition of incompleteness.{" "}
+ We define a taxonomy over the scenarios we would ideally like to
+ evaluate, select scenarios and metrics to cover the space and make
+ explicit what is missing.
+
+
+ -
+ Multi-metric measurement. Rather than focus on
+ isolated metrics such as accuracy, we simultaneously measure
+ multiple metrics (e.g., accuracy, robustness, calibration,
+ efficiency) for each scenario, allowing analysis of tradeoffs.
+
+
+ -
+ Standardization. We evaluate all the models that we
+ have access to on the same scenarios with the same adaptation
+ strategy (e.g., prompting), allowing for controlled comparisons.
+ Thanks to all the companies for providing API access to the
+ limited-access and closed models and{" "}
+
+ Together
+ {" "}
+ for providing the infrastructure to run the open models.
+ + {logos.map((logo, idx) => ( ++++ ))} +
+
+
+
+
+ );
+}
diff --git a/src/helm-frontend/src/routes/Models.tsx b/src/helm-frontend/src/routes/Models.tsx
new file mode 100644
index 0000000000..2c841411df
--- /dev/null
+++ b/src/helm-frontend/src/routes/Models.tsx
@@ -0,0 +1,140 @@
+import { useEffect, useState } from "react";
+import {
+ Card,
+ CategoryBar,
+ DonutChart,
+ Legend,
+ Metric,
+ Text,
+} from "@tremor/react";
+import type Model from "@/types/Model";
+import getSchema from "@/services/getSchema";
+import PageTitle from "@/components/PageTitle";
+import AccessBadge from "@/components/AccessBadge";
+import MarkdownValue from "@/components/MarkdownValue";
+import Loading from "@/components/Loading";
+
+export default function Models() {
+ const [models, setModels] = useState
+
+
+
+ Models
+ {models.length}
+
+
+ Creator Organizations
+
+
+
+
+
+
+
+
+
+
+ );
+}
diff --git a/src/helm-frontend/src/routes/Run.tsx b/src/helm-frontend/src/routes/Run.tsx
new file mode 100644
index 0000000000..4ee5e06e2e
--- /dev/null
+++ b/src/helm-frontend/src/routes/Run.tsx
@@ -0,0 +1,314 @@
+import { useEffect, useState } from "react";
+import { useParams, useSearchParams } from "react-router-dom";
+import { Badge, Card, List, ListItem } from "@tremor/react";
+import { ArrowDownTrayIcon } from "@heroicons/react/24/solid";
+import getSchema from "@/services/getSchema";
+import getRunSpecs from "@/services/getRunSpecs";
+import type RunSpec from "@/types/RunSpec";
+import getInstances from "@/services/getInstances";
+import type Instance from "@/types/Instance";
+import getStatsByName from "@/services/getStatsByName";
+import type Stat from "@/types/Stat";
+import getDisplayRequestsByName from "@/services/getDisplayRequestsByName";
+import type DisplayRequestsMap from "@/types/DisplayRequestsMap";
+import getDisplayPredictionsByName from "@/services/getDisplayPredictionsByName";
+import type DisplayPredictionsMap from "@/types/DisplayPredictionsMap";
+import getScenarioByName from "@/services/getScenarioByName";
+import type Scenario from "@/types/Scenario";
+import { getRunSpecByNameUrl } from "@/services/getRunSpecByName";
+import { getScenarioStateByNameUrl } from "@/services/getScenarioStateByName";
+import Tab from "@/components/Tab";
+import Tabs from "@/components/Tabs";
+import InstanceData from "@/components/InstanceData";
+import Loading from "@/components/Loading";
+import Pagination from "@/components/Pagination";
+import Model from "@/types/Model";
+import MarkdownValue from "@/components/MarkdownValue";
+import StatNameDisplay from "@/components/StatNameDisplay";
+
+const INSTANCES_PAGE_SIZE = 10;
+const METRICS_PAGE_SIZE = 50;
+
+export default function Run() {
+ const { runName } = useParams();
+ const [searchParams, setSearchParams] = useSearchParams();
+ const [activeTab, setActiveTab] = useState| Creator | +Model | +Description | +Access | +
|---|---|---|---|
| {model.creator_organization} | +
+ {model.display_name}
+ + {model.name} + |
+
+ |
+
+ |
+
+
+
+
+
+
+
+
+ {scenario.name}
+
+
+ {runSpec.adapter_spec.model}
+
+
+
+ {scenario.tags.map((tag) => (
+
+ {tag}
+
+ ))}
+
+
+
+ Adapter Specification
+ +
+
+ -
+ {Object.entries(runSpec.adapter_spec).map(([key, value], idx) => (
+
+
+ setActiveTab(0)}
+ >
+ Instances + Predictions
+
+ setActiveTab(1)}
+ >
+ All metrics
+
+
+
+ {activeTab === 0 ? (
+ <>
+
+ {pagedInstances.map((instance, idx) => (
+
+
+ {
+ const nextMetricsPage = Math.min(
+ currentMetricsPage + 1,
+ totalMetricsPages,
+ );
+ setCurrentMetricsPage(nextMetricsPage);
+ searchParams.set("metricsPage", String(nextMetricsPage));
+ setSearchParams(searchParams);
+ }}
+ onPrevPage={() => {
+ const prevMetricsPage = Math.max(currentMetricsPage - 1, 1);
+ setCurrentMetricsPage(prevMetricsPage);
+ searchParams.set("metricsPage", String(prevMetricsPage));
+ setSearchParams(searchParams);
+ }}
+ currentPage={currentMetricsPage}
+ totalPages={totalMetricsPages}
+ />
+
+ )}
+
+ );
+}
diff --git a/src/helm-frontend/src/routes/Runs.tsx b/src/helm-frontend/src/routes/Runs.tsx
new file mode 100644
index 0000000000..1b9b51ad9f
--- /dev/null
+++ b/src/helm-frontend/src/routes/Runs.tsx
@@ -0,0 +1,163 @@
+import { SyntheticEvent, useEffect, useState } from "react";
+import { useSearchParams } from "react-router-dom";
+import getRunSpecs from "@/services/getRunSpecs";
+import type RunSpec from "@/types/RunSpec";
+import PageTitle from "@/components/PageTitle";
+import Link from "@/components/Link";
+import Loading from "@/components/Loading";
+import Pagination from "@/components/Pagination";
+import { MagnifyingGlassIcon } from "@heroicons/react/24/solid";
+
+const PAGE_SIZE = 100;
+
+export default function Runs() {
+ const [searchParams, setSearchParams] = useSearchParams();
+ const [runSpecs, setRunSpecs] = useState
+
+
+
+
+ | {key} | + ))} +|
|---|---|
|
+
+ {
+ /* eslint-disable-next-line @typescript-eslint/no-unsafe-member-access */
+ value.name
+ }
+
+ |
+ );
+ }
+ return {value} | ; + })} +
+
+
+
+
+ {totalPages > 0 ? (
+ | Run | +Model | +Groups | +Adapter method | +Subject / Task | +
|---|---|---|---|---|
| + {runSpec.name} + | +{runSpec.adapter_spec.model} | +{runSpec.groups.join(", ")} | +{runSpec.adapter_spec.method} | ++ {runSpec.scenario_spec.args.subject || + runSpec.scenario_spec.args.task || + "-"} + | +
No results
+ )}
+
+ );
+}
diff --git a/src/helm-frontend/src/routes/Scenarios.tsx b/src/helm-frontend/src/routes/Scenarios.tsx
new file mode 100644
index 0000000000..a8935ccbe9
--- /dev/null
+++ b/src/helm-frontend/src/routes/Scenarios.tsx
@@ -0,0 +1,117 @@
+import { useEffect, useState } from "react";
+import { BarList, Card, Metric, Text } from "@tremor/react";
+import type RunGroup from "@/types/RunGroup";
+import getSchema from "@/services/getSchema";
+import MarkdownValue from "@/components/MarkdownValue";
+import PageTitle from "@/components/PageTitle";
+import Link from "@/components/Link";
+import Loading from "@/components/Loading";
+
+export default function Scenarios() {
+ const [runGroups, setRunGroups] = useState
+
+ Total Scenarios
+
+ {runGroups.length}
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+}
diff --git a/src/helm-frontend/src/services/getDisplayPredictionsByName.ts b/src/helm-frontend/src/services/getDisplayPredictionsByName.ts
new file mode 100644
index 0000000000..b6e1a3eadc
--- /dev/null
+++ b/src/helm-frontend/src/services/getDisplayPredictionsByName.ts
@@ -0,0 +1,22 @@
+import type DisplayPrediction from "@/types/DisplayPrediction";
+import getBenchmarkEndpoint from "@/utils/getBenchmarkEndpoint";
+import getBenchmarkSuite from "@/utils/getBenchmarkSuite";
+
+export default async function getDisplayPredictionsByName(
+ runName: string,
+ signal: AbortSignal,
+): Promise| Scenario | +Task | +What | +Who | +When | +Language | +Description | +
|---|---|---|---|---|---|---|
| + + {runGroup.display_name} + + {runGroup.name} + | +{runGroup.taxonomy?.task || ""} | +{runGroup.taxonomy?.what || ""} | +{runGroup.taxonomy?.who || ""} | +{runGroup.taxonomy?.when || ""} | +{runGroup.taxonomy?.language || ""} | +
+ |
+