diff --git a/README.md b/README.md index 9a238ffb..a47ea9b9 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ [![Backend Docker Image Version](https://img.shields.io/docker/v/tidbai/backend?sort=semver&arch=amd64&label=tidbai%2Fbackend&color=blue&logo=fastapi)](https://hub.docker.com/r/tidbai/backend) [![Frontend Docker Image Version](https://img.shields.io/docker/v/tidbai/frontend?sort=semver&arch=amd64&label=tidbai%2Ffrontend&&color=blue&logo=next.js)](https://hub.docker.com/r/tidbai/frontend) -[![E2E Status](https://img.shields.io/github/check-runs/pingcap/tidb.ai/main?nameFilter=E2E%20Test&label=e2e)](https://github.com/pingcap/tidb.ai/actions/workflows/release.yml) +[![E2E Status](https://img.shields.io/github/check-runs/pingcap/tidb.ai/main?nameFilter=E2E%20Test&label=e2e)](https://tidb-ai-playwright.vercel.app/) ## Introduction diff --git a/frontend/app/.gitignore b/frontend/app/.gitignore index fe0839b7..cf3f4251 100644 --- a/frontend/app/.gitignore +++ b/frontend/app/.gitignore @@ -43,4 +43,5 @@ public/robots.txt public/sitemap.xml public/sitemap-*.xml -*storybook.log \ No newline at end of file +*storybook.log +storybook-static diff --git a/frontend/app/src/experimental/chat-verify-service/api.ts b/frontend/app/src/experimental/chat-verify-service/api.ts index e79fb43d..3bb3e9a4 100644 --- a/frontend/app/src/experimental/chat-verify-service/api.ts +++ b/frontend/app/src/experimental/chat-verify-service/api.ts @@ -1,9 +1,9 @@ import { handleResponse } from '@/lib/request'; -import { z } from 'zod'; +import { z, type ZodType } from 'zod'; const HOST = 'https://verify.tidb.ai'; -export const enum VerifyState { +export const enum VerifyStatus { CREATED = 'CREATED', EXTRACTING = 'EXTRACTING', VALIDATING = 'VALIDATING', @@ -12,12 +12,34 @@ export const enum VerifyState { SKIPPED = 'SKIPPED' } +export interface MessageVerifyResponse { + status: VerifyStatus; + message?: string | null; + runs: MessageVerifyResponse.Run[]; +} + +export namespace MessageVerifyResponse { + export type Run = { + sql: string + explanation: string + } & + ({ + success: true, + results?: any[][] + } | { + success: false, + sql_error_code?: number | null, + sql_error_message?: string | null, + warnings: string[] + }) +} + const verifyResponse = z.object({ job_id: z.string(), }); const getVerifyResponse = z.object({ - status: z.enum([VerifyState.CREATED, VerifyState.EXTRACTING, VerifyState.VALIDATING, VerifyState.SUCCESS, VerifyState.FAILED]), + status: z.enum([VerifyStatus.CREATED, VerifyStatus.EXTRACTING, VerifyStatus.VALIDATING, VerifyStatus.SUCCESS, VerifyStatus.FAILED, VerifyStatus.SKIPPED]), message: z.string().nullish(), runs: z.object({ sql: z.string(), @@ -32,7 +54,7 @@ const getVerifyResponse = z.object({ sql_error_message: z.string().nullish(), warnings: z.string().array(), }))).array(), -}); +}) satisfies ZodType; export async function verify (question: string, answer: string) { return await fetch(`${HOST}/api/v1/sqls-validation`, { @@ -48,10 +70,10 @@ export async function getVerify (id: string) { return await fetch(`${HOST}/api/v1/sqls-validation/${id}`).then(handleResponse(getVerifyResponse)); } -export function isFinalVerifyState (state: VerifyState) { - return [VerifyState.SUCCESS, VerifyState.FAILED, VerifyState.SKIPPED].includes(state); +export function isFinalVerifyState (state: VerifyStatus) { + return [VerifyStatus.SUCCESS, VerifyStatus.FAILED, VerifyStatus.SKIPPED].includes(state); } -export function isVisibleVerifyState (state: VerifyState) { - return [VerifyState.SUCCESS, VerifyState.FAILED, VerifyState.VALIDATING].includes(state); -} \ No newline at end of file +export function isVisibleVerifyState (state: VerifyStatus) { + return [VerifyStatus.SUCCESS, VerifyStatus.FAILED].includes(state); +} diff --git a/frontend/app/src/experimental/chat-verify-service/message-verify.stories.tsx b/frontend/app/src/experimental/chat-verify-service/message-verify.stories.tsx index a1f83e72..95efa3b3 100644 --- a/frontend/app/src/experimental/chat-verify-service/message-verify.stories.tsx +++ b/frontend/app/src/experimental/chat-verify-service/message-verify.stories.tsx @@ -1,6 +1,6 @@ import { AuthProvider } from '@/components/auth/AuthProvider'; import { ChatMessageController } from '@/components/chat/chat-message-controller'; -import { getVerify, verify, VerifyState } from '@/experimental/chat-verify-service/api.mock'; +import { getVerify, verify, VerifyStatus } from '@/experimental/chat-verify-service/api.mock'; import type { Meta, StoryObj } from '@storybook/react'; import { mutate } from 'swr'; import { MessageVerify } from './message-verify'; @@ -34,10 +34,12 @@ const meta = { render (_, { id }) { return ( {}}> - +
+ +
); }, @@ -46,15 +48,28 @@ const meta = { export default meta; type Story = StoryObj; -// More on writing stories with args: https://storybook.js.org/docs/writing-stories/args -export const Verified: Story = { +export const Creating: Story = { + beforeEach: () => { + getVerify.mockReturnValue(new Promise(() => {})); + }, +}; + +export const Created: Story = { beforeEach: () => { getVerify.mockReturnValue(Promise.resolve({ - status: VerifyState.SUCCESS, - message: 'This is a success message returned from server', - runs: [ - { sql: 'SOME A FROM B FROM C FROM D FROM E FROM F FROM G', results: [], success: true, explanation: 'some description for this SQL' }, - ], + status: VerifyStatus.CREATED, + message: 'This is a created message returned from server', + runs: [], + })); + }, +}; + +export const Extracting: Story = { + beforeEach: () => { + getVerify.mockReturnValue(Promise.resolve({ + status: VerifyStatus.EXTRACTING, + message: 'This is a extracting message returned from server', + runs: [], })); }, }; @@ -62,17 +77,29 @@ export const Verified: Story = { export const Validating: Story = { beforeEach: () => { getVerify.mockReturnValue(Promise.resolve({ - status: VerifyState.VALIDATING, + status: VerifyStatus.VALIDATING, message: 'This is a validating message returned from server', runs: [], })); }, }; +export const Verified: Story = { + beforeEach: () => { + getVerify.mockReturnValue(Promise.resolve({ + status: VerifyStatus.SUCCESS, + message: 'This is a success message returned from server', + runs: [ + { sql: 'SOME A FROM B FROM C FROM D FROM E FROM F FROM G', results: [], success: true, explanation: 'some description for this SQL' }, + ], + })); + }, +}; + export const Failed: Story = { beforeEach: () => { getVerify.mockReturnValue(Promise.resolve({ - status: VerifyState.FAILED, + status: VerifyStatus.FAILED, message: 'This is a failed message returned from server', runs: [ { sql: 'SOME A FROM B FROM C FROM D FROM E FROM F FROM G', results: [], success: true, explanation: 'Some description for this SQL' }, @@ -82,3 +109,18 @@ export const Failed: Story = { }, }; +export const Skipped: Story = { + beforeEach: () => { + getVerify.mockReturnValue(Promise.resolve({ + status: VerifyStatus.SKIPPED, + message: 'This is a skipped message returned from server', + runs: [], + })); + }, +}; + +export const ApiError: Story = { + beforeEach: () => { + getVerify.mockReturnValue(Promise.reject(new Error('This is error from server'))); + }, +}; \ No newline at end of file diff --git a/frontend/app/src/experimental/chat-verify-service/message-verify.tsx b/frontend/app/src/experimental/chat-verify-service/message-verify.tsx index 5d3a8bcc..797dfd83 100644 --- a/frontend/app/src/experimental/chat-verify-service/message-verify.tsx +++ b/frontend/app/src/experimental/chat-verify-service/message-verify.tsx @@ -1,16 +1,20 @@ +import { getVerify, isFinalVerifyState, isVisibleVerifyState, type MessageVerifyResponse, verify, VerifyStatus } from '#experimental/chat-verify-service/api'; import { useAuth } from '@/components/auth/AuthProvider'; import { useChatMessageField, useChatMessageStreamState } from '@/components/chat/chat-hooks'; import type { ChatMessageController } from '@/components/chat/chat-message-controller'; import { isNotFinished } from '@/components/chat/utils'; import { Button } from '@/components/ui/button'; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible'; -import { getVerify, isFinalVerifyState, isVisibleVerifyState, verify, VerifyState } from '#experimental/chat-verify-service/api'; -import { CheckCircle2Icon, CheckIcon, ChevronDownIcon, Loader2Icon, XIcon } from 'lucide-react'; +import { getErrorMessage } from '@/lib/errors'; +import { cn } from '@/lib/utils'; +import { AnimatePresence, motion } from 'framer-motion'; +import { CheckCircle2Icon, CheckIcon, ChevronDownIcon, CircleMinus, Loader2Icon, XIcon } from 'lucide-react'; import { InformationCircleIcon } from 'nextra/icons'; -import { useEffect, useState } from 'react'; +import { type ReactElement, useEffect, useState } from 'react'; import useSWR from 'swr'; export function MessageVerify ({ user, assistant }: { user: ChatMessageController | undefined, assistant: ChatMessageController | undefined }) { + const [open, setOpen] = useState(false); const messageState = useChatMessageStreamState(assistant); const question = useChatMessageField(user, 'content'); const answer = useChatMessageField(assistant, 'content'); @@ -18,12 +22,13 @@ export function MessageVerify ({ user, assistant }: { user: ChatMessageControlle const me = useAuth(); const [verifyId, setVerifyId] = useState(); const [verifying, setVerifying] = useState(false); + const [verifyError, setVerifyError] = useState(); const enabled = useExperimentalMessageVerifyFeature(); const isSuperuser = !!me.me?.is_superuser; const shouldPoll = enabled && !!verifyId && !!assistant && isSuperuser; - const { data: result } = useSWR( + const { data: result, isLoading: isLoadingResult, error: pollError } = useSWR( shouldPoll && `experimental.chat-message.${assistant.id}.verify`, () => getVerify(verifyId!), { revalidateOnMount: true, @@ -39,15 +44,14 @@ export function MessageVerify ({ user, assistant }: { user: ChatMessageControlle ); const messageFinished = !isNotFinished(messageState); - const verifyFinished = result ? isFinalVerifyState(result.status) : false; - const shouldDisplayContent = result ? isVisibleVerifyState(result.status) : false; + const canOpen = result ? isVisibleVerifyState(result.status) : false; + const creating = verifying || !!(verifyId && !result && isLoadingResult); + const error: unknown = verifyError ?? pollError; useEffect(() => { if (enabled && !verifyId && question && answer && messageFinished && !verifying) { verify(question, answer) - .then(result => { - setVerifyId(result.job_id); - }) + .then(result => setVerifyId(result.job_id), error => setVerifyError(error)) .finally(() => { setVerifying(false); }); @@ -55,56 +59,132 @@ export function MessageVerify ({ user, assistant }: { user: ChatMessageControlle }, [enabled, verifyId, messageFinished, question, answer, verifying]); useEffect(() => { - console.debug(`[message-verify] display=${shouldDisplayContent}:`, result); - }, [shouldDisplayContent, result]); + console.debug(`[message-verify]`, result); + }, [result]); - if (!isSuperuser || !enabled || !messageFinished || !shouldDisplayContent) { + if (!isSuperuser || !enabled || !messageFinished) { return null; } return ( - + - - - {result &&
    - {result.runs.map(((run, index) => ( -
  • -
    - 
    -                    {run.success ?  : }
-                    {run.sql}
-
    -

    - {run.explanation} -

    - {run.success && 
    {JSON.stringify(run.results)}
    } - {!run.success && 
    {run.sql_error_code} {run.sql_error_message}
    } -
    -
    • - )))} -
} - {result && result.runs.length === 0 && ( -
- Empty result. -
- )} + + + {open && result && +
    + {result.runs.map(((run, index) => ( +
  • + +
    • + )))} +
+ } + +
+ Powered by TiDB Serverless +
); } +const defaultMessages = { + 'creating': 'Prepare to validate message...', + 'error': 'Failed to validate message.', + [VerifyStatus.CREATED]: 'Prepare to validate message...', + [VerifyStatus.EXTRACTING]: 'Extracting SQL...', + [VerifyStatus.VALIDATING]: 'Validation SQL...', + [VerifyStatus.SUCCESS]: 'Message validation succeed.', + [VerifyStatus.FAILED]: 'Message validation failed.', + [VerifyStatus.SKIPPED]: 'Message validated skipped.', +}; + +const skippedIcon = ; +const loadingIcon = ; +const succeedIcon = ; +const failedIcon = ; +const errorIcon = ; + +function MessageVerifyHeader ({ creating, error, result }: { creating?: boolean, error: unknown, result: MessageVerifyResponse | undefined }) { + let icon: ReactElement | undefined; + let message: string | undefined; + const indicatorVisible = result ? isVisibleVerifyState(result.status) : false; + + if (creating) { + icon = loadingIcon; + message = defaultMessages.creating; + } else if (error) { + icon = errorIcon; + message = getErrorMessage(error) ?? defaultMessages.error; + } else { + switch (result?.status) { + case VerifyStatus.CREATED: + case VerifyStatus.EXTRACTING: + case VerifyStatus.VALIDATING: + icon = loadingIcon; + break; + case VerifyStatus.SUCCESS: + icon = succeedIcon; + break; + case VerifyStatus.FAILED: + icon = failedIcon; + break; + case VerifyStatus.SKIPPED: + icon = skippedIcon; + break; + default: + icon = undefined; + break; + } + message = result?.message ?? (result ? defaultMessages[result.status] : undefined) ?? 'Unknown validation state.'; + } + + return ( + <> + {icon} + {message} + + + ); +} + +function MessageVerifyRun ({ run }: { run: MessageVerifyResponse.Run }) { + return ( +
+ 
+        {run.success ?  : }
+        {run.sql}
+
+

+ {run.explanation} +

+ {run.success && 
{JSON.stringify(run.results)}
} + {!run.success && 
{run.sql_error_code} {run.sql_error_message}
} +
+ ); +} + function useExperimentalMessageVerifyFeature () { const [enabled, setEnabled] = useState(false); diff --git a/frontend/package.json b/frontend/package.json index cab8215e..10284873 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -2,16 +2,17 @@ "name": "tidb-ai-parent", "private": true, "version": "0.0.0", - "packageManager": "pnpm@9.6.0", "license": "Apache-2.0", "scripts": { "test": "pnpm run --filter '*' test", "dev": "pnpm run --filter 'app' dev", + "storybook": "pnpm run --filter 'app' storybook", "build:widget-react": "pnpm run --filter '@tidb.ai/react' build", "build:docker": "pnpm run --filter 'app' build:standalone", "build": "pnpm run --filter 'app' build", "verify": "pnpm run test && pnpm run build && pnpm run build:widget-react" }, + "packageManager": "pnpm@9.6.0", "pnpm": { "patchedDependencies": { "jest-runtime@29.7.0": "patches/jest-runtime@29.7.0.patch",