From 960a7aeac1c23f91436c1fdddc09cf24ad93e6b9 Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Fri, 26 Jul 2024 13:27:37 +0800 Subject: [PATCH] feat(frontend): more datasource support --- frontend/app/src/api/datasources.ts | 42 ++++++- frontend/app/src/api/rag.ts | 26 ++-- .../(main)/(admin)/datasources/[id]/page.tsx | 46 +++++-- .../(admin)/datasources/create/layout.tsx | 4 +- .../create/web-single-page/page.tsx | 113 ++++++++++++++++++ .../datasources/create/web-sitemap/page.tsx | 113 ++++++++++++++++++ .../app/(main)/(admin)/datasources/page.tsx | 17 +-- .../(main)/(admin)/index-progress/page.tsx | 52 +------- .../app/src/components/charts/TotalCard.tsx | 22 ++++ 9 files changed, 346 insertions(+), 89 deletions(-) create mode 100644 frontend/app/src/app/(main)/(admin)/datasources/create/web-single-page/page.tsx create mode 100644 frontend/app/src/app/(main)/(admin)/datasources/create/web-sitemap/page.tsx create mode 100644 frontend/app/src/components/charts/TotalCard.tsx diff --git a/frontend/app/src/api/datasources.ts b/frontend/app/src/api/datasources.ts index 605d27e0..1570a55f 100644 --- a/frontend/app/src/api/datasources.ts +++ b/frontend/app/src/api/datasources.ts @@ -1,3 +1,4 @@ +import { type IndexProgress, indexSchema, type IndexTotalStats, totalSchema } from '@/api/rag'; import { BASE_URL, buildUrlParams, handleResponse, opaqueCookieHeader, type Page, type PageParams, zodPage } from '@/lib/request'; import { zodJsonDate } from '@/lib/zod'; import { z, type ZodType } from 'zod'; @@ -10,12 +11,23 @@ interface DatasourceBase { updated_at: Date; user_id: string; build_kg_index: boolean; - data_source_type: 'file'; } -export type Datasource = DatasourceBase & { +export type Datasource = DatasourceBase & ({ data_source_type: 'file' config: { file_id: number, file_name: string }[] +} | { + data_source_type: 'web_sitemap' | 'web_single_page' + config: { url: string } +}) + +export type DataSourceIndexProgress = { + vector_index: IndexProgress + documents: IndexTotalStats + chunks: IndexTotalStats + kg_index?: IndexProgress + entities?: IndexTotalStats + relationships?: IndexTotalStats } export interface BaseCreateDatasourceParams { @@ -24,10 +36,13 @@ export interface BaseCreateDatasourceParams { build_kg_index: boolean; } -export type CreateDatasourceParams = BaseCreateDatasourceParams & { +export type CreateDatasourceParams = BaseCreateDatasourceParams & ({ data_source_type: 'file' config: { file_id: number, file_name: string }[] -} +} | { + data_source_type: 'web_sitemap' | 'web_single_page' + config: { url: string } +}) export interface Upload { created_at?: Date; @@ -55,6 +70,10 @@ const datasourceSchema = baseDatasourceSchema z.object({ data_source_type: z.literal('file'), config: z.array(z.object({ file_id: z.number(), file_name: z.string() })), + }), + z.object({ + data_source_type: z.enum(['web_sitemap', 'web_single_page']), + config: z.object({ url: z.string() }), })], )) satisfies ZodType; @@ -69,6 +88,15 @@ const uploadSchema = z.object({ updated_at: zodJsonDate().optional(), }) satisfies ZodType; +const datasourceOverviewSchema = z.object({ + vector_index: indexSchema, + documents: totalSchema, + chunks: totalSchema, + kg_index: indexSchema.optional(), + entities: totalSchema.optional(), + relationships: totalSchema.optional(), +}) satisfies ZodType; + export async function listDataSources ({ page = 1, size = 10 }: PageParams = {}): Promise> { return fetch(`${BASE_URL}/api/v1/admin/datasources?${buildUrlParams({ page, size }).toString()}`, { headers: await opaqueCookieHeader(), @@ -81,6 +109,12 @@ export async function getDatasource (id: number): Promise { }).then(handleResponse(datasourceSchema)); } +export async function getDatasourceOverview (id: number): Promise { + return fetch(`${BASE_URL}/api/v1/admin/datasources/${id}/overview`, { + headers: await opaqueCookieHeader(), + }).then(handleResponse(datasourceOverviewSchema)); +} + export async function createDatasource (params: CreateDatasourceParams) { return fetch(`${BASE_URL}/api/v1/admin/datasources`, { method: 'POST', diff --git a/frontend/app/src/api/rag.ts b/frontend/app/src/api/rag.ts index 3adafae0..9a269159 100644 --- a/frontend/app/src/api/rag.ts +++ b/frontend/app/src/api/rag.ts @@ -9,28 +9,24 @@ export type IndexProgress = { failed?: number } +export type IndexTotalStats = { + total: number +} + export interface RagIndexProgress { kg_index: IndexProgress; vector_index: IndexProgress; - documents: { - total: number - }; - chunks: { - total: number - }; - entities: { - total: number - }; - relationships: { - total: number - }; + documents: IndexTotalStats; + chunks: IndexTotalStats; + entities: IndexTotalStats; + relationships: IndexTotalStats; } -const totalSchema = z.object({ +export const totalSchema = z.object({ total: z.number(), -}); +}) satisfies ZodType; -const indexSchema = z.object({ +export const indexSchema = z.object({ not_started: z.number().optional(), pending: z.number().optional(), running: z.number().optional(), diff --git a/frontend/app/src/app/(main)/(admin)/datasources/[id]/page.tsx b/frontend/app/src/app/(main)/(admin)/datasources/[id]/page.tsx index 9db1cf1f..927d6a7e 100644 --- a/frontend/app/src/app/(main)/(admin)/datasources/[id]/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/datasources/[id]/page.tsx @@ -1,18 +1,21 @@ 'use client'; -import { getDatasource } from '@/api/datasources'; +import { getDatasource, getDatasourceOverview } from '@/api/datasources'; import { AdminPageHeading } from '@/components/admin-page-heading'; +import { IndexProgressChart } from '@/components/charts/IndexProgressChart'; +import { TotalCard } from '@/components/charts/TotalCard'; import { DateFormat } from '@/components/date-format'; import { OptionDetail } from '@/components/option-detail'; import { Badge } from '@/components/ui/badge'; -import { Separator } from '@/components/ui/separator'; -import { Loader2Icon } from 'lucide-react'; +import { ArrowRightIcon, FileTextIcon, Loader2Icon, MapPinIcon, PuzzleIcon, RouteIcon } from 'lucide-react'; +import Link from 'next/link'; import useSWR from 'swr'; export default function DatasourcePage ({ params }: { params: { id: string } }) { const id = parseInt(params.id); const { data: datasource } = useSWR(`api.datasource.get?id=${id}`, () => getDatasource(id)); + const { data: progress } = useSWR(`api.datasource.get-overview?id=${id}`, () => getDatasourceOverview(id)); return (
@@ -25,6 +28,9 @@ export default function DatasourcePage ({ params }: { params: { id: string } })
+ {(datasource?.data_source_type === 'web_single_page' || datasource?.data_source_type === 'web_sitemap') && ( + + )} } /> @@ -32,12 +38,38 @@ export default function DatasourcePage ({ params }: { params: { id: string } })
-
-

Files

+ {progress && ( + <> +
+ } + total={progress.documents.total} + > + All documents + + } total={progress.chunks.total} /> + {datasource?.build_kg_index && progress.entities && } + total={progress.entities.total} + > + Graph Editor + } + {datasource?.build_kg_index && progress.relationships && } total={progress.relationships.total} />} +
+
+ + {datasource?.build_kg_index && progress.kg_index && } +
+ + )} + {datasource?.data_source_type === 'file' &&
+

Files

{datasource?.data_source_type === 'file' && (
{datasource.config.map(file => - + {file.file_name} @@ -45,7 +77,7 @@ export default function DatasourcePage ({ params }: { params: { id: string } }) )}
)} -
+
}
); } \ No newline at end of file diff --git a/frontend/app/src/app/(main)/(admin)/datasources/create/layout.tsx b/frontend/app/src/app/(main)/(admin)/datasources/create/layout.tsx index 68a7147a..7c07e1cd 100644 --- a/frontend/app/src/app/(main)/(admin)/datasources/create/layout.tsx +++ b/frontend/app/src/app/(main)/(admin)/datasources/create/layout.tsx @@ -23,10 +23,10 @@ export default function Layout ({ children }: { children: ReactNode }) { File - + Web Sitemap - + Web Single Page diff --git a/frontend/app/src/app/(main)/(admin)/datasources/create/web-single-page/page.tsx b/frontend/app/src/app/(main)/(admin)/datasources/create/web-single-page/page.tsx new file mode 100644 index 00000000..4ee31704 --- /dev/null +++ b/frontend/app/src/app/(main)/(admin)/datasources/create/web-single-page/page.tsx @@ -0,0 +1,113 @@ +'use client'; + +import { type BaseCreateDatasourceParams, createDatasource } from '@/api/datasources'; +import { Button } from '@/components/ui/button'; +import { Form, FormControl, FormDescription, FormField, FormItem, FormLabel, FormMessage } from '@/components/ui/form'; +import { Input } from '@/components/ui/input'; +import { Separator } from '@/components/ui/separator'; +import { Switch } from '@/components/ui/switch'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { Loader2Icon } from 'lucide-react'; +import { useRouter } from 'next/navigation'; +import { useTransition } from 'react'; +import { useForm } from 'react-hook-form'; +import { z, type ZodType } from 'zod'; + +const schema = z.object({ + name: z.string(), + description: z.string(), + build_kg_index: z.boolean(), + url: z.string().url(), +}) satisfies ZodType; + +export default function Page () { + const [navigating, startNavigation] = useTransition(); + const router = useRouter(); + const form = useForm({ + resolver: zodResolver(schema), + defaultValues: { + name: '', + description: '', + build_kg_index: false, + url: '', + }, + }); + + const handleSubmit = form.handleSubmit(async ({ url, ...data }) => { + const createdDatasource = await createDatasource({ + ...data, + data_source_type: 'web_single_page', + config: { url }, + }); + startNavigation(() => { + router.push(`/datasources/${createdDatasource.id}`); + }); + }); + + return ( +
+ + ( + + Name + + + + + + )} + /> + ( + + Description + + + + + + )} + /> + ( + + URL + + + + + + )} + /> + +
+ + ( + +
+ Build KnowledgeGraph Index + + Enable to build knowledge graph index. + +
+ + + + +
+ )} + /> + +
+ + ); +} diff --git a/frontend/app/src/app/(main)/(admin)/datasources/create/web-sitemap/page.tsx b/frontend/app/src/app/(main)/(admin)/datasources/create/web-sitemap/page.tsx new file mode 100644 index 00000000..5d747397 --- /dev/null +++ b/frontend/app/src/app/(main)/(admin)/datasources/create/web-sitemap/page.tsx @@ -0,0 +1,113 @@ +'use client'; + +import { type BaseCreateDatasourceParams, createDatasource } from '@/api/datasources'; +import { Button } from '@/components/ui/button'; +import { Form, FormControl, FormDescription, FormField, FormItem, FormLabel, FormMessage } from '@/components/ui/form'; +import { Input } from '@/components/ui/input'; +import { Separator } from '@/components/ui/separator'; +import { Switch } from '@/components/ui/switch'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { Loader2Icon } from 'lucide-react'; +import { useRouter } from 'next/navigation'; +import { useTransition } from 'react'; +import { useForm } from 'react-hook-form'; +import { z, type ZodType } from 'zod'; + +const schema = z.object({ + name: z.string(), + description: z.string(), + build_kg_index: z.boolean(), + url: z.string().url(), +}) satisfies ZodType; + +export default function Page () { + const [navigating, startNavigation] = useTransition(); + const router = useRouter(); + const form = useForm({ + resolver: zodResolver(schema), + defaultValues: { + name: '', + description: '', + build_kg_index: false, + url: '', + }, + }); + + const handleSubmit = form.handleSubmit(async ({ url, ...data }) => { + const createdDatasource = await createDatasource({ + ...data, + data_source_type: 'web_sitemap', + config: { url }, + }); + startNavigation(() => { + router.push(`/datasources/${createdDatasource.id}`); + }); + }); + + return ( +
+ + ( + + Name + + + + + + )} + /> + ( + + Description + + + + + + )} + /> + ( + + URL + + + + + + )} + /> + +
+ + ( + +
+ Build KnowledgeGraph Index + + Enable to build knowledge graph index. + +
+ + + + +
+ )} + /> + +
+ + ); +} diff --git a/frontend/app/src/app/(main)/(admin)/datasources/page.tsx b/frontend/app/src/app/(main)/(admin)/datasources/page.tsx index e08dd6a6..acdfdbbe 100644 --- a/frontend/app/src/app/(main)/(admin)/datasources/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/datasources/page.tsx @@ -2,31 +2,26 @@ import { type Datasource, listDataSources } from '@/api/datasources'; import { AdminPageHeading } from '@/components/admin-page-heading'; -import { datetime } from '@/components/cells/datetime'; import { DataTableHeading } from '@/components/data-table-heading'; import { DataTableRemote } from '@/components/data-table-remote'; import { buttonVariants } from '@/components/ui/button'; -import type { CellContext, ColumnDef } from '@tanstack/react-table'; +import type { ColumnDef } from '@tanstack/react-table'; import { createColumnHelper } from '@tanstack/table-core'; import { PlusIcon } from 'lucide-react'; import Link from 'next/link'; const helper = createColumnHelper(); -const mono = (cell: CellContext) => {cell.getValue()}; - const columns = [ - helper.accessor('id', { - cell: cell => ( - {cell.getValue()} + helper.display({ + header: 'Name', + cell: ({ row }) => ( + {row.original.name} ), }), - helper.accessor('name', {}), - helper.accessor('description', {}), + helper.accessor('data_source_type', {}), helper.accessor('build_kg_index', {}), helper.accessor('user_id', {}), - helper.accessor('created_at', { cell: datetime }), - helper.accessor('updated_at', { cell: datetime }), ] as ColumnDef[]; export default function ChatEnginesPage () { diff --git a/frontend/app/src/app/(main)/(admin)/index-progress/page.tsx b/frontend/app/src/app/(main)/(admin)/index-progress/page.tsx index b7e755b4..332a6f23 100644 --- a/frontend/app/src/app/(main)/(admin)/index-progress/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/index-progress/page.tsx @@ -1,15 +1,10 @@ -import { getIndexProgress, type IndexProgress } from '@/api/rag'; +import { getIndexProgress } from '@/api/rag'; import { AdminPageHeading } from '@/components/admin-page-heading'; import { IndexProgressChart } from '@/components/charts/IndexProgressChart'; - -import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { TotalCard } from '@/components/charts/TotalCard'; import { requireAuth } from '@/lib/auth'; -import { cn } from '@/lib/utils'; import { ArrowRightIcon, FileTextIcon, MapPinIcon, PuzzleIcon, RouteIcon } from 'lucide-react'; import Link from 'next/link'; -import type { ReactNode } from 'react'; - -const nf = new Intl.NumberFormat('en-US', {}); export default async function IndexProgressPage () { await requireAuth(); @@ -44,46 +39,3 @@ export default async function IndexProgressPage () { ); } - -function TotalCard ({ title, icon, total, children }: { title: string, icon: ReactNode, total: number, children?: ReactNode }) { - return ( - - - {title} - {icon} - - -
{nf.format(total)}
-

- {children} -

-
-
- ); -} - -function IndexCard ({ title, progress }: { title: string, progress: IndexProgress }) { - return ( - - - {title} - - - {progress.not_started && } - {progress.pending && } - {progress.running && } - {progress.completed && } - {progress.failed && } - - - ); -} - -function Detail ({ valueClassName, title, value }: { valueClassName?: string, title: string, value: number }) { - return ( -
-
{title}
-
{nf.format(value)}
-
- ); -} diff --git a/frontend/app/src/components/charts/TotalCard.tsx b/frontend/app/src/components/charts/TotalCard.tsx new file mode 100644 index 00000000..ef130e4a --- /dev/null +++ b/frontend/app/src/components/charts/TotalCard.tsx @@ -0,0 +1,22 @@ +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import type { ReactNode } from 'react'; + +const nf = new Intl.NumberFormat('en-US', {}); + +export function TotalCard ({ title, icon, total, children }: { title: string, icon: ReactNode, total: number, children?: ReactNode }) { + return ( + + + {title} + {icon} + + +
{nf.format(total)}
+

+ {children} +

+
+
+ ); +} +