Skip to content

Commit

Permalink
feat(frontend): more datasource support
Browse files Browse the repository at this point in the history
  • Loading branch information
634750802 committed Jul 26, 2024
1 parent 66e67f7 commit 960a7ae
Show file tree
Hide file tree
Showing 9 changed files with 346 additions and 89 deletions.
42 changes: 38 additions & 4 deletions frontend/app/src/api/datasources.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { type IndexProgress, indexSchema, type IndexTotalStats, totalSchema } from '@/api/rag';
import { BASE_URL, buildUrlParams, handleResponse, opaqueCookieHeader, type Page, type PageParams, zodPage } from '@/lib/request';
import { zodJsonDate } from '@/lib/zod';
import { z, type ZodType } from 'zod';
Expand All @@ -10,12 +11,23 @@ interface DatasourceBase {
updated_at: Date;
user_id: string;
build_kg_index: boolean;
data_source_type: 'file';
}

export type Datasource = DatasourceBase & {
export type Datasource = DatasourceBase & ({
data_source_type: 'file'
config: { file_id: number, file_name: string }[]
} | {
data_source_type: 'web_sitemap' | 'web_single_page'
config: { url: string }
})

export type DataSourceIndexProgress = {
vector_index: IndexProgress
documents: IndexTotalStats
chunks: IndexTotalStats
kg_index?: IndexProgress
entities?: IndexTotalStats
relationships?: IndexTotalStats
}

export interface BaseCreateDatasourceParams {
Expand All @@ -24,10 +36,13 @@ export interface BaseCreateDatasourceParams {
build_kg_index: boolean;
}

export type CreateDatasourceParams = BaseCreateDatasourceParams & {
export type CreateDatasourceParams = BaseCreateDatasourceParams & ({
data_source_type: 'file'
config: { file_id: number, file_name: string }[]
}
} | {
data_source_type: 'web_sitemap' | 'web_single_page'
config: { url: string }
})

export interface Upload {
created_at?: Date;
Expand Down Expand Up @@ -55,6 +70,10 @@ const datasourceSchema = baseDatasourceSchema
z.object({
data_source_type: z.literal('file'),
config: z.array(z.object({ file_id: z.number(), file_name: z.string() })),
}),
z.object({
data_source_type: z.enum(['web_sitemap', 'web_single_page']),
config: z.object({ url: z.string() }),
})],
)) satisfies ZodType<Datasource, any, any>;

Expand All @@ -69,6 +88,15 @@ const uploadSchema = z.object({
updated_at: zodJsonDate().optional(),
}) satisfies ZodType<Upload, any, any>;

const datasourceOverviewSchema = z.object({
vector_index: indexSchema,
documents: totalSchema,
chunks: totalSchema,
kg_index: indexSchema.optional(),
entities: totalSchema.optional(),
relationships: totalSchema.optional(),
}) satisfies ZodType<DataSourceIndexProgress>;

export async function listDataSources ({ page = 1, size = 10 }: PageParams = {}): Promise<Page<Datasource>> {
return fetch(`${BASE_URL}/api/v1/admin/datasources?${buildUrlParams({ page, size }).toString()}`, {
headers: await opaqueCookieHeader(),
Expand All @@ -81,6 +109,12 @@ export async function getDatasource (id: number): Promise<Datasource> {
}).then(handleResponse(datasourceSchema));
}

export async function getDatasourceOverview (id: number): Promise<DataSourceIndexProgress> {
return fetch(`${BASE_URL}/api/v1/admin/datasources/${id}/overview`, {
headers: await opaqueCookieHeader(),
}).then(handleResponse(datasourceOverviewSchema));
}

export async function createDatasource (params: CreateDatasourceParams) {
return fetch(`${BASE_URL}/api/v1/admin/datasources`, {
method: 'POST',
Expand Down
26 changes: 11 additions & 15 deletions frontend/app/src/api/rag.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,28 +9,24 @@ export type IndexProgress = {
failed?: number
}

export type IndexTotalStats = {
total: number
}

export interface RagIndexProgress {
kg_index: IndexProgress;
vector_index: IndexProgress;
documents: {
total: number
};
chunks: {
total: number
};
entities: {
total: number
};
relationships: {
total: number
};
documents: IndexTotalStats;
chunks: IndexTotalStats;
entities: IndexTotalStats;
relationships: IndexTotalStats;
}

const totalSchema = z.object({
export const totalSchema = z.object({
total: z.number(),
});
}) satisfies ZodType<IndexTotalStats>;

const indexSchema = z.object({
export const indexSchema = z.object({
not_started: z.number().optional(),
pending: z.number().optional(),
running: z.number().optional(),
Expand Down
46 changes: 39 additions & 7 deletions frontend/app/src/app/(main)/(admin)/datasources/[id]/page.tsx
Original file line number Diff line number Diff line change
@@ -1,18 +1,21 @@
'use client';

import { getDatasource } from '@/api/datasources';
import { getDatasource, getDatasourceOverview } from '@/api/datasources';
import { AdminPageHeading } from '@/components/admin-page-heading';
import { IndexProgressChart } from '@/components/charts/IndexProgressChart';
import { TotalCard } from '@/components/charts/TotalCard';
import { DateFormat } from '@/components/date-format';
import { OptionDetail } from '@/components/option-detail';
import { Badge } from '@/components/ui/badge';
import { Separator } from '@/components/ui/separator';
import { Loader2Icon } from 'lucide-react';
import { ArrowRightIcon, FileTextIcon, Loader2Icon, MapPinIcon, PuzzleIcon, RouteIcon } from 'lucide-react';
import Link from 'next/link';
import useSWR from 'swr';

export default function DatasourcePage ({ params }: { params: { id: string } }) {
const id = parseInt(params.id);

const { data: datasource } = useSWR(`api.datasource.get?id=${id}`, () => getDatasource(id));
const { data: progress } = useSWR(`api.datasource.get-overview?id=${id}`, () => getDatasourceOverview(id));

return (
<div className="max-w-screen-md space-y-8">
Expand All @@ -25,27 +28,56 @@ export default function DatasourcePage ({ params }: { params: { id: string } })
<div className="space-y-2 text-sm rounded p-4 border">
<OptionDetail title="ID" value={id} />
<OptionDetail title="Type" value={datasource?.data_source_type} />
{(datasource?.data_source_type === 'web_single_page' || datasource?.data_source_type === 'web_sitemap') && (
<OptionDetail title="URL" value={datasource?.config.url} />
)}
<OptionDetail title="Name" value={datasource?.name} />
<OptionDetail title="Description" value={datasource?.description} />
<OptionDetail title="Created at" value={datasource?.created_at && <DateFormat date={datasource.created_at} />} />
<OptionDetail title="Updated at" value={datasource?.updated_at && <DateFormat date={datasource.created_at} />} />
<OptionDetail title="User ID" value={datasource?.user_id} />
<OptionDetail title="Build KnowledegeGraph Index" value={datasource?.build_kg_index ? 'Yes' : 'No'} valueClassName={datasource?.build_kg_index ? 'text-green-500' : 'text-muted-foreground'} />
</div>
<section className='space-y-4'>
<h3 className='font-medium'>Files</h3>
{progress && (
<>
<div className="grid sm:grid-cols-2 md:grid-cols-4 gap-4">
<TotalCard
title="Documents"
icon={<FileTextIcon className="h-4 w-4 text-muted-foreground" />}
total={progress.documents.total}
>
<Link className="flex gap-2 items-center" href="/documents">All documents <ArrowRightIcon className="size-3" /></Link>
</TotalCard>
<TotalCard title="Chunks" icon={<PuzzleIcon className="h-4 w-4 text-muted-foreground" />} total={progress.chunks.total} />
{datasource?.build_kg_index && progress.entities && <TotalCard
title="Entities"
icon={<MapPinIcon className="h-4 w-4 text-muted-foreground" />}
total={progress.entities.total}
>
<Link className="flex gap-2 items-center" href="/knowledge-graph">Graph Editor <ArrowRightIcon className="size-3" /></Link>
</TotalCard>}
{datasource?.build_kg_index && progress.relationships && <TotalCard title="Relationships" icon={<RouteIcon className="h-4 w-4 text-muted-foreground" />} total={progress.relationships.total} />}
</div>
<div className="mt-4 grid grid-cols-2 gap-4">
<IndexProgressChart title="Vector Index" data={progress.vector_index} />
{datasource?.build_kg_index && progress.kg_index && <IndexProgressChart title="Knowledge Graph Index" data={progress.kg_index} />}
</div>
</>
)}
{datasource?.data_source_type === 'file' && <section className="space-y-4">
<h3 className="font-medium">Files</h3>
{datasource?.data_source_type === 'file' && (
<div className="flex gap-2 flex-wrap">
{datasource.config.map(file =>
<Badge key={file.file_id} variant="secondary" className='gap-1'>
<Badge key={file.file_id} variant="secondary" className="gap-1">
<span>
{file.file_name}
</span>
<span className="font-normal text-muted-foreground">#{file.file_id}</span>
</Badge>)}
</div>
)}
</section>
</section>}
</div>
);
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@ export default function Layout ({ children }: { children: ReactNode }) {
<TabsTrigger value="file">
File
</TabsTrigger>
<TabsTrigger value="web-sitemap" disabled>
<TabsTrigger value="web-sitemap">
Web Sitemap
</TabsTrigger>
<TabsTrigger value="web-signlepage" disabled>
<TabsTrigger value="web-single-page">
Web Single Page
</TabsTrigger>
</TabsList>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
'use client';

import { type BaseCreateDatasourceParams, createDatasource } from '@/api/datasources';
import { Button } from '@/components/ui/button';
import { Form, FormControl, FormDescription, FormField, FormItem, FormLabel, FormMessage } from '@/components/ui/form';
import { Input } from '@/components/ui/input';
import { Separator } from '@/components/ui/separator';
import { Switch } from '@/components/ui/switch';
import { zodResolver } from '@hookform/resolvers/zod';
import { Loader2Icon } from 'lucide-react';
import { useRouter } from 'next/navigation';
import { useTransition } from 'react';
import { useForm } from 'react-hook-form';
import { z, type ZodType } from 'zod';

const schema = z.object({
name: z.string(),
description: z.string(),
build_kg_index: z.boolean(),
url: z.string().url(),
}) satisfies ZodType<BaseCreateDatasourceParams, any, any>;

export default function Page () {
const [navigating, startNavigation] = useTransition();
const router = useRouter();
const form = useForm({
resolver: zodResolver(schema),
defaultValues: {
name: '',
description: '',
build_kg_index: false,
url: '',
},
});

const handleSubmit = form.handleSubmit(async ({ url, ...data }) => {
const createdDatasource = await createDatasource({
...data,
data_source_type: 'web_single_page',
config: { url },
});
startNavigation(() => {
router.push(`/datasources/${createdDatasource.id}`);
});
});

return (
<Form {...form}>
<form id="create-datasource-form" className="space-y-4" onSubmit={handleSubmit}>
<FormField
name="name"
render={({ field }) => (
<FormItem>
<FormLabel>Name</FormLabel>
<FormControl>
<Input {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
name="description"
render={({ field }) => (
<FormItem>
<FormLabel>Description</FormLabel>
<FormControl>
<Input {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
name="url"
render={({ field }) => (
<FormItem>
<FormLabel>URL</FormLabel>
<FormControl>
<Input {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
</form>
<div className="mt-4 space-y-4">
<Separator />
<FormField
name="build_kg_index"
render={({ field }) => (
<FormItem className="rounded-lg border p-4 flex items-center justify-between text-sky-500 bg-sky-500/5 border-sky-500/30">
<div className="space-y-2">
<FormLabel>Build KnowledgeGraph Index</FormLabel>
<FormDescription className="text-sky-500/70">
Enable to build knowledge graph index.
</FormDescription>
</div>
<FormControl>
<Switch {...field} onChange={undefined} checked={field.value} onCheckedChange={field.onChange} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<Button type="submit" disabled={navigating || form.formState.isSubmitting} className="gap-2" form="create-datasource-form">
{(navigating || form.formState.isSubmitting) && <Loader2Icon className="size-4 animate-spin repeat-infinite" />}
<span>Create</span>
</Button>
</div>
</Form>
);
}
Loading

0 comments on commit 960a7ae

Please sign in to comment.