From e4280a120c80ac0fc9264a87a11770470270998f Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 9 Apr 2024 17:25:09 -0700 Subject: [PATCH] Add logic for template -> ui flow conversion Signed-off-by: Tyler Ohlsen --- common/interfaces.ts | 4 +- public/pages/workflow_detail/utils/index.ts | 1 - public/pages/workflow_detail/utils/utils.ts | 13 +--- .../workspace/resizable_workspace.tsx | 74 ++++++++---------- .../workflow_detail/workspace/workspace.tsx | 6 +- .../workflows/new_workflow/new_workflow.tsx | 57 +++++++------- .../new_workflow/utils.ts} | 77 +++++++++++++++---- public/store/reducers/presets_reducer.ts | 8 +- .../resources/templates/semantic_search.json | 29 +------ .../routes/flow_framework_routes_service.ts | 13 ++-- 10 files changed, 143 insertions(+), 139 deletions(-) rename public/pages/{workflow_detail/utils/template_to_workflow_utils.ts => workflows/new_workflow/utils.ts} (52%) diff --git a/common/interfaces.ts b/common/interfaces.ts index bf0f81f0..b756ad07 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -29,7 +29,7 @@ type ReactFlowViewport = { }; export type UIState = { - workspaceFlow: WorkspaceFlowState; + workspace_flow: WorkspaceFlowState; }; export type WorkspaceFlowState = { @@ -126,7 +126,7 @@ export type Workflow = WorkflowTemplate & { }; export enum USE_CASE { - PROVISION = 'PROVISION', + SEMANTIC_SEARCH = 'SEMANTIC_SEARCH', } /** diff --git a/public/pages/workflow_detail/utils/index.ts b/public/pages/workflow_detail/utils/index.ts index 6f086efb..91b6465b 100644 --- a/public/pages/workflow_detail/utils/index.ts +++ b/public/pages/workflow_detail/utils/index.ts @@ -4,5 +4,4 @@ */ export * from './utils'; -export * from './template_to_workflow_utils'; export * from './workflow_to_template_utils'; diff --git a/public/pages/workflow_detail/utils/utils.ts b/public/pages/workflow_detail/utils/utils.ts index 81112dca..fd26a1a8 100644 --- a/public/pages/workflow_detail/utils/utils.ts +++ b/public/pages/workflow_detail/utils/utils.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { WorkspaceFlowState, WorkflowTemplate } from '../../../../common'; +import { WorkspaceFlowState } from '../../../../common'; // TODO: implement this /** @@ -16,14 +16,3 @@ export function validateWorkspaceFlow( ): boolean { return true; } - -// TODO: implement this -/** - * Validates the backend template. May be used when parsing persisted templates on server-side, - * or when importing/exporting on the UI. - */ -export function validateWorkflowTemplate( - workflowTemplate: WorkflowTemplate -): boolean { - return true; -} diff --git a/public/pages/workflow_detail/workspace/resizable_workspace.tsx b/public/pages/workflow_detail/workspace/resizable_workspace.tsx index ae1588a5..5bdebc22 100644 --- a/public/pages/workflow_detail/workspace/resizable_workspace.tsx +++ b/public/pages/workflow_detail/workspace/resizable_workspace.tsx @@ -18,6 +18,8 @@ import { EuiPageHeader, EuiResizableContainer, } from '@elastic/eui'; +import { getCore } from '../../../services'; + import { Workflow, WorkspaceFormValues, @@ -27,18 +29,11 @@ import { componentDataToFormik, getComponentSchema, WorkspaceFlowState, - DEFAULT_NEW_WORKFLOW_NAME, - DEFAULT_NEW_WORKFLOW_DESCRIPTION, - USE_CASE, WORKFLOW_STATE, processNodes, reduceToTemplate, } from '../../../../common'; -import { - toWorkspaceFlow, - validateWorkspaceFlow, - toTemplateFlows, -} from '../utils'; +import { validateWorkspaceFlow, toTemplateFlows } from '../utils'; import { AppState, createWorkflow, @@ -150,40 +145,33 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { } } - // Hook to update some default values for the workflow, if applicable. Flow state - // may not exist if it is a backend-only-created workflow, or a new, unsaved workflow. - // Metadata fields (name/description/use_case/etc.) may not exist if the user - // cold reloads the page on a new, unsaved workflow. + // Hook to update some default values for the workflow, if applicable. + // We need to handle different scenarios: + // 1. Rendering backend-only-created workflow / an already-created workflow with no ui_metadata. + // In this case, we revert to the home page with a warn toast that we don't support it, for now. + // This is because we initially have guardrails and a static set of readonly nodes/edges that we handle. + // 2. Rendering empty/null workflow, if refreshing the editor page where there is no cached workflow and + // no workflow ID in the URL. + // In this case, revert to home page and a warn toast that we don't support it for now. + // This is because we initially don't support building / drag-and-drop components. + // 3. Rendering a cached workflow via navigation from create workflow tab + // 4. Rendering a created workflow with ui_metadata. + // In these cases, just render what is persisted, no action needed. useEffect(() => { - if (props.workflow) { - let workflowCopy = { ...props.workflow } as Workflow; - if ( - !workflowCopy.ui_metadata || - !workflowCopy.ui_metadata.workspaceFlow - ) { - workflowCopy.ui_metadata = { - ...(workflowCopy.ui_metadata || {}), - workspaceFlow: toWorkspaceFlow(workflowCopy.workflows), - }; - console.debug( - `There is no saved UI flow for workflow: ${workflowCopy.name}. Generating a default one.` + const missingUiFlow = + props.workflow && !props.workflow?.ui_metadata?.workspace_flow; + const missingCachedWorkflow = props.isNewWorkflow && !props.workflow; + if (missingUiFlow || missingCachedWorkflow) { + history.replace('/workflows'); + if (missingCachedWorkflow) { + getCore().notifications.toasts.addWarning('No workflow found'); + } else { + getCore().notifications.toasts.addWarning( + `There is no ui_metadata for workflow: ${props.workflow?.name}` ); } - - // TODO: tune some of the defaults, like use_case and version as these will change - workflowCopy = { - ...workflowCopy, - name: workflowCopy.name || DEFAULT_NEW_WORKFLOW_NAME, - description: - workflowCopy.description || DEFAULT_NEW_WORKFLOW_DESCRIPTION, - use_case: workflowCopy.use_case || USE_CASE.PROVISION, - version: workflowCopy.version || { - template: '1.0.0', - compatibility: ['2.12.0', '3.0.0'], - }, - }; - - setWorkflow(workflowCopy); + } else { + setWorkflow(props.workflow); } }, [props.workflow]); @@ -202,10 +190,10 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { // Initialize the form state to an existing workflow, if applicable. useEffect(() => { - if (workflow?.ui_metadata?.workspaceFlow) { + if (workflow?.ui_metadata?.workspace_flow) { const initFormValues = {} as WorkspaceFormValues; const initSchemaObj = {} as WorkspaceSchemaObj; - workflow.ui_metadata.workspaceFlow.nodes.forEach((node) => { + workflow.ui_metadata.workspace_flow.nodes.forEach((node) => { initFormValues[node.id] = componentDataToFormik(node.data); initSchemaObj[node.id] = getComponentSchema(node.data); }); @@ -282,9 +270,9 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { ...workflow, ui_metadata: { ...workflow?.ui_metadata, - workspaceFlow: curFlowState, + workspace_flow: curFlowState, }, - workflows: toTemplateFlows(curFlowState, formikProps.values), + workflows: toTemplateFlows(curFlowState), } as Workflow; processWorkflowFn(updatedWorkflow); } else { diff --git a/public/pages/workflow_detail/workspace/workspace.tsx b/public/pages/workflow_detail/workspace/workspace.tsx index b85ea94d..e1fea712 100644 --- a/public/pages/workflow_detail/workspace/workspace.tsx +++ b/public/pages/workflow_detail/workspace/workspace.tsx @@ -104,9 +104,9 @@ export function Workspace(props: WorkspaceProps) { // Initialization. Set the nodes and edges to an existing workflow state, useEffect(() => { const workflow = { ...props.workflow }; - if (workflow?.ui_metadata?.workspaceFlow) { - setNodes(workflow.ui_metadata.workspaceFlow.nodes); - setEdges(workflow.ui_metadata.workspaceFlow.edges); + if (workflow?.ui_metadata?.workspace_flow) { + setNodes(workflow.ui_metadata.workspace_flow.nodes); + setEdges(workflow.ui_metadata.workspace_flow.edges); } }, [props.workflow]); diff --git a/public/pages/workflows/new_workflow/new_workflow.tsx b/public/pages/workflows/new_workflow/new_workflow.tsx index 53242d4d..e0ca3036 100644 --- a/public/pages/workflows/new_workflow/new_workflow.tsx +++ b/public/pages/workflows/new_workflow/new_workflow.tsx @@ -14,13 +14,17 @@ import { } from '@elastic/eui'; import { useSelector } from 'react-redux'; import { UseCase } from './use_case'; +import { Workflow, WorkflowTemplate } from '../../../../common'; import { - DEFAULT_NEW_WORKFLOW_NAME, - START_FROM_SCRATCH_WORKFLOW_NAME, - Workflow, -} from '../../../../common'; -import { AppState, cacheWorkflow, useAppDispatch } from '../../../store'; -import { getWorkflowPresets } from '../../../store/reducers'; + AppState, + cacheWorkflow, + useAppDispatch, + getWorkflowPresets, +} from '../../../store'; +import { + enrichPresetWorkflowWithUiMetadata, + processWorkflowName, +} from './utils'; interface NewWorkflowProps {} @@ -31,10 +35,15 @@ interface NewWorkflowProps {} */ export function NewWorkflow(props: NewWorkflowProps) { const dispatch = useAppDispatch(); + + // workflows state const { presetWorkflows, loading } = useSelector( (state: AppState) => state.presets ); - const [filteredWorkflows, setFilteredWorkflows] = useState([]); + const [allWorkflows, setAllWorkflows] = useState([]); + const [filteredWorkflows, setFilteredWorkflows] = useState< + WorkflowTemplate[] + >([]); // search bar state const [searchQuery, setSearchQuery] = useState(''); @@ -47,13 +56,26 @@ export function NewWorkflow(props: NewWorkflowProps) { dispatch(getWorkflowPresets()); }, []); + // initial hook to populate all workflows + // enrich them with dynamically-generated UI flows based on use case useEffect(() => { - setFilteredWorkflows(presetWorkflows); + if (presetWorkflows) { + setAllWorkflows( + presetWorkflows.map((presetWorkflow) => + enrichPresetWorkflowWithUiMetadata(presetWorkflow) + ) + ); + } }, [presetWorkflows]); + // initial hook to populate filtered workflows + useEffect(() => { + setFilteredWorkflows(allWorkflows); + }, [allWorkflows]); + // When search query updated, re-filter preset list useEffect(() => { - setFilteredWorkflows(fetchFilteredWorkflows(presetWorkflows, searchQuery)); + setFilteredWorkflows(fetchFilteredWorkflows(allWorkflows, searchQuery)); }, [searchQuery]); return ( @@ -106,20 +128,3 @@ function fetchFilteredWorkflows( workflow.name.toLowerCase().includes(searchQuery.toLowerCase()) ); } - -// Utility fn to process workflow names from their presentable/readable titles -// on the UI, to a valid name format. -// This leads to less friction if users decide to save the name later on. -function processWorkflowName(workflowName: string): string { - return workflowName === START_FROM_SCRATCH_WORKFLOW_NAME - ? DEFAULT_NEW_WORKFLOW_NAME - : toSnakeCase(workflowName); -} - -function toSnakeCase(text: string): string { - return text - .replace(/\W+/g, ' ') - .split(/ |\B(?=[A-Z])/) - .map((word) => word.toLowerCase()) - .join('_'); -} diff --git a/public/pages/workflow_detail/utils/template_to_workflow_utils.ts b/public/pages/workflows/new_workflow/utils.ts similarity index 52% rename from public/pages/workflow_detail/utils/template_to_workflow_utils.ts rename to public/pages/workflows/new_workflow/utils.ts index 2b21f4f3..7cc5b4dc 100644 --- a/public/pages/workflow_detail/utils/template_to_workflow_utils.ts +++ b/public/pages/workflows/new_workflow/utils.ts @@ -12,21 +12,53 @@ import { KnnIndexer, generateId, ReactFlowEdge, - TemplateFlows, COMPONENT_CATEGORY, NODE_CATEGORY, + USE_CASE, + WorkflowTemplate, + COMPONENT_CLASS, + START_FROM_SCRATCH_WORKFLOW_NAME, + DEFAULT_NEW_WORKFLOW_NAME, } from '../../../../common'; -// TODO: implement this and remove hardcoded return values -/** - * Converts a backend set of provision/ingest/search sub-workflows into a UI-compatible set of - * ReactFlow nodes and edges - */ -export function toWorkspaceFlow( - templateFlows: TemplateFlows -): WorkspaceFlowState { - const ingestId1 = generateId('text_embedding_processor'); - const ingestId2 = generateId('knn_index'); +// Fn to produce the complete preset template with all necessary UI metadata. +// Some UI metadata we want to generate on-the-fly using our component classes we have on client-side. +// Thus, we only persist a minimal subset of a full template on server-side. We generate +// the rest dynamically based on the set of supported preset use cases. +export function enrichPresetWorkflowWithUiMetadata( + presetWorkflow: Partial +): WorkflowTemplate { + let workspaceFlowState = {} as WorkspaceFlowState; + switch (presetWorkflow.use_case) { + case USE_CASE.SEMANTIC_SEARCH: { + workspaceFlowState = fetchSemanticSearchWorkspaceFlow(); + break; + } + default: { + workspaceFlowState = fetchEmptyWorkspaceFlow(); + break; + } + } + + return { + ...presetWorkflow, + ui_metadata: { + ...presetWorkflow.ui_metadata, + workspace_flow: workspaceFlowState, + }, + } as WorkflowTemplate; +} + +function fetchEmptyWorkspaceFlow(): WorkspaceFlowState { + return { + nodes: [], + edges: [], + }; +} + +function fetchSemanticSearchWorkspaceFlow(): WorkspaceFlowState { + const ingestId1 = generateId(COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER); + const ingestId2 = generateId(COMPONENT_CLASS.KNN_INDEXER); const ingestGroupId = generateId(COMPONENT_CATEGORY.INGEST); const searchGroupId = generateId(COMPONENT_CATEGORY.SEARCH); const edgeId = generateId('edge'); @@ -43,6 +75,7 @@ export function toWorkspaceFlow( }, className: 'reactflow__group-node__ingest', selectable: true, + draggable: false, deletable: false, }, { @@ -55,7 +88,7 @@ export function toWorkspaceFlow( type: NODE_CATEGORY.CUSTOM, parentNode: ingestGroupId, extent: 'parent', - draggable: true, + draggable: false, deletable: false, }, { @@ -65,7 +98,7 @@ export function toWorkspaceFlow( type: NODE_CATEGORY.CUSTOM, parentNode: ingestGroupId, extent: 'parent', - draggable: true, + draggable: false, deletable: false, }, ] as ReactFlowComponent[]; @@ -82,6 +115,7 @@ export function toWorkspaceFlow( }, className: 'reactflow__group-node__search', selectable: true, + draggable: false, deletable: false, }, ] as ReactFlowComponent[]; @@ -105,3 +139,20 @@ export function toWorkspaceFlow( ] as ReactFlowEdge[], }; } + +// Utility fn to process workflow names from their presentable/readable titles +// on the UI, to a valid name format. +// This leads to less friction if users decide to save the name later on. +export function processWorkflowName(workflowName: string): string { + return workflowName === START_FROM_SCRATCH_WORKFLOW_NAME + ? DEFAULT_NEW_WORKFLOW_NAME + : toSnakeCase(workflowName); +} + +function toSnakeCase(text: string): string { + return text + .replace(/\W+/g, ' ') + .split(/ |\B(?=[A-Z])/) + .map((word) => word.toLowerCase()) + .join('_'); +} diff --git a/public/store/reducers/presets_reducer.ts b/public/store/reducers/presets_reducer.ts index a0cf7d4f..01321ac4 100644 --- a/public/store/reducers/presets_reducer.ts +++ b/public/store/reducers/presets_reducer.ts @@ -4,14 +4,14 @@ */ import { createAsyncThunk, createSlice } from '@reduxjs/toolkit'; -import { Workflow } from '../../../common'; +import { WorkflowTemplate } from '../../../common'; import { HttpFetchError } from '../../../../../src/core/public'; import { getRouteService } from '../../services'; const initialState = { loading: false, errorMessage: '', - presetWorkflows: [] as Workflow[], + presetWorkflows: [] as Partial[], }; const PRESET_ACTION_PREFIX = 'presets'; @@ -44,7 +44,9 @@ const presetsSlice = createSlice({ state.errorMessage = ''; }) .addCase(getWorkflowPresets.fulfilled, (state, action) => { - state.presetWorkflows = action.payload.workflowTemplates; + state.presetWorkflows = action.payload.workflowTemplates as Partial< + WorkflowTemplate + >[]; state.loading = false; state.errorMessage = ''; }) diff --git a/server/resources/templates/semantic_search.json b/server/resources/templates/semantic_search.json index 5f3beee6..83995824 100644 --- a/server/resources/templates/semantic_search.json +++ b/server/resources/templates/semantic_search.json @@ -1,39 +1,12 @@ { "name": "Semantic Search", "description": "This semantic search workflow includes the essential ingestion and search pipelines that covers the most common search use cases.", - "use_case": "PROVISION", + "use_case": "SEMANTIC_SEARCH", "version": { "template": "1.0.0", "compatibility": [ "2.12.0", "3.0.0" ] - }, - "workflows": { - "provision": { - "nodes": [ - { - "id": "create_ingest_pipeline", - "type": "create_ingest_pipeline", - "user_inputs": { - "pipeline_id": "text-embedding-pipeline", - "model_id": "my-model-id", - "configurations": { - "description": "A text embedding pipeline", - "processors": [ - { - "text_embedding": { - "model_id": "${{user_inputs.model_id}}", - "field_map": { - "passage_text": "${{output}}" - } - } - } - ] - } - } - } - ] - } } } \ No newline at end of file diff --git a/server/routes/flow_framework_routes_service.ts b/server/routes/flow_framework_routes_service.ts index 29f9f42d..880efc89 100644 --- a/server/routes/flow_framework_routes_service.ts +++ b/server/routes/flow_framework_routes_service.ts @@ -27,7 +27,6 @@ import { Workflow, WorkflowDict, WorkflowTemplate, - validateWorkflowTemplate, } from '../../common'; import { generateCustomError, @@ -330,17 +329,15 @@ export class FlowFrameworkRoutesService { const jsonTemplates = fs .readdirSync(jsonTemplateDir) .filter((file) => path.extname(file) === '.json'); - const workflowTemplates = [] as WorkflowTemplate[]; + const workflowTemplates = [] as Partial[]; jsonTemplates.forEach((jsonTemplate) => { const templateData = fs.readFileSync( path.join(jsonTemplateDir, jsonTemplate) ); - const workflowTemplate = JSON.parse( - templateData.toString() - ) as WorkflowTemplate; - if (validateWorkflowTemplate(workflowTemplate)) { - workflowTemplates.push(workflowTemplate); - } + const workflowTemplate = JSON.parse(templateData.toString()) as Partial< + WorkflowTemplate + >; + workflowTemplates.push(workflowTemplate); }); return res.ok({ body: { workflowTemplates } });