From 34ef573edb4efdb179d28364fd1fc64974e54c20 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 10 Apr 2024 12:31:53 -0700 Subject: [PATCH] Set up basic logic for parsing template -> UI flow and vice versa (#131) Signed-off-by: Tyler Ohlsen --- common/constants.ts | 8 +- common/index.ts | 1 + common/interfaces.ts | 49 +++- common/utils.ts | 212 +-------------- public/component_types/indexer/knn_indexer.ts | 3 + .../transformer/text_embedding_transformer.ts | 3 + public/pages/workflow_detail/utils/index.ts | 7 + public/pages/workflow_detail/utils/utils.ts | 18 ++ .../utils/workflow_to_template_utils.ts | 246 ++++++++++++++++++ .../workspace/resizable_workspace.tsx | 72 +++-- .../workflow_detail/workspace/workspace.tsx | 6 +- .../workflows/new_workflow/new_workflow.tsx | 57 ++-- public/pages/workflows/new_workflow/utils.ts | 158 +++++++++++ public/store/reducers/presets_reducer.ts | 8 +- public/utils/constants.ts | 7 +- .../resources/templates/semantic_search.json | 29 +-- .../templates/start_from_scratch.json | 13 - .../routes/flow_framework_routes_service.ts | 13 +- 18 files changed, 568 insertions(+), 342 deletions(-) create mode 100644 public/pages/workflow_detail/utils/index.ts create mode 100644 public/pages/workflow_detail/utils/utils.ts create mode 100644 public/pages/workflow_detail/utils/workflow_to_template_utils.ts create mode 100644 public/pages/workflows/new_workflow/utils.ts delete mode 100644 server/resources/templates/start_from_scratch.json diff --git a/common/constants.ts b/common/constants.ts index 51cfc328..fb891729 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { WORKFLOW_STATE } from './interfaces'; +import { TemplateNode, WORKFLOW_STATE } from './interfaces'; export const PLUGIN_ID = 'flow-framework'; @@ -47,6 +47,12 @@ export const GET_PRESET_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH export const BASE_MODEL_NODE_API_PATH = `${BASE_NODE_API_PATH}/model`; export const SEARCH_MODELS_NODE_API_PATH = `${BASE_MODEL_NODE_API_PATH}/search`; +/** + * BACKEND INTERFACES + */ +export const CREATE_INGEST_PIPELINE_STEP_TYPE = 'create_ingest_pipeline'; +export const CREATE_INDEX_STEP_TYPE = 'create_index'; + /** * MISCELLANEOUS */ diff --git a/common/index.ts b/common/index.ts index bd00c56a..78615eb2 100644 --- a/common/index.ts +++ b/common/index.ts @@ -8,3 +8,4 @@ export * from './interfaces'; export * from './utils'; export * from '../public/component_types'; export * from '../public/utils'; +export * from '../public/pages/workflow_detail/utils'; diff --git a/common/interfaces.ts b/common/interfaces.ts index eb412f1e..b756ad07 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -29,7 +29,7 @@ type ReactFlowViewport = { }; export type UIState = { - workspaceFlow: WorkspaceFlowState; + workspace_flow: WorkspaceFlowState; }; export type WorkspaceFlowState = { @@ -42,21 +42,56 @@ export type WorkspaceFlowState = { ********** USE CASE TEMPLATE TYPES/INTERFACES ********** */ +export type IngestProcessor = { + description?: string; +}; + +export type TextEmbeddingProcessor = IngestProcessor & { + text_embedding: { + model_id: string; + field_map: {}; + }; +}; + export type TemplateNode = { id: string; type: string; - previous_node_inputs?: Map; - user_inputs?: Map; + previous_node_inputs?: {}; + user_inputs?: {}; +}; + +export type CreateIngestPipelineNode = TemplateNode & { + user_inputs: { + pipeline_id: string; + model_id?: string; + input_field?: string; + output_field?: string; + configurations: { + description?: string; + processors: IngestProcessor[]; + }; + }; +}; + +export type CreateIndexNode = TemplateNode & { + previous_node_inputs?: { + [ingest_pipeline_step_id: string]: string; + }; + user_inputs: { + index_name: string; + configurations: { + settings: {}; + mappings: {}; + }; + }; }; export type TemplateEdge = { source: string; - target: string; + dest: string; }; export type TemplateFlow = { - user_inputs?: Map; - previous_node_inputs?: Map; nodes: TemplateNode[]; edges?: TemplateEdge[]; }; @@ -91,7 +126,7 @@ export type Workflow = WorkflowTemplate & { }; export enum USE_CASE { - PROVISION = 'PROVISION', + SEMANTIC_SEARCH = 'SEMANTIC_SEARCH', } /** diff --git a/common/utils.ts b/common/utils.ts index 0a01b684..6a5b9c0f 100644 --- a/common/utils.ts +++ b/common/utils.ts @@ -4,217 +4,7 @@ */ import moment from 'moment'; -import { MarkerType } from 'reactflow'; -import { - WorkspaceFlowState, - ReactFlowComponent, - initComponentData, - TextEmbeddingTransformer, - KnnIndexer, - generateId, - ReactFlowEdge, - TemplateFlows, - WorkflowTemplate, - DATE_FORMAT_PATTERN, - COMPONENT_CATEGORY, - NODE_CATEGORY, - WorkspaceFormValues, -} from './'; - -// TODO: implement this and remove hardcoded return values -/** - * Given a ReactFlow workspace flow and the set of current form values within such flow, - * generate a backend-compatible set of sub-workflows. - * - */ -export function toTemplateFlows( - workspaceFlow: WorkspaceFlowState, - formValues: WorkspaceFormValues -): TemplateFlows { - const textEmbeddingTransformerNodeId = Object.keys(formValues).find((key) => - key.includes('text_embedding') - ) as string; - const knnIndexerNodeId = Object.keys(formValues).find((key) => - key.includes('knn') - ) as string; - const textEmbeddingFields = formValues[textEmbeddingTransformerNodeId]; - const knnIndexerFields = formValues[knnIndexerNodeId]; - - return { - provision: { - nodes: [ - { - id: 'create_ingest_pipeline', - type: 'create_ingest_pipeline', - user_inputs: { - pipeline_id: 'test-pipeline', - model_id: textEmbeddingFields['modelId'], - input_field: textEmbeddingFields['inputField'], - output_field: textEmbeddingFields['vectorField'], - configurations: { - description: 'A text embedding ingest pipeline', - processors: [ - { - text_embedding: { - model_id: textEmbeddingFields['modelId'], - field_map: { - [textEmbeddingFields['inputField']]: - textEmbeddingFields['vectorField'], - }, - }, - }, - ], - }, - }, - }, - { - id: 'create_index', - type: 'create_index', - previous_node_inputs: { - create_ingest_pipeline: 'pipeline_id', - }, - user_inputs: { - index_name: knnIndexerFields['indexName'], - configurations: { - settings: { - default_pipeline: '${{create_ingest_pipeline.pipeline_id}}', - }, - mappings: { - properties: { - [textEmbeddingFields['vectorField']]: { - type: 'knn_vector', - dimension: 768, - method: { - engine: 'lucene', - space_type: 'l2', - name: 'hnsw', - parameters: {}, - }, - }, - [textEmbeddingFields['inputField']]: { - type: 'text', - }, - }, - }, - }, - }, - }, - ], - }, - }; -} - -// TODO: implement this and remove hardcoded return values -/** - * Converts a backend set of provision/ingest/search sub-workflows into a UI-compatible set of - * ReactFlow nodes and edges - */ -export function toWorkspaceFlow( - templateFlows: TemplateFlows -): WorkspaceFlowState { - const ingestId1 = generateId('text_embedding_processor'); - const ingestId2 = generateId('knn_index'); - const ingestGroupId = generateId(COMPONENT_CATEGORY.INGEST); - const searchGroupId = generateId(COMPONENT_CATEGORY.SEARCH); - const edgeId = generateId('edge'); - - const ingestNodes = [ - { - id: ingestGroupId, - position: { x: 400, y: 400 }, - type: NODE_CATEGORY.INGEST_GROUP, - data: { label: COMPONENT_CATEGORY.INGEST }, - style: { - width: 900, - height: 400, - }, - className: 'reactflow__group-node__ingest', - selectable: true, - deletable: false, - }, - { - id: ingestId1, - position: { x: 100, y: 70 }, - data: initComponentData( - new TextEmbeddingTransformer().toObj(), - ingestId1 - ), - type: NODE_CATEGORY.CUSTOM, - parentNode: ingestGroupId, - extent: 'parent', - draggable: true, - deletable: false, - }, - { - id: ingestId2, - position: { x: 500, y: 70 }, - data: initComponentData(new KnnIndexer().toObj(), ingestId2), - type: NODE_CATEGORY.CUSTOM, - parentNode: ingestGroupId, - extent: 'parent', - draggable: true, - deletable: false, - }, - ] as ReactFlowComponent[]; - - const searchNodes = [ - { - id: searchGroupId, - position: { x: 400, y: 1000 }, - type: NODE_CATEGORY.SEARCH_GROUP, - data: { label: COMPONENT_CATEGORY.SEARCH }, - style: { - width: 900, - height: 400, - }, - className: 'reactflow__group-node__search', - selectable: true, - deletable: false, - }, - ] as ReactFlowComponent[]; - - return { - nodes: [...ingestNodes, ...searchNodes], - edges: [ - { - id: edgeId, - key: edgeId, - source: ingestId1, - target: ingestId2, - markerEnd: { - type: MarkerType.ArrowClosed, - width: 20, - height: 20, - }, - zIndex: 2, - deletable: false, - }, - ] as ReactFlowEdge[], - }; -} - -// TODO: implement this -/** - * Validates the UI workflow state. - * Note we don't have to validate connections since that is done via input/output handlers. - * But we need to validate there are no open connections - */ -export function validateWorkspaceFlow( - workspaceFlow: WorkspaceFlowState -): boolean { - return true; -} - -// TODO: implement this -/** - * Validates the backend template. May be used when parsing persisted templates on server-side, - * or when importing/exporting on the UI. - */ -export function validateWorkflowTemplate( - workflowTemplate: WorkflowTemplate -): boolean { - return true; -} +import { DATE_FORMAT_PATTERN } from './'; export function toFormattedDate(timestampMillis: number): String { return moment(new Date(timestampMillis)).format(DATE_FORMAT_PATTERN); diff --git a/public/component_types/indexer/knn_indexer.ts b/public/component_types/indexer/knn_indexer.ts index faf7b766..d0eba67d 100644 --- a/public/component_types/indexer/knn_indexer.ts +++ b/public/component_types/indexer/knn_indexer.ts @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { COMPONENT_CLASS } from '../../../common'; import { Indexer } from './indexer'; /** @@ -11,8 +12,10 @@ import { Indexer } from './indexer'; export class KnnIndexer extends Indexer { constructor() { super(); + this.type = COMPONENT_CLASS.KNN_INDEXER; this.label = 'K-NN Indexer'; this.description = 'A specialized indexer for K-NN indices'; + this.baseClasses = [...this.baseClasses, this.type]; this.createFields = [ // @ts-ignore ...this.createFields, diff --git a/public/component_types/transformer/text_embedding_transformer.ts b/public/component_types/transformer/text_embedding_transformer.ts index eee236ab..c856381e 100644 --- a/public/component_types/transformer/text_embedding_transformer.ts +++ b/public/component_types/transformer/text_embedding_transformer.ts @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { COMPONENT_CLASS } from '../../../common'; import { MLTransformer } from '.'; /** @@ -11,8 +12,10 @@ import { MLTransformer } from '.'; export class TextEmbeddingTransformer extends MLTransformer { constructor() { super(); + this.type = COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER; this.label = 'Text Embedding Transformer'; this.description = 'A specialized ML transformer for embedding text'; + this.baseClasses = [...this.baseClasses, this.type]; this.inputs = []; this.createFields = [ { diff --git a/public/pages/workflow_detail/utils/index.ts b/public/pages/workflow_detail/utils/index.ts new file mode 100644 index 00000000..91b6465b --- /dev/null +++ b/public/pages/workflow_detail/utils/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './utils'; +export * from './workflow_to_template_utils'; diff --git a/public/pages/workflow_detail/utils/utils.ts b/public/pages/workflow_detail/utils/utils.ts new file mode 100644 index 00000000..fd26a1a8 --- /dev/null +++ b/public/pages/workflow_detail/utils/utils.ts @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { WorkspaceFlowState } from '../../../../common'; + +// TODO: implement this +/** + * Validates the UI workflow state. + * Note we don't have to validate connections since that is done via input/output handlers. + * But we need to validate there are no open connections + */ +export function validateWorkspaceFlow( + workspaceFlow: WorkspaceFlowState +): boolean { + return true; +} diff --git a/public/pages/workflow_detail/utils/workflow_to_template_utils.ts b/public/pages/workflow_detail/utils/workflow_to_template_utils.ts new file mode 100644 index 00000000..3e9ce61f --- /dev/null +++ b/public/pages/workflow_detail/utils/workflow_to_template_utils.ts @@ -0,0 +1,246 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { FormikValues } from 'formik'; +import { + WorkspaceFlowState, + ReactFlowComponent, + TemplateFlows, + NODE_CATEGORY, + TemplateNode, + COMPONENT_CLASS, + CREATE_INGEST_PIPELINE_STEP_TYPE, + CREATE_INDEX_STEP_TYPE, + CreateIngestPipelineNode, + TextEmbeddingProcessor, + componentDataToFormik, + ReactFlowEdge, + CreateIndexNode, + TemplateFlow, + TemplateEdge, +} from '../../../../common'; + +/** + * Given a ReactFlow workspace flow with fully populated input values, + * generate a backend-compatible set of sub-workflows. + * + */ +export function toTemplateFlows( + workspaceFlow: WorkspaceFlowState +): TemplateFlows { + const { ingestNodes, ingestEdges } = getIngestNodesAndEdges( + workspaceFlow.nodes, + workspaceFlow.edges + ); + const provisionFlow = toProvisionTemplateFlow(ingestNodes, ingestEdges); + + // TODO: support beyond provision + return { + provision: provisionFlow, + }; +} + +function getIngestNodesAndEdges( + allNodes: ReactFlowComponent[], + allEdges: ReactFlowEdge[] +): { ingestNodes: ReactFlowComponent[]; ingestEdges: ReactFlowEdge[] } { + const ingestParentId = allNodes.find( + (node) => node.type === NODE_CATEGORY.INGEST_GROUP + )?.id as string; + const ingestNodes = allNodes.filter( + (node) => node.parentNode === ingestParentId + ); + const ingestIds = ingestNodes.map((node) => node.id); + const ingestEdges = allEdges.filter( + (edge) => ingestIds.includes(edge.source) || ingestIds.includes(edge.target) + ); + return { + ingestNodes, + ingestEdges, + }; +} + +// Generates the end-to-end provision subflow, if applicable +function toProvisionTemplateFlow( + nodes: ReactFlowComponent[], + edges: ReactFlowEdge[] +): TemplateFlow { + const prevNodes = [] as ReactFlowComponent[]; + const templateNodes = [] as TemplateNode[]; + const templateEdges = [] as TemplateEdge[]; + nodes.forEach((node) => { + const templateNode = toTemplateNode(node, prevNodes, edges); + // it may be undefined if the node is not convertible for some reason + if (templateNode) { + templateNodes.push(templateNode); + prevNodes.push(node); + } + }); + + edges.forEach((edge) => { + templateEdges.push(toTemplateEdge(edge)); + }); + + return { + nodes: templateNodes, + edges: templateEdges, + }; +} + +function toTemplateNode( + flowNode: ReactFlowComponent, + prevNodes: ReactFlowComponent[], + edges: ReactFlowEdge[] +): TemplateNode | undefined { + if (flowNode.data.baseClasses?.includes(COMPONENT_CLASS.ML_TRANSFORMER)) { + return toIngestPipelineNode(flowNode); + } else if (flowNode.data.baseClasses?.includes(COMPONENT_CLASS.INDEXER)) { + return toIndexerNode(flowNode, prevNodes, edges); + } +} + +function toTemplateEdge(flowEdge: ReactFlowEdge): TemplateEdge { + return { + source: flowEdge.source, + dest: flowEdge.target, + }; +} + +// General fn to process all ML transform nodes. Convert into a final +// ingest pipeline with a processor specific to the final class of the node. +function toIngestPipelineNode( + flowNode: ReactFlowComponent +): CreateIngestPipelineNode { + // TODO a few improvements to make here: + // 1. Consideration of multiple ingest processors and how to collect them all, and finally create + // a single ingest pipeline with all of them, in the same order as done on the UI + // 2. Support more than just text embedding transformers + switch (flowNode.data.type) { + case COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER: + default: { + const { modelId, inputField, vectorField } = componentDataToFormik( + flowNode.data + ); + + return { + id: flowNode.data.id, + type: CREATE_INGEST_PIPELINE_STEP_TYPE, + user_inputs: { + // TODO: expose as customizable + pipeline_id: 'test-pipeline', + model_id: modelId, + input_field: inputField, + output_field: vectorField, + configurations: { + description: 'An ingest pipeline with a text embedding processor.', + processors: [ + { + text_embedding: { + model_id: modelId, + field_map: { + [inputField]: vectorField, + }, + }, + } as TextEmbeddingProcessor, + ], + }, + }, + }; + } + } +} + +// General fn to convert an indexer node to a final CreateIndexNode template node. +function toIndexerNode( + flowNode: ReactFlowComponent, + prevNodes: ReactFlowComponent[], + edges: ReactFlowEdge[] +): CreateIndexNode { + switch (flowNode.data.type) { + case COMPONENT_CLASS.KNN_INDEXER: + default: { + const { indexName } = componentDataToFormik(flowNode.data); + // TODO: remove hardcoded logic here that is assuming each indexer node has + // exactly 1 directly connected create_ingest_pipeline predecessor node that + // contains an inputField and vectorField + const directlyConnectedNodeId = getDirectlyConnectedNodes( + flowNode, + edges + )[0]; + const { inputField, vectorField } = getDirectlyConnectedNodeInputs( + flowNode, + prevNodes, + edges + ); + + return { + id: flowNode.data.id, + type: CREATE_INDEX_STEP_TYPE, + previous_node_inputs: { + [directlyConnectedNodeId]: 'pipeline_id', + }, + user_inputs: { + index_name: indexName, + configurations: { + settings: { + default_pipeline: `\${{${directlyConnectedNodeId}.pipeline_id}}`, + }, + mappings: { + properties: { + [vectorField]: { + type: 'knn_vector', + dimension: 768, + method: { + engine: 'lucene', + space_type: 'l2', + name: 'hnsw', + parameters: {}, + }, + }, + [inputField]: { + type: 'text', + }, + }, + }, + }, + }, + }; + } + } +} + +// Fetch all directly connected predecessor node inputs +function getDirectlyConnectedNodeInputs( + node: ReactFlowComponent, + prevNodes: ReactFlowComponent[], + edges: ReactFlowEdge[] +): FormikValues { + const directlyConnectedNodeIds = getDirectlyConnectedNodes(node, edges); + const directlyConnectedNodes = prevNodes.filter((prevNode) => + directlyConnectedNodeIds.includes(prevNode.id) + ); + let values = {} as FormikValues; + directlyConnectedNodes.forEach((node) => { + values = { + ...values, + ...componentDataToFormik(node.data), + }; + }); + return values; +} + +// Simple utility fn to fetch all direct predecessor node IDs for a given node +function getDirectlyConnectedNodes( + flowNode: ReactFlowComponent, + edges: ReactFlowEdge[] +): string[] { + const incomingNodes = [] as string[]; + edges.forEach((edge) => { + if (edge.target === flowNode.id) { + incomingNodes.push(edge.source); + } + }); + return incomingNodes; +} diff --git a/public/pages/workflow_detail/workspace/resizable_workspace.tsx b/public/pages/workflow_detail/workspace/resizable_workspace.tsx index e5843d8c..5bdebc22 100644 --- a/public/pages/workflow_detail/workspace/resizable_workspace.tsx +++ b/public/pages/workflow_detail/workspace/resizable_workspace.tsx @@ -18,6 +18,8 @@ import { EuiPageHeader, EuiResizableContainer, } from '@elastic/eui'; +import { getCore } from '../../../services'; + import { Workflow, WorkspaceFormValues, @@ -26,17 +28,12 @@ import { WorkspaceSchemaObj, componentDataToFormik, getComponentSchema, - toWorkspaceFlow, - validateWorkspaceFlow, WorkspaceFlowState, - toTemplateFlows, - DEFAULT_NEW_WORKFLOW_NAME, - DEFAULT_NEW_WORKFLOW_DESCRIPTION, - USE_CASE, WORKFLOW_STATE, processNodes, reduceToTemplate, } from '../../../../common'; +import { validateWorkspaceFlow, toTemplateFlows } from '../utils'; import { AppState, createWorkflow, @@ -148,40 +145,33 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { } } - // Hook to update some default values for the workflow, if applicable. Flow state - // may not exist if it is a backend-only-created workflow, or a new, unsaved workflow. - // Metadata fields (name/description/use_case/etc.) may not exist if the user - // cold reloads the page on a new, unsaved workflow. + // Hook to update some default values for the workflow, if applicable. + // We need to handle different scenarios: + // 1. Rendering backend-only-created workflow / an already-created workflow with no ui_metadata. + // In this case, we revert to the home page with a warn toast that we don't support it, for now. + // This is because we initially have guardrails and a static set of readonly nodes/edges that we handle. + // 2. Rendering empty/null workflow, if refreshing the editor page where there is no cached workflow and + // no workflow ID in the URL. + // In this case, revert to home page and a warn toast that we don't support it for now. + // This is because we initially don't support building / drag-and-drop components. + // 3. Rendering a cached workflow via navigation from create workflow tab + // 4. Rendering a created workflow with ui_metadata. + // In these cases, just render what is persisted, no action needed. useEffect(() => { - if (props.workflow) { - let workflowCopy = { ...props.workflow } as Workflow; - if ( - !workflowCopy.ui_metadata || - !workflowCopy.ui_metadata.workspaceFlow - ) { - workflowCopy.ui_metadata = { - ...(workflowCopy.ui_metadata || {}), - workspaceFlow: toWorkspaceFlow(workflowCopy.workflows), - }; - console.debug( - `There is no saved UI flow for workflow: ${workflowCopy.name}. Generating a default one.` + const missingUiFlow = + props.workflow && !props.workflow?.ui_metadata?.workspace_flow; + const missingCachedWorkflow = props.isNewWorkflow && !props.workflow; + if (missingUiFlow || missingCachedWorkflow) { + history.replace('/workflows'); + if (missingCachedWorkflow) { + getCore().notifications.toasts.addWarning('No workflow found'); + } else { + getCore().notifications.toasts.addWarning( + `There is no ui_metadata for workflow: ${props.workflow?.name}` ); } - - // TODO: tune some of the defaults, like use_case and version as these will change - workflowCopy = { - ...workflowCopy, - name: workflowCopy.name || DEFAULT_NEW_WORKFLOW_NAME, - description: - workflowCopy.description || DEFAULT_NEW_WORKFLOW_DESCRIPTION, - use_case: workflowCopy.use_case || USE_CASE.PROVISION, - version: workflowCopy.version || { - template: '1.0.0', - compatibility: ['2.12.0', '3.0.0'], - }, - }; - - setWorkflow(workflowCopy); + } else { + setWorkflow(props.workflow); } }, [props.workflow]); @@ -200,10 +190,10 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { // Initialize the form state to an existing workflow, if applicable. useEffect(() => { - if (workflow?.ui_metadata?.workspaceFlow) { + if (workflow?.ui_metadata?.workspace_flow) { const initFormValues = {} as WorkspaceFormValues; const initSchemaObj = {} as WorkspaceSchemaObj; - workflow.ui_metadata.workspaceFlow.nodes.forEach((node) => { + workflow.ui_metadata.workspace_flow.nodes.forEach((node) => { initFormValues[node.id] = componentDataToFormik(node.data); initSchemaObj[node.id] = getComponentSchema(node.data); }); @@ -280,9 +270,9 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { ...workflow, ui_metadata: { ...workflow?.ui_metadata, - workspaceFlow: curFlowState, + workspace_flow: curFlowState, }, - workflows: toTemplateFlows(curFlowState, formikProps.values), + workflows: toTemplateFlows(curFlowState), } as Workflow; processWorkflowFn(updatedWorkflow); } else { diff --git a/public/pages/workflow_detail/workspace/workspace.tsx b/public/pages/workflow_detail/workspace/workspace.tsx index b85ea94d..e1fea712 100644 --- a/public/pages/workflow_detail/workspace/workspace.tsx +++ b/public/pages/workflow_detail/workspace/workspace.tsx @@ -104,9 +104,9 @@ export function Workspace(props: WorkspaceProps) { // Initialization. Set the nodes and edges to an existing workflow state, useEffect(() => { const workflow = { ...props.workflow }; - if (workflow?.ui_metadata?.workspaceFlow) { - setNodes(workflow.ui_metadata.workspaceFlow.nodes); - setEdges(workflow.ui_metadata.workspaceFlow.edges); + if (workflow?.ui_metadata?.workspace_flow) { + setNodes(workflow.ui_metadata.workspace_flow.nodes); + setEdges(workflow.ui_metadata.workspace_flow.edges); } }, [props.workflow]); diff --git a/public/pages/workflows/new_workflow/new_workflow.tsx b/public/pages/workflows/new_workflow/new_workflow.tsx index 53242d4d..e0ca3036 100644 --- a/public/pages/workflows/new_workflow/new_workflow.tsx +++ b/public/pages/workflows/new_workflow/new_workflow.tsx @@ -14,13 +14,17 @@ import { } from '@elastic/eui'; import { useSelector } from 'react-redux'; import { UseCase } from './use_case'; +import { Workflow, WorkflowTemplate } from '../../../../common'; import { - DEFAULT_NEW_WORKFLOW_NAME, - START_FROM_SCRATCH_WORKFLOW_NAME, - Workflow, -} from '../../../../common'; -import { AppState, cacheWorkflow, useAppDispatch } from '../../../store'; -import { getWorkflowPresets } from '../../../store/reducers'; + AppState, + cacheWorkflow, + useAppDispatch, + getWorkflowPresets, +} from '../../../store'; +import { + enrichPresetWorkflowWithUiMetadata, + processWorkflowName, +} from './utils'; interface NewWorkflowProps {} @@ -31,10 +35,15 @@ interface NewWorkflowProps {} */ export function NewWorkflow(props: NewWorkflowProps) { const dispatch = useAppDispatch(); + + // workflows state const { presetWorkflows, loading } = useSelector( (state: AppState) => state.presets ); - const [filteredWorkflows, setFilteredWorkflows] = useState([]); + const [allWorkflows, setAllWorkflows] = useState([]); + const [filteredWorkflows, setFilteredWorkflows] = useState< + WorkflowTemplate[] + >([]); // search bar state const [searchQuery, setSearchQuery] = useState(''); @@ -47,13 +56,26 @@ export function NewWorkflow(props: NewWorkflowProps) { dispatch(getWorkflowPresets()); }, []); + // initial hook to populate all workflows + // enrich them with dynamically-generated UI flows based on use case useEffect(() => { - setFilteredWorkflows(presetWorkflows); + if (presetWorkflows) { + setAllWorkflows( + presetWorkflows.map((presetWorkflow) => + enrichPresetWorkflowWithUiMetadata(presetWorkflow) + ) + ); + } }, [presetWorkflows]); + // initial hook to populate filtered workflows + useEffect(() => { + setFilteredWorkflows(allWorkflows); + }, [allWorkflows]); + // When search query updated, re-filter preset list useEffect(() => { - setFilteredWorkflows(fetchFilteredWorkflows(presetWorkflows, searchQuery)); + setFilteredWorkflows(fetchFilteredWorkflows(allWorkflows, searchQuery)); }, [searchQuery]); return ( @@ -106,20 +128,3 @@ function fetchFilteredWorkflows( workflow.name.toLowerCase().includes(searchQuery.toLowerCase()) ); } - -// Utility fn to process workflow names from their presentable/readable titles -// on the UI, to a valid name format. -// This leads to less friction if users decide to save the name later on. -function processWorkflowName(workflowName: string): string { - return workflowName === START_FROM_SCRATCH_WORKFLOW_NAME - ? DEFAULT_NEW_WORKFLOW_NAME - : toSnakeCase(workflowName); -} - -function toSnakeCase(text: string): string { - return text - .replace(/\W+/g, ' ') - .split(/ |\B(?=[A-Z])/) - .map((word) => word.toLowerCase()) - .join('_'); -} diff --git a/public/pages/workflows/new_workflow/utils.ts b/public/pages/workflows/new_workflow/utils.ts new file mode 100644 index 00000000..7cc5b4dc --- /dev/null +++ b/public/pages/workflows/new_workflow/utils.ts @@ -0,0 +1,158 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { MarkerType } from 'reactflow'; +import { + WorkspaceFlowState, + ReactFlowComponent, + initComponentData, + TextEmbeddingTransformer, + KnnIndexer, + generateId, + ReactFlowEdge, + COMPONENT_CATEGORY, + NODE_CATEGORY, + USE_CASE, + WorkflowTemplate, + COMPONENT_CLASS, + START_FROM_SCRATCH_WORKFLOW_NAME, + DEFAULT_NEW_WORKFLOW_NAME, +} from '../../../../common'; + +// Fn to produce the complete preset template with all necessary UI metadata. +// Some UI metadata we want to generate on-the-fly using our component classes we have on client-side. +// Thus, we only persist a minimal subset of a full template on server-side. We generate +// the rest dynamically based on the set of supported preset use cases. +export function enrichPresetWorkflowWithUiMetadata( + presetWorkflow: Partial +): WorkflowTemplate { + let workspaceFlowState = {} as WorkspaceFlowState; + switch (presetWorkflow.use_case) { + case USE_CASE.SEMANTIC_SEARCH: { + workspaceFlowState = fetchSemanticSearchWorkspaceFlow(); + break; + } + default: { + workspaceFlowState = fetchEmptyWorkspaceFlow(); + break; + } + } + + return { + ...presetWorkflow, + ui_metadata: { + ...presetWorkflow.ui_metadata, + workspace_flow: workspaceFlowState, + }, + } as WorkflowTemplate; +} + +function fetchEmptyWorkspaceFlow(): WorkspaceFlowState { + return { + nodes: [], + edges: [], + }; +} + +function fetchSemanticSearchWorkspaceFlow(): WorkspaceFlowState { + const ingestId1 = generateId(COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER); + const ingestId2 = generateId(COMPONENT_CLASS.KNN_INDEXER); + const ingestGroupId = generateId(COMPONENT_CATEGORY.INGEST); + const searchGroupId = generateId(COMPONENT_CATEGORY.SEARCH); + const edgeId = generateId('edge'); + + const ingestNodes = [ + { + id: ingestGroupId, + position: { x: 400, y: 400 }, + type: NODE_CATEGORY.INGEST_GROUP, + data: { label: COMPONENT_CATEGORY.INGEST }, + style: { + width: 900, + height: 400, + }, + className: 'reactflow__group-node__ingest', + selectable: true, + draggable: false, + deletable: false, + }, + { + id: ingestId1, + position: { x: 100, y: 70 }, + data: initComponentData( + new TextEmbeddingTransformer().toObj(), + ingestId1 + ), + type: NODE_CATEGORY.CUSTOM, + parentNode: ingestGroupId, + extent: 'parent', + draggable: false, + deletable: false, + }, + { + id: ingestId2, + position: { x: 500, y: 70 }, + data: initComponentData(new KnnIndexer().toObj(), ingestId2), + type: NODE_CATEGORY.CUSTOM, + parentNode: ingestGroupId, + extent: 'parent', + draggable: false, + deletable: false, + }, + ] as ReactFlowComponent[]; + + const searchNodes = [ + { + id: searchGroupId, + position: { x: 400, y: 1000 }, + type: NODE_CATEGORY.SEARCH_GROUP, + data: { label: COMPONENT_CATEGORY.SEARCH }, + style: { + width: 900, + height: 400, + }, + className: 'reactflow__group-node__search', + selectable: true, + draggable: false, + deletable: false, + }, + ] as ReactFlowComponent[]; + + return { + nodes: [...ingestNodes, ...searchNodes], + edges: [ + { + id: edgeId, + key: edgeId, + source: ingestId1, + target: ingestId2, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + zIndex: 2, + deletable: false, + }, + ] as ReactFlowEdge[], + }; +} + +// Utility fn to process workflow names from their presentable/readable titles +// on the UI, to a valid name format. +// This leads to less friction if users decide to save the name later on. +export function processWorkflowName(workflowName: string): string { + return workflowName === START_FROM_SCRATCH_WORKFLOW_NAME + ? DEFAULT_NEW_WORKFLOW_NAME + : toSnakeCase(workflowName); +} + +function toSnakeCase(text: string): string { + return text + .replace(/\W+/g, ' ') + .split(/ |\B(?=[A-Z])/) + .map((word) => word.toLowerCase()) + .join('_'); +} diff --git a/public/store/reducers/presets_reducer.ts b/public/store/reducers/presets_reducer.ts index a0cf7d4f..01321ac4 100644 --- a/public/store/reducers/presets_reducer.ts +++ b/public/store/reducers/presets_reducer.ts @@ -4,14 +4,14 @@ */ import { createAsyncThunk, createSlice } from '@reduxjs/toolkit'; -import { Workflow } from '../../../common'; +import { WorkflowTemplate } from '../../../common'; import { HttpFetchError } from '../../../../../src/core/public'; import { getRouteService } from '../../services'; const initialState = { loading: false, errorMessage: '', - presetWorkflows: [] as Workflow[], + presetWorkflows: [] as Partial[], }; const PRESET_ACTION_PREFIX = 'presets'; @@ -44,7 +44,9 @@ const presetsSlice = createSlice({ state.errorMessage = ''; }) .addCase(getWorkflowPresets.fulfilled, (state, action) => { - state.presetWorkflows = action.payload.workflowTemplates; + state.presetWorkflows = action.payload.workflowTemplates as Partial< + WorkflowTemplate + >[]; state.loading = false; state.errorMessage = ''; }) diff --git a/public/utils/constants.ts b/public/utils/constants.ts index e7f9f800..1c0c4617 100644 --- a/public/utils/constants.ts +++ b/public/utils/constants.ts @@ -38,15 +38,20 @@ export enum NODE_CATEGORY { SEARCH_GROUP = 'searchGroup', } -// TODO: subject to change /** * A base set of component classes / types. */ export enum COMPONENT_CLASS { + // Indexer-related classes INDEXER = 'indexer', + KNN_INDEXER = 'knn_indexer', + // Retriever-related classes RETRIEVER = 'retriever', + // Transformer-related classes TRANSFORMER = 'transformer', JSON_TO_JSON_TRANSFORMER = 'json_to_json_transformer', ML_TRANSFORMER = 'ml_transformer', + TEXT_EMBEDDING_TRANSFORMER = 'text_embedding_transformer', + // Query-related classes QUERY = 'query', } diff --git a/server/resources/templates/semantic_search.json b/server/resources/templates/semantic_search.json index 5f3beee6..83995824 100644 --- a/server/resources/templates/semantic_search.json +++ b/server/resources/templates/semantic_search.json @@ -1,39 +1,12 @@ { "name": "Semantic Search", "description": "This semantic search workflow includes the essential ingestion and search pipelines that covers the most common search use cases.", - "use_case": "PROVISION", + "use_case": "SEMANTIC_SEARCH", "version": { "template": "1.0.0", "compatibility": [ "2.12.0", "3.0.0" ] - }, - "workflows": { - "provision": { - "nodes": [ - { - "id": "create_ingest_pipeline", - "type": "create_ingest_pipeline", - "user_inputs": { - "pipeline_id": "text-embedding-pipeline", - "model_id": "my-model-id", - "configurations": { - "description": "A text embedding pipeline", - "processors": [ - { - "text_embedding": { - "model_id": "${{user_inputs.model_id}}", - "field_map": { - "passage_text": "${{output}}" - } - } - } - ] - } - } - } - ] - } } } \ No newline at end of file diff --git a/server/resources/templates/start_from_scratch.json b/server/resources/templates/start_from_scratch.json deleted file mode 100644 index 7090db2c..00000000 --- a/server/resources/templates/start_from_scratch.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "Start From Scratch", - "description": "Build your workflow from scratch according to your specific use cases. Start by adding components for your ingest or query needs.", - "use_case": "CUSTOM", - "version": { - "template": "1.0.0", - "compatibility": [ - "2.12.0", - "3.0.0" - ] - }, - "workflows": {} -} \ No newline at end of file diff --git a/server/routes/flow_framework_routes_service.ts b/server/routes/flow_framework_routes_service.ts index 29f9f42d..880efc89 100644 --- a/server/routes/flow_framework_routes_service.ts +++ b/server/routes/flow_framework_routes_service.ts @@ -27,7 +27,6 @@ import { Workflow, WorkflowDict, WorkflowTemplate, - validateWorkflowTemplate, } from '../../common'; import { generateCustomError, @@ -330,17 +329,15 @@ export class FlowFrameworkRoutesService { const jsonTemplates = fs .readdirSync(jsonTemplateDir) .filter((file) => path.extname(file) === '.json'); - const workflowTemplates = [] as WorkflowTemplate[]; + const workflowTemplates = [] as Partial[]; jsonTemplates.forEach((jsonTemplate) => { const templateData = fs.readFileSync( path.join(jsonTemplateDir, jsonTemplate) ); - const workflowTemplate = JSON.parse( - templateData.toString() - ) as WorkflowTemplate; - if (validateWorkflowTemplate(workflowTemplate)) { - workflowTemplates.push(workflowTemplate); - } + const workflowTemplate = JSON.parse(templateData.toString()) as Partial< + WorkflowTemplate + >; + workflowTemplates.push(workflowTemplate); }); return res.ok({ body: { workflowTemplates } });