From 44861ddae6d45606b2200a2dcb12d875544c2f56 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 10 Jun 2024 12:16:01 -0700 Subject: [PATCH] Support dynamic workflow generation when adding processors (ingest) Signed-off-by: Tyler Ohlsen --- .../workflow_detail/resizable_workspace.tsx | 6 +- .../ingest_inputs/processors_list.tsx | 9 +- .../workflow_detail/workspace/workspace.tsx | 33 +++---- public/utils/utils.ts | 93 ++++++++++++------- 4 files changed, 82 insertions(+), 59 deletions(-) diff --git a/public/pages/workflow_detail/resizable_workspace.tsx b/public/pages/workflow_detail/resizable_workspace.tsx index 4d888849..e28bcc13 100644 --- a/public/pages/workflow_detail/resizable_workspace.tsx +++ b/public/pages/workflow_detail/resizable_workspace.tsx @@ -231,11 +231,7 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { style={{ height: '100%' }} > - + diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/processors_list.tsx index 1a9d7f04..9f70c111 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/processors_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/processors_list.tsx @@ -17,6 +17,8 @@ import { cloneDeep } from 'lodash'; import { useFormikContext } from 'formik'; import { IConfig, + IModelProcessorConfig, + MODEL_TYPE, PROCESSOR_TYPE, WorkflowConfig, WorkflowFormValues, @@ -39,6 +41,8 @@ export function ProcessorsList(props: ProcessorsListProps) { // Adding a processor to the config. Fetch the existing one // (getting any updated/interim values along the way) and add to // the list of processors + // TODO: enhance this to either choose from a list of preset + // processors, or at the least a usable generic processor function addProcessor(processorIdToAdd: string): void { const existingConfig = cloneDeep(props.uiConfig as WorkflowConfig); let newConfig = formikToUiConfig(values, existingConfig); @@ -46,9 +50,10 @@ export function ProcessorsList(props: ProcessorsListProps) { ...newConfig.ingest.enrich.processors, { type: PROCESSOR_TYPE.MODEL, + modelType: MODEL_TYPE.TEXT_EMBEDDING, id: processorIdToAdd, fields: [], - }, + } as IModelProcessorConfig, ]; props.setUiConfig(newConfig); } @@ -104,8 +109,6 @@ export function ProcessorsList(props: ProcessorsListProps) {
{ - // TODO: enhance this to either choose from a list of preset - // processors, or at the least a usable generic processor addProcessor(generateId('test-processor')); }} > diff --git a/public/pages/workflow_detail/workspace/workspace.tsx b/public/pages/workflow_detail/workspace/workspace.tsx index 031f34b8..59534a29 100644 --- a/public/pages/workflow_detail/workspace/workspace.tsx +++ b/public/pages/workflow_detail/workspace/workspace.tsx @@ -15,7 +15,7 @@ import ReactFlow, { } from 'reactflow'; import { EuiFlexItem, EuiFlexGroup } from '@elastic/eui'; import { setDirty, useAppDispatch } from '../../../store'; -import { IComponentData, Workflow, WorkflowConfig } from '../../../../common'; +import { IComponentData, WorkflowConfig } from '../../../../common'; import { IngestGroupComponent, SearchGroupComponent, @@ -31,9 +31,7 @@ import './workspace-styles.scss'; import './workspace_edge/deletable-edge-styles.scss'; interface WorkspaceProps { - workflow?: Workflow; - readonly: boolean; - id: string; + uiConfig?: WorkflowConfig; } const nodeTypes = { @@ -77,15 +75,12 @@ export function Workspace(props: WorkspaceProps) { // Initialization. Generate the nodes and edges based on the workflow config. useEffect(() => { - const workflow = { ...props.workflow }; - if (workflow?.ui_metadata?.config) { - const proposedWorkspaceFlow = uiConfigToWorkspaceFlow( - workflow.ui_metadata?.config as WorkflowConfig - ); + if (props.uiConfig) { + const proposedWorkspaceFlow = uiConfigToWorkspaceFlow(props.uiConfig); setNodes(proposedWorkspaceFlow.nodes); setEdges(proposedWorkspaceFlow.edges); } - }, [props.workflow]); + }, [props.uiConfig]); return (
processorConfig.type === PROCESSOR_TYPE.MODEL - ) as IModelProcessorConfig; - - let transformer = {} as MLTransformer; - let transformerNodeId = ''; - switch (modelProcessorConfig.modelType) { - case MODEL_TYPE.TEXT_EMBEDDING: { - transformer = new TextEmbeddingTransformer(); - transformerNodeId = generateId( - COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER - ); - break; + ) as IModelProcessorConfig[]; + + modelProcessorConfigs.forEach((modelProcessorConfig) => { + let transformer = {} as MLTransformer; + let transformerNodeId = ''; + switch (modelProcessorConfig.modelType) { + case MODEL_TYPE.TEXT_EMBEDDING: { + transformer = new TextEmbeddingTransformer(); + transformerNodeId = generateId( + COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER + ); + break; + } + case MODEL_TYPE.SPARSE_ENCODER: { + transformer = new SparseEncoderTransformer(); + transformerNodeId = generateId( + COMPONENT_CLASS.SPARSE_ENCODER_TRANSFORMER + ); + break; + } } - case MODEL_TYPE.SPARSE_ENCODER: { - transformer = new SparseEncoderTransformer(); - transformerNodeId = generateId( - COMPONENT_CLASS.SPARSE_ENCODER_TRANSFORMER + + nodes.push({ + id: transformerNodeId, + position: { x: xPosition, y: NODE_HEIGHT_Y }, + data: initComponentData(transformer, transformerNodeId), + type: NODE_CATEGORY.CUSTOM, + parentNode: parentNodeId, + extent: 'parent', + }); + xPosition += NODE_SPACING + NODE_WIDTH; + + if (prevNodeId) { + edges.push( + generateReactFlowEdge(generateId('edge'), prevNodeId, transformerNodeId) ); - break; } - } - - nodes.push({ - id: transformerNodeId, - position: { x: 500, y: 70 }, - data: initComponentData(transformer, transformerNodeId), - type: NODE_CATEGORY.CUSTOM, - parentNode: parentNodeId, - extent: 'parent', + prevNodeId = transformerNodeId; }); return { nodes, @@ -505,7 +534,7 @@ function searchConfigToWorkspaceFlow( data: { label: COMPONENT_CATEGORY.SEARCH }, style: { width: 1300, - height: 400, + height: PARENT_NODE_HEIGHT, }, className: 'reactflow__group-node__search', } as ReactFlowComponent;