Skip to content

Commit

Permalink
Support dynamic workflow generation when adding processors (ingest)
Browse files Browse the repository at this point in the history 
Signed-off-by: Tyler Ohlsen <[email protected]>
  • Loading branch information
@ohltyler
ohltyler committed Jun 10, 2024
1 parent 976cab2 commit 44861dd
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 59 deletions.
6 changes: 1 addition & 5 deletions public/pages/workflow_detail/resizable_workspace.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,11 +231,7 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) {
style={{ height: '100%' }}
>
<EuiFlexItem>
<Workspace
id="ingest"
workflow={workflow}
readonly={false}
/>
<Workspace uiConfig={uiConfig} />
</EuiFlexItem>
</EuiFlexGroup>
</EuiResizablePanel>
Expand Down
9 changes: 6 additions & 3 deletions public/pages/workflow_detail/workflow_inputs/ingest_inputs/processors_list.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import { cloneDeep } from 'lodash';
import { useFormikContext } from 'formik';
import {
IConfig,
IModelProcessorConfig,
MODEL_TYPE,
PROCESSOR_TYPE,
WorkflowConfig,
WorkflowFormValues,
Expand All @@ -39,16 +41,19 @@ export function ProcessorsList(props: ProcessorsListProps) {
// Adding a processor to the config. Fetch the existing one
// (getting any updated/interim values along the way) and add to
// the list of processors
// TODO: enhance this to either choose from a list of preset
// processors, or at the least a usable generic processor
function addProcessor(processorIdToAdd: string): void {
const existingConfig = cloneDeep(props.uiConfig as WorkflowConfig);
let newConfig = formikToUiConfig(values, existingConfig);
newConfig.ingest.enrich.processors = [
...newConfig.ingest.enrich.processors,
{
type: PROCESSOR_TYPE.MODEL,
modelType: MODEL_TYPE.TEXT_EMBEDDING,
id: processorIdToAdd,
fields: [],
},
} as IModelProcessorConfig,
];
props.setUiConfig(newConfig);
}
Expand Down Expand Up @@ -104,8 +109,6 @@ export function ProcessorsList(props: ProcessorsListProps) {
<div>
<EuiButton
onClick={() => {
// TODO: enhance this to either choose from a list of preset
// processors, or at the least a usable generic processor
addProcessor(generateId('test-processor'));
}}
>
Expand Down
33 changes: 14 additions & 19 deletions public/pages/workflow_detail/workspace/workspace.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import ReactFlow, {
} from 'reactflow';
import { EuiFlexItem, EuiFlexGroup } from '@elastic/eui';
import { setDirty, useAppDispatch } from '../../../store';
import { IComponentData, Workflow, WorkflowConfig } from '../../../../common';
import { IComponentData, WorkflowConfig } from '../../../../common';
import {
IngestGroupComponent,
SearchGroupComponent,
Expand All @@ -31,9 +31,7 @@ import './workspace-styles.scss';
import './workspace_edge/deletable-edge-styles.scss';

interface WorkspaceProps {
workflow?: Workflow;
readonly: boolean;
id: string;
uiConfig?: WorkflowConfig;
}

const nodeTypes = {
Expand Down Expand Up @@ -77,15 +75,12 @@ export function Workspace(props: WorkspaceProps) {

// Initialization. Generate the nodes and edges based on the workflow config.
useEffect(() => {
const workflow = { ...props.workflow };
if (workflow?.ui_metadata?.config) {
const proposedWorkspaceFlow = uiConfigToWorkspaceFlow(
workflow.ui_metadata?.config as WorkflowConfig
);
if (props.uiConfig) {
const proposedWorkspaceFlow = uiConfigToWorkspaceFlow(props.uiConfig);
setNodes(proposedWorkspaceFlow.nodes);
setEdges(proposedWorkspaceFlow.edges);
}
}, [props.workflow]);
}, [props.uiConfig]);

return (
<EuiFlexGroup
Expand All @@ -101,7 +96,7 @@ export function Workspace(props: WorkspaceProps) {
<div className="reactflow-parent-wrapper">
<div className="reactflow-wrapper" ref={reactFlowWrapper}>
<ReactFlow
id={props.id}
id="workspace"
nodes={nodes}
edges={edges}
nodeTypes={nodeTypes}
Expand All @@ -113,14 +108,14 @@ export function Workspace(props: WorkspaceProps) {
className="reactflow-workspace"
fitView
minZoom={0.2}
edgesUpdatable={!props.readonly}
edgesFocusable={!props.readonly}
nodesDraggable={!props.readonly}
nodesConnectable={!props.readonly}
nodesFocusable={!props.readonly}
draggable={!props.readonly}
panOnDrag={!props.readonly}
elementsSelectable={!props.readonly}
edgesUpdatable={false}
edgesFocusable={false}
nodesDraggable={false}
nodesConnectable={false}
nodesFocusable={false}
draggable={true}
panOnDrag={true}
elementsSelectable={false}
>
<Controls
showFitView={false}
Expand Down
93 changes: 61 additions & 32 deletions public/utils/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,11 @@ export function getStateOptions(): EuiFilterSelectItem[] {
**************** ReactFlow workspace utils **********************
*/

const PARENT_NODE_HEIGHT = 350;
const NODE_HEIGHT_Y = 70;
const NODE_WIDTH = 300; // based off of the value set in reactflow-styles.scss
const NODE_SPACING = 100; // the margin (in # pixels) between the components

export function uiConfigToWorkspaceFlow(
config: WorkflowConfig
): WorkspaceFlowState {
Expand All @@ -351,6 +356,15 @@ export function uiConfigToWorkspaceFlow(
};
}

// Helper fn for determining the ingest parent width, based on the number of processors and the specified
// spacing/margin between nodes
function generateIngestParentWidth(ingestConfig: IngestConfig): number {
return (
(ingestConfig.enrich.processors.length + 2) * (NODE_WIDTH + NODE_SPACING) +
NODE_SPACING
);
}

function ingestConfigToWorkspaceFlow(
ingestConfig: IngestConfig
): WorkspaceFlowState {
Expand All @@ -364,8 +378,8 @@ function ingestConfigToWorkspaceFlow(
type: NODE_CATEGORY.INGEST_GROUP,
data: { label: COMPONENT_CATEGORY.INGEST },
style: {
width: 1300,
height: 400,
width: generateIngestParentWidth(ingestConfig),
height: PARENT_NODE_HEIGHT,
},
className: 'reactflow__group-node__ingest',
} as ReactFlowComponent;
Expand All @@ -385,7 +399,10 @@ function ingestConfigToWorkspaceFlow(
const indexNodeId = generateId(COMPONENT_CLASS.KNN_INDEXER);
const indexNode = {
id: indexNodeId,
position: { x: 900, y: 70 },
position: {
x: parentNode.style.width - (NODE_WIDTH + NODE_SPACING),
y: NODE_HEIGHT_Y,
},
data: initComponentData(new KnnIndexer().toObj(), indexNodeId),
type: NODE_CATEGORY.CUSTOM,
parentNode: parentNode.id,
Expand All @@ -411,45 +428,57 @@ function ingestConfigToWorkspaceFlow(
};
}

// TODO: support non-model-type processor configs
function enrichConfigToWorkspaceFlow(
enrichConfig: EnrichConfig,
parentNodeId: string
): WorkspaceFlowState {
const nodes = [] as ReactFlowComponent[];
const edges = [] as ReactFlowEdge[];

// TODO: few assumptions are made here, such as there will always be
// a single model-related processor. In the future make this more flexible and generic.
const modelProcessorConfig = enrichConfig.processors.find(
let xPosition = NODE_WIDTH + NODE_SPACING * 2; // node padding + (width of doc node) + node padding
let prevNodeId = undefined as string | undefined;

const modelProcessorConfigs = enrichConfig.processors.filter(
(processorConfig) => processorConfig.type === PROCESSOR_TYPE.MODEL
) as IModelProcessorConfig;

let transformer = {} as MLTransformer;
let transformerNodeId = '';
switch (modelProcessorConfig.modelType) {
case MODEL_TYPE.TEXT_EMBEDDING: {
transformer = new TextEmbeddingTransformer();
transformerNodeId = generateId(
COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER
);
break;
) as IModelProcessorConfig[];

modelProcessorConfigs.forEach((modelProcessorConfig) => {
let transformer = {} as MLTransformer;
let transformerNodeId = '';
switch (modelProcessorConfig.modelType) {
case MODEL_TYPE.TEXT_EMBEDDING: {
transformer = new TextEmbeddingTransformer();
transformerNodeId = generateId(
COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER
);
break;
}
case MODEL_TYPE.SPARSE_ENCODER: {
transformer = new SparseEncoderTransformer();
transformerNodeId = generateId(
COMPONENT_CLASS.SPARSE_ENCODER_TRANSFORMER
);
break;
}
}
case MODEL_TYPE.SPARSE_ENCODER: {
transformer = new SparseEncoderTransformer();
transformerNodeId = generateId(
COMPONENT_CLASS.SPARSE_ENCODER_TRANSFORMER

nodes.push({
id: transformerNodeId,
position: { x: xPosition, y: NODE_HEIGHT_Y },
data: initComponentData(transformer, transformerNodeId),
type: NODE_CATEGORY.CUSTOM,
parentNode: parentNodeId,
extent: 'parent',
});
xPosition += NODE_SPACING + NODE_WIDTH;

if (prevNodeId) {
edges.push(
generateReactFlowEdge(generateId('edge'), prevNodeId, transformerNodeId)
);
break;
}
}

nodes.push({
id: transformerNodeId,
position: { x: 500, y: 70 },
data: initComponentData(transformer, transformerNodeId),
type: NODE_CATEGORY.CUSTOM,
parentNode: parentNodeId,
extent: 'parent',
prevNodeId = transformerNodeId;
});
return {
nodes,
Expand Down Expand Up @@ -505,7 +534,7 @@ function searchConfigToWorkspaceFlow(
data: { label: COMPONENT_CATEGORY.SEARCH },
style: {
width: 1300,
height: 400,
height: PARENT_NODE_HEIGHT,
},
className: 'reactflow__group-node__search',
} as ReactFlowComponent;
Expand Down

0 comments on commit 44861dd

Please sign in to comment.