Skip to content

Commit

Permalink
Add server-side workflow template library (#112)
Browse files Browse the repository at this point in the history
Signed-off-by: Tyler Ohlsen <[email protected]>
(cherry picked from commit 83835c2)
  • Loading branch information
ohltyler authored and github-actions[bot] committed Mar 25, 2024
1 parent 052b416 commit d8ab5b5
Show file tree
Hide file tree
Showing 19 changed files with 347 additions and 198 deletions.
1 change: 1 addition & 0 deletions common/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ export const SEARCH_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/se
export const GET_WORKFLOW_STATE_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/state`;
export const CREATE_WORKFLOW_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/create`;
export const DELETE_WORKFLOW_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/delete`;
export const GET_PRESET_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/presets`;

/**
* MISCELLANEOUS
Expand Down
1 change: 1 addition & 0 deletions common/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,6 @@

export * from './constants';
export * from './interfaces';
export * from './utils';
export * from '../public/component_types';
export * from '../public/utils';
29 changes: 14 additions & 15 deletions common/interfaces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,9 @@ export type WorkspaceFlowState = {

export type TemplateNode = {
id: string;
inputs: {};
type: string;
previous_node_inputs?: Map<string, any>;
user_inputs?: Map<string, any>;
};

export type TemplateEdge = {
Expand All @@ -49,32 +51,30 @@ export type TemplateEdge = {
};

export type TemplateFlow = {
userParams: {};
user_params?: Map<string, any>;
nodes: TemplateNode[];
edges: TemplateEdge[];
edges?: TemplateEdge[];
};

export type TemplateFlows = {
provision: TemplateFlow;
ingest: TemplateFlow;
query: TemplateFlow;
};

export type UseCaseTemplate = {
type: string;
// A stateless template of a workflow
export type WorkflowTemplate = {
name: string;
description: string;
userInputs: {};
use_case: USE_CASE;
// TODO: finalize on version type when that is implemented
// https://github.com/opensearch-project/flow-framework/issues/526
version: any;
workflows: TemplateFlows;
};

export type Workflow = {
// An instance of a workflow based on a workflow template
export type Workflow = WorkflowTemplate & {
// won't exist until created in backend
id?: string;
name: string;
useCase: string;
template: UseCaseTemplate;
description?: string;
// ReactFlow state may not exist if a workflow is created via API/backend-only.
workspaceFlowState?: WorkspaceFlowState;
// won't exist until created in backend
Expand All @@ -86,8 +86,7 @@ export type Workflow = {
};

export enum USE_CASE {
SEMANTIC_SEARCH = 'semantic_search',
CUSTOM = 'custom',
PROVISION = 'PROVISION',
}

/**
Expand Down
93 changes: 93 additions & 0 deletions common/utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

import {
WorkspaceFlowState,
ReactFlowComponent,
initComponentData,
TextEmbeddingTransformer,
KnnIndexer,
generateId,
ReactFlowEdge,
TemplateFlows,
WorkflowTemplate,
} from './';

// TODO: implement this and remove hardcoded return values
/**
* Converts a ReactFlow workspace flow to a backend-compatible set of ingest and/or search sub-workflows,
* along with a provision sub-workflow if resources are to be created.
*/
export function toTemplateFlows(
workspaceFlow: WorkspaceFlowState
): TemplateFlows {
return {
provision: {
user_params: {} as Map<string, any>,
nodes: [],
edges: [],
},
};
}

// TODO: implement this and remove hardcoded return values
/**
* Converts a backend set of provision/ingest/search sub-workflows into a UI-compatible set of
* ReactFlow nodes and edges
*/
export function toWorkspaceFlow(
templateFlows: TemplateFlows
): WorkspaceFlowState {
const id1 = generateId('text_embedding_processor');
const id2 = generateId('text_embedding_processor');
const id3 = generateId('knn_index');
const dummyNodes = [
{
id: id1,
position: { x: 0, y: 500 },
data: initComponentData(new TextEmbeddingTransformer().toObj(), id1),
type: 'customComponent',
},
{
id: id2,
position: { x: 0, y: 200 },
data: initComponentData(new TextEmbeddingTransformer().toObj(), id2),
type: 'customComponent',
},
{
id: id3,
position: { x: 500, y: 500 },
data: initComponentData(new KnnIndexer().toObj(), id3),
type: 'customComponent',
},
] as ReactFlowComponent[];

return {
nodes: dummyNodes,
edges: [] as ReactFlowEdge[],
};
}

// TODO: implement this
/**
* Validates the UI workflow state.
* Note we don't have to validate connections since that is done via input/output handlers.
*/
export function validateWorkspaceFlow(
workspaceFlow: WorkspaceFlowState
): boolean {
return true;
}

// TODO: implement this
/**
* Validates the backend template. May be used when parsing persisted templates on server-side,
* or when importing/exporting on the UI.
*/
export function validateWorkflowTemplate(
workflowTemplate: WorkflowTemplate
): boolean {
return true;
}
44 changes: 4 additions & 40 deletions public/pages/workflow_detail/utils/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@

import {
WorkspaceFlowState,
UseCaseTemplate,
Workflow,
USE_CASE,
ReactFlowComponent,
toTemplateFlows,
validateWorkspaceFlow,
} from '../../../../common';

export function saveWorkflow(workflow: Workflow, rfInstance: any): void {
Expand All @@ -19,12 +19,12 @@ export function saveWorkflow(workflow: Workflow, rfInstance: any): void {
nodes: processNodes(curFlowState.nodes),
};

const isValid = validateFlowState(curFlowState);
const isValid = validateWorkspaceFlow(curFlowState);
if (isValid) {
const updatedWorkflow = {
...workflow,
workspaceFlowState: curFlowState,
template: generateUseCaseTemplate(curFlowState),
workflows: toTemplateFlows(curFlowState),
} as Workflow;
if (workflow.id) {
// TODO: implement connection to update workflow API
Expand All @@ -36,42 +36,6 @@ export function saveWorkflow(workflow: Workflow, rfInstance: any): void {
}
}

// TODO: implement this. Need more info on UX side to finalize what we need
// to persist, what validation to do, etc.
// Note we don't have to validate connections since that is done via input/output handlers.
function validateFlowState(flowState: WorkspaceFlowState): boolean {
return true;
}

// TODO: implement this
function generateUseCaseTemplate(
flowState: WorkspaceFlowState
): UseCaseTemplate {
return {
name: 'example-name',
description: 'example description',
type: USE_CASE.SEMANTIC_SEARCH,
userInputs: {},
workflows: {
provision: {
userParams: {},
nodes: [],
edges: [],
},
ingest: {
userParams: {},
nodes: [],
edges: [],
},
query: {
userParams: {},
nodes: [],
edges: [],
},
},
} as UseCaseTemplate;
}

// Process the raw ReactFlow nodes to only persist the fields we need
function processNodes(nodes: ReactFlowComponent[]): ReactFlowComponent[] {
return nodes
Expand Down
11 changes: 9 additions & 2 deletions public/pages/workflow_detail/workflow_detail.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,15 @@ export function WorkflowDetail(props: WorkflowDetailProps) {
// TODO: can optimize to only fetch a single workflow
dispatch(searchWorkflows({ query: { match_all: {} } }));
}
window.onbeforeunload = (e) =>
isDirty || isNewWorkflow ? true : undefined;

// TODO: below has the following issue:
// 1. user starts to create new unsaved workflow changes
// 2. user navigates to other parts of the plugin without refreshing - no warning happens
// 3. user refreshes at any later time: if isDirty is still true, shows browser warning
// tune to only handle the check if still on the workflow details page, or consider adding a check / warning
// if navigating away from the details page without refreshing (where it is currently not being triggered)
// window.onbeforeunload = (e) =>
// isDirty || isNewWorkflow ? true : undefined;
}, []);

const tabs = [
Expand Down
19 changes: 11 additions & 8 deletions public/pages/workflow_detail/workspace/workspace.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ import {
IComponentData,
ReactFlowComponent,
Workflow,
toWorkspaceFlow,
} from '../../../../common';
import { generateId, initComponentData } from '../../../utils';
import { getCore } from '../../../services';
import { WorkspaceComponent } from '../workspace_component';
import { DeletableEdge } from '../workspace_edge';

Expand Down Expand Up @@ -119,16 +119,19 @@ export function Workspace(props: WorkspaceProps) {
// Initialization. Set the nodes and edges to an existing workflow,
// if applicable.
useEffect(() => {
const workflow = props.workflow;
const workflow = { ...props.workflow };
if (workflow) {
if (workflow.workspaceFlowState) {
setNodes(workflow.workspaceFlowState.nodes);
setEdges(workflow.workspaceFlowState.edges);
} else {
getCore().notifications.toasts.addWarning(
`There is no configured UI flow for workflow: ${workflow.name}`
if (!workflow.workspaceFlowState) {
// No existing workspace state. This could be due to it being a backend-only-created
// workflow, or a new, unsaved workflow
// @ts-ignore
workflow.workspaceFlowState = toWorkspaceFlow(workflow.workflows);
console.debug(
`There is no saved UI flow for workflow: ${workflow.name}. Generating a default one.`
);
}
setNodes(workflow.workspaceFlowState.nodes);
setEdges(workflow.workspaceFlowState.edges);
}
}, [props.workflow]);

Expand Down
Loading

0 comments on commit d8ab5b5

Please sign in to comment.