From 83835c2e5da3b72bd89f2f36c559d715f755bf3b Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 25 Mar 2024 09:41:44 -0700 Subject: [PATCH] Add server-side workflow template library (#112) Signed-off-by: Tyler Ohlsen --- common/constants.ts | 1 + common/index.ts | 1 + common/interfaces.ts | 29 +++--- common/utils.ts | 93 +++++++++++++++++++ public/pages/workflow_detail/utils/utils.ts | 44 +-------- .../pages/workflow_detail/workflow_detail.tsx | 11 ++- .../workflow_detail/workspace/workspace.tsx | 19 ++-- .../workflows/new_workflow/new_workflow.tsx | 69 ++++++++------ .../pages/workflows/new_workflow/presets.tsx | 63 ------------- .../pages/workflows/workflow_list/columns.tsx | 2 +- public/route_service.ts | 12 +++ public/store/reducers/index.ts | 1 + public/store/reducers/presets_reducer.ts | 58 ++++++++++++ public/store/reducers/workflows_reducer.ts | 38 +------- public/store/store.ts | 2 + .../resources/templates/semantic_search.json | 39 ++++++++ .../templates/start_from_scratch.json | 13 +++ .../routes/flow_framework_routes_service.ts | 44 +++++++++ server/routes/helpers.ts | 6 +- 19 files changed, 347 insertions(+), 198 deletions(-) create mode 100644 common/utils.ts delete mode 100644 public/pages/workflows/new_workflow/presets.tsx create mode 100644 public/store/reducers/presets_reducer.ts create mode 100644 server/resources/templates/semantic_search.json create mode 100644 server/resources/templates/start_from_scratch.json diff --git a/common/constants.ts b/common/constants.ts index 77d1cf52..20a7a2a7 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -29,6 +29,7 @@ export const SEARCH_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/se export const GET_WORKFLOW_STATE_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/state`; export const CREATE_WORKFLOW_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/create`; export const DELETE_WORKFLOW_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/delete`; +export const GET_PRESET_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/presets`; /** * MISCELLANEOUS diff --git a/common/index.ts b/common/index.ts index a78ce423..bd00c56a 100644 --- a/common/index.ts +++ b/common/index.ts @@ -5,5 +5,6 @@ export * from './constants'; export * from './interfaces'; +export * from './utils'; export * from '../public/component_types'; export * from '../public/utils'; diff --git a/common/interfaces.ts b/common/interfaces.ts index a927f0e8..b559d47e 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -40,7 +40,9 @@ export type WorkspaceFlowState = { export type TemplateNode = { id: string; - inputs: {}; + type: string; + previous_node_inputs?: Map; + user_inputs?: Map; }; export type TemplateEdge = { @@ -49,32 +51,30 @@ export type TemplateEdge = { }; export type TemplateFlow = { - userParams: {}; + user_params?: Map; nodes: TemplateNode[]; - edges: TemplateEdge[]; + edges?: TemplateEdge[]; }; export type TemplateFlows = { provision: TemplateFlow; - ingest: TemplateFlow; - query: TemplateFlow; }; -export type UseCaseTemplate = { - type: string; +// A stateless template of a workflow +export type WorkflowTemplate = { name: string; description: string; - userInputs: {}; + use_case: USE_CASE; + // TODO: finalize on version type when that is implemented + // https://github.com/opensearch-project/flow-framework/issues/526 + version: any; workflows: TemplateFlows; }; -export type Workflow = { +// An instance of a workflow based on a workflow template +export type Workflow = WorkflowTemplate & { // won't exist until created in backend id?: string; - name: string; - useCase: string; - template: UseCaseTemplate; - description?: string; // ReactFlow state may not exist if a workflow is created via API/backend-only. workspaceFlowState?: WorkspaceFlowState; // won't exist until created in backend @@ -86,8 +86,7 @@ export type Workflow = { }; export enum USE_CASE { - SEMANTIC_SEARCH = 'semantic_search', - CUSTOM = 'custom', + PROVISION = 'PROVISION', } /** diff --git a/common/utils.ts b/common/utils.ts new file mode 100644 index 00000000..fabdb327 --- /dev/null +++ b/common/utils.ts @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + WorkspaceFlowState, + ReactFlowComponent, + initComponentData, + TextEmbeddingTransformer, + KnnIndexer, + generateId, + ReactFlowEdge, + TemplateFlows, + WorkflowTemplate, +} from './'; + +// TODO: implement this and remove hardcoded return values +/** + * Converts a ReactFlow workspace flow to a backend-compatible set of ingest and/or search sub-workflows, + * along with a provision sub-workflow if resources are to be created. + */ +export function toTemplateFlows( + workspaceFlow: WorkspaceFlowState +): TemplateFlows { + return { + provision: { + user_params: {} as Map, + nodes: [], + edges: [], + }, + }; +} + +// TODO: implement this and remove hardcoded return values +/** + * Converts a backend set of provision/ingest/search sub-workflows into a UI-compatible set of + * ReactFlow nodes and edges + */ +export function toWorkspaceFlow( + templateFlows: TemplateFlows +): WorkspaceFlowState { + const id1 = generateId('text_embedding_processor'); + const id2 = generateId('text_embedding_processor'); + const id3 = generateId('knn_index'); + const dummyNodes = [ + { + id: id1, + position: { x: 0, y: 500 }, + data: initComponentData(new TextEmbeddingTransformer().toObj(), id1), + type: 'customComponent', + }, + { + id: id2, + position: { x: 0, y: 200 }, + data: initComponentData(new TextEmbeddingTransformer().toObj(), id2), + type: 'customComponent', + }, + { + id: id3, + position: { x: 500, y: 500 }, + data: initComponentData(new KnnIndexer().toObj(), id3), + type: 'customComponent', + }, + ] as ReactFlowComponent[]; + + return { + nodes: dummyNodes, + edges: [] as ReactFlowEdge[], + }; +} + +// TODO: implement this +/** + * Validates the UI workflow state. + * Note we don't have to validate connections since that is done via input/output handlers. + */ +export function validateWorkspaceFlow( + workspaceFlow: WorkspaceFlowState +): boolean { + return true; +} + +// TODO: implement this +/** + * Validates the backend template. May be used when parsing persisted templates on server-side, + * or when importing/exporting on the UI. + */ +export function validateWorkflowTemplate( + workflowTemplate: WorkflowTemplate +): boolean { + return true; +} diff --git a/public/pages/workflow_detail/utils/utils.ts b/public/pages/workflow_detail/utils/utils.ts index d1aa76cc..f536ab25 100644 --- a/public/pages/workflow_detail/utils/utils.ts +++ b/public/pages/workflow_detail/utils/utils.ts @@ -5,10 +5,10 @@ import { WorkspaceFlowState, - UseCaseTemplate, Workflow, - USE_CASE, ReactFlowComponent, + toTemplateFlows, + validateWorkspaceFlow, } from '../../../../common'; export function saveWorkflow(workflow: Workflow, rfInstance: any): void { @@ -19,12 +19,12 @@ export function saveWorkflow(workflow: Workflow, rfInstance: any): void { nodes: processNodes(curFlowState.nodes), }; - const isValid = validateFlowState(curFlowState); + const isValid = validateWorkspaceFlow(curFlowState); if (isValid) { const updatedWorkflow = { ...workflow, workspaceFlowState: curFlowState, - template: generateUseCaseTemplate(curFlowState), + workflows: toTemplateFlows(curFlowState), } as Workflow; if (workflow.id) { // TODO: implement connection to update workflow API @@ -36,42 +36,6 @@ export function saveWorkflow(workflow: Workflow, rfInstance: any): void { } } -// TODO: implement this. Need more info on UX side to finalize what we need -// to persist, what validation to do, etc. -// Note we don't have to validate connections since that is done via input/output handlers. -function validateFlowState(flowState: WorkspaceFlowState): boolean { - return true; -} - -// TODO: implement this -function generateUseCaseTemplate( - flowState: WorkspaceFlowState -): UseCaseTemplate { - return { - name: 'example-name', - description: 'example description', - type: USE_CASE.SEMANTIC_SEARCH, - userInputs: {}, - workflows: { - provision: { - userParams: {}, - nodes: [], - edges: [], - }, - ingest: { - userParams: {}, - nodes: [], - edges: [], - }, - query: { - userParams: {}, - nodes: [], - edges: [], - }, - }, - } as UseCaseTemplate; -} - // Process the raw ReactFlow nodes to only persist the fields we need function processNodes(nodes: ReactFlowComponent[]): ReactFlowComponent[] { return nodes diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx index 53f23372..38e15f2e 100644 --- a/public/pages/workflow_detail/workflow_detail.tsx +++ b/public/pages/workflow_detail/workflow_detail.tsx @@ -105,8 +105,15 @@ export function WorkflowDetail(props: WorkflowDetailProps) { // TODO: can optimize to only fetch a single workflow dispatch(searchWorkflows({ query: { match_all: {} } })); } - window.onbeforeunload = (e) => - isDirty || isNewWorkflow ? true : undefined; + + // TODO: below has the following issue: + // 1. user starts to create new unsaved workflow changes + // 2. user navigates to other parts of the plugin without refreshing - no warning happens + // 3. user refreshes at any later time: if isDirty is still true, shows browser warning + // tune to only handle the check if still on the workflow details page, or consider adding a check / warning + // if navigating away from the details page without refreshing (where it is currently not being triggered) + // window.onbeforeunload = (e) => + // isDirty || isNewWorkflow ? true : undefined; }, []); const tabs = [ diff --git a/public/pages/workflow_detail/workspace/workspace.tsx b/public/pages/workflow_detail/workspace/workspace.tsx index c53f23e2..6fb9f5b4 100644 --- a/public/pages/workflow_detail/workspace/workspace.tsx +++ b/public/pages/workflow_detail/workspace/workspace.tsx @@ -21,9 +21,9 @@ import { IComponentData, ReactFlowComponent, Workflow, + toWorkspaceFlow, } from '../../../../common'; import { generateId, initComponentData } from '../../../utils'; -import { getCore } from '../../../services'; import { WorkspaceComponent } from '../workspace_component'; import { DeletableEdge } from '../workspace_edge'; @@ -119,16 +119,19 @@ export function Workspace(props: WorkspaceProps) { // Initialization. Set the nodes and edges to an existing workflow, // if applicable. useEffect(() => { - const workflow = props.workflow; + const workflow = { ...props.workflow }; if (workflow) { - if (workflow.workspaceFlowState) { - setNodes(workflow.workspaceFlowState.nodes); - setEdges(workflow.workspaceFlowState.edges); - } else { - getCore().notifications.toasts.addWarning( - `There is no configured UI flow for workflow: ${workflow.name}` + if (!workflow.workspaceFlowState) { + // No existing workspace state. This could be due to it being a backend-only-created + // workflow, or a new, unsaved workflow + // @ts-ignore + workflow.workspaceFlowState = toWorkspaceFlow(workflow.workflows); + console.debug( + `There is no saved UI flow for workflow: ${workflow.name}. Generating a default one.` ); } + setNodes(workflow.workspaceFlowState.nodes); + setEdges(workflow.workspaceFlowState.edges); } }, [props.workflow]); diff --git a/public/pages/workflows/new_workflow/new_workflow.tsx b/public/pages/workflows/new_workflow/new_workflow.tsx index c255fd50..266143ce 100644 --- a/public/pages/workflows/new_workflow/new_workflow.tsx +++ b/public/pages/workflows/new_workflow/new_workflow.tsx @@ -10,33 +10,31 @@ import { EuiFlexGrid, EuiFlexGroup, EuiFieldSearch, + EuiLoadingSpinner, } from '@elastic/eui'; -import { useDispatch } from 'react-redux'; +import { useDispatch, useSelector } from 'react-redux'; import { UseCase } from './use_case'; -import { getPresetWorkflows } from './presets'; import { DEFAULT_NEW_WORKFLOW_NAME, START_FROM_SCRATCH_WORKFLOW_NAME, Workflow, } from '../../../../common'; -import { cacheWorkflow } from '../../../store'; +import { AppState, cacheWorkflow } from '../../../store'; +import { getWorkflowPresets } from '../../../store/reducers'; interface NewWorkflowProps {} /** - * TODO: may rename this later on. - * * Contains the searchable library of templated workflows based * on a variety of use cases. Can click on them to load in a pre-configured * workflow for users to start with. */ export function NewWorkflow(props: NewWorkflowProps) { const dispatch = useDispatch(); - // preset workflow state - const presetWorkflows = getPresetWorkflows(); - const [filteredWorkflows, setFilteredWorkflows] = useState( - getPresetWorkflows() + const { presetWorkflows, loading } = useSelector( + (state: AppState) => state.presets ); + const [filteredWorkflows, setFilteredWorkflows] = useState([]); // search bar state const [searchQuery, setSearchQuery] = useState(''); @@ -44,6 +42,15 @@ export function NewWorkflow(props: NewWorkflowProps) { setSearchQuery(query); }, 200); + // initial state + useEffect(() => { + dispatch(getWorkflowPresets()); + }, []); + + useEffect(() => { + setFilteredWorkflows(presetWorkflows); + }, [presetWorkflows]); + // When search query updated, re-filter preset list useEffect(() => { setFilteredWorkflows(fetchFilteredWorkflows(presetWorkflows, searchQuery)); @@ -59,26 +66,30 @@ export function NewWorkflow(props: NewWorkflowProps) { /> - - {filteredWorkflows.map((workflow: Workflow, index) => { - return ( - - - dispatch( - cacheWorkflow({ - ...workflow, - name: processWorkflowName(workflow.name), - }) - ) - } - /> - - ); - })} - + {loading ? ( + + ) : ( + + {filteredWorkflows.map((workflow: Workflow, index) => { + return ( + + + dispatch( + cacheWorkflow({ + ...workflow, + name: processWorkflowName(workflow.name), + }) + ) + } + /> + + ); + })} + + )} ); diff --git a/public/pages/workflows/new_workflow/presets.tsx b/public/pages/workflows/new_workflow/presets.tsx deleted file mode 100644 index 0cf7d382..00000000 --- a/public/pages/workflows/new_workflow/presets.tsx +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { - START_FROM_SCRATCH_WORKFLOW_NAME, - Workflow, - WorkspaceFlowState, -} from '../../../../common'; - -// TODO: fetch from the backend when the workflow library is complete. -/** - * Used to fetch the library of preset workflows to provide to users. - */ -export function getPresetWorkflows(): Workflow[] { - return [ - { - name: 'Semantic Search', - description: - 'This semantic search workflow includes the essential ingestion and search pipelines that covers the most common search use cases.', - useCase: 'SEMANTIC_SEARCH', - template: {}, - workspaceFlowState: { - nodes: [], - edges: [], - } as WorkspaceFlowState, - }, - { - name: 'Semantic Search with Reranking', - description: - 'This semantic search workflow variation includes an ML processor to rerank fetched results.', - useCase: 'SEMANTIC_SEARCH_WITH_RERANK', - template: {}, - workspaceFlowState: { - nodes: [], - edges: [], - } as WorkspaceFlowState, - }, - { - name: START_FROM_SCRATCH_WORKFLOW_NAME, - description: - 'Build your workflow from scratch according to your specific use cases. Start by adding components for your ingest or query needs.', - useCase: '', - template: {}, - workspaceFlowState: { - nodes: [], - edges: [], - } as WorkspaceFlowState, - }, - { - name: 'Visual Search', - description: - 'Build an application that will return results based on images.', - useCase: 'SEMANTIC_SEARCH', - template: {}, - workspaceFlowState: { - nodes: [], - edges: [], - } as WorkspaceFlowState, - }, - ] as Workflow[]; -} diff --git a/public/pages/workflows/workflow_list/columns.tsx b/public/pages/workflows/workflow_list/columns.tsx index f4079893..9f7ae6ae 100644 --- a/public/pages/workflows/workflow_list/columns.tsx +++ b/public/pages/workflows/workflow_list/columns.tsx @@ -22,7 +22,7 @@ export const columns = (actions: any[]) => [ sortable: true, }, { - field: 'useCase', + field: 'use_case', name: 'Type', sortable: true, }, diff --git a/public/route_service.ts b/public/route_service.ts index ad9407bb..6130e2a6 100644 --- a/public/route_service.ts +++ b/public/route_service.ts @@ -11,6 +11,7 @@ import { GET_WORKFLOW_NODE_API_PATH, GET_WORKFLOW_STATE_NODE_API_PATH, SEARCH_WORKFLOWS_NODE_API_PATH, + GET_PRESET_WORKFLOWS_NODE_API_PATH, } from '../common'; /** @@ -26,6 +27,7 @@ export interface RouteService { getWorkflowState: (workflowId: string) => Promise; createWorkflow: (body: {}) => Promise; deleteWorkflow: (workflowId: string) => Promise; + getWorkflowPresets: () => Promise; catIndices: (pattern: string) => Promise; } @@ -87,6 +89,16 @@ export function configureRoutes(core: CoreStart): RouteService { return e as HttpFetchError; } }, + getWorkflowPresets: async () => { + try { + const response = await core.http.get<{ respString: string }>( + GET_PRESET_WORKFLOWS_NODE_API_PATH + ); + return response; + } catch (e: any) { + return e as HttpFetchError; + } + }, catIndices: async (pattern: string) => { try { const response = await core.http.get<{ respString: string }>( diff --git a/public/store/reducers/index.ts b/public/store/reducers/index.ts index ef2d0f33..522987d9 100644 --- a/public/store/reducers/index.ts +++ b/public/store/reducers/index.ts @@ -6,3 +6,4 @@ export * from './workspace_reducer'; export * from './opensearch_reducer'; export * from './workflows_reducer'; +export * from './presets_reducer'; diff --git a/public/store/reducers/presets_reducer.ts b/public/store/reducers/presets_reducer.ts new file mode 100644 index 00000000..a0cf7d4f --- /dev/null +++ b/public/store/reducers/presets_reducer.ts @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createAsyncThunk, createSlice } from '@reduxjs/toolkit'; +import { Workflow } from '../../../common'; +import { HttpFetchError } from '../../../../../src/core/public'; +import { getRouteService } from '../../services'; + +const initialState = { + loading: false, + errorMessage: '', + presetWorkflows: [] as Workflow[], +}; + +const PRESET_ACTION_PREFIX = 'presets'; +const GET_WORKFLOW_PRESETS_ACTION = `${PRESET_ACTION_PREFIX}/getPresets`; + +export const getWorkflowPresets = createAsyncThunk( + GET_WORKFLOW_PRESETS_ACTION, + async (_, { rejectWithValue }) => { + const response: + | any + | HttpFetchError = await getRouteService().getWorkflowPresets(); + if (response instanceof HttpFetchError) { + return rejectWithValue( + 'Error getting workflow presets: ' + response.body.message + ); + } else { + return response; + } + } +); + +const presetsSlice = createSlice({ + name: 'presets', + initialState, + reducers: {}, + extraReducers: (builder) => { + builder + .addCase(getWorkflowPresets.pending, (state, action) => { + state.loading = true; + state.errorMessage = ''; + }) + .addCase(getWorkflowPresets.fulfilled, (state, action) => { + state.presetWorkflows = action.payload.workflowTemplates; + state.loading = false; + state.errorMessage = ''; + }) + .addCase(getWorkflowPresets.rejected, (state, action) => { + state.errorMessage = action.payload as string; + state.loading = false; + }); + }, +}); + +export const presetsReducer = presetsSlice.reducer; diff --git a/public/store/reducers/workflows_reducer.ts b/public/store/reducers/workflows_reducer.ts index 4dddf786..1092d605 100644 --- a/public/store/reducers/workflows_reducer.ts +++ b/public/store/reducers/workflows_reducer.ts @@ -14,36 +14,11 @@ import { initComponentData, WORKFLOW_STATE, WorkflowDict, + WorkflowTemplate, } from '../../../common'; import { HttpFetchError } from '../../../../../src/core/public'; import { getRouteService } from '../../services'; -// TODO: remove hardcoded dummy node data below after fetching from server side, -// and workflow data model interface is more defined. -const id1 = generateId('text_embedding_processor'); -const id2 = generateId('text_embedding_processor'); -const id3 = generateId('knn_index'); -const dummyNodes = [ - { - id: id1, - position: { x: 0, y: 500 }, - data: initComponentData(new TextEmbeddingTransformer().toObj(), id1), - type: 'customComponent', - }, - { - id: id2, - position: { x: 0, y: 200 }, - data: initComponentData(new TextEmbeddingTransformer().toObj(), id2), - type: 'customComponent', - }, - { - id: id3, - position: { x: 500, y: 500 }, - data: initComponentData(new KnnIndexer().toObj(), id3), - type: 'customComponent', - }, -] as ReactFlowComponent[]; - const initialState = { loading: false, errorMessage: '', @@ -195,17 +170,6 @@ const workflowsSlice = createSlice({ }) .addCase(searchWorkflows.fulfilled, (state, action) => { const { workflows } = action.payload as { workflows: WorkflowDict }; - - // TODO: remove hardcoded workspace flow state. For testing purposes only - Object.entries(workflows).forEach(([workflowId, workflow]) => { - workflows[workflowId] = { - ...workflows[workflowId], - workspaceFlowState: { - nodes: dummyNodes, - edges: [] as ReactFlowEdge[], - }, - }; - }); state.workflows = workflows; state.loading = false; state.errorMessage = ''; diff --git a/public/store/store.ts b/public/store/store.ts index 0f22eada..fa906b99 100644 --- a/public/store/store.ts +++ b/public/store/store.ts @@ -9,11 +9,13 @@ import { workspaceReducer, opensearchReducer, workflowsReducer, + presetsReducer, } from './reducers'; const rootReducer = combineReducers({ workspace: workspaceReducer, workflows: workflowsReducer, + presets: presetsReducer, opensearch: opensearchReducer, }); diff --git a/server/resources/templates/semantic_search.json b/server/resources/templates/semantic_search.json new file mode 100644 index 00000000..5f3beee6 --- /dev/null +++ b/server/resources/templates/semantic_search.json @@ -0,0 +1,39 @@ +{ + "name": "Semantic Search", + "description": "This semantic search workflow includes the essential ingestion and search pipelines that covers the most common search use cases.", + "use_case": "PROVISION", + "version": { + "template": "1.0.0", + "compatibility": [ + "2.12.0", + "3.0.0" + ] + }, + "workflows": { + "provision": { + "nodes": [ + { + "id": "create_ingest_pipeline", + "type": "create_ingest_pipeline", + "user_inputs": { + "pipeline_id": "text-embedding-pipeline", + "model_id": "my-model-id", + "configurations": { + "description": "A text embedding pipeline", + "processors": [ + { + "text_embedding": { + "model_id": "${{user_inputs.model_id}}", + "field_map": { + "passage_text": "${{output}}" + } + } + } + ] + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/server/resources/templates/start_from_scratch.json b/server/resources/templates/start_from_scratch.json new file mode 100644 index 00000000..7090db2c --- /dev/null +++ b/server/resources/templates/start_from_scratch.json @@ -0,0 +1,13 @@ +{ + "name": "Start From Scratch", + "description": "Build your workflow from scratch according to your specific use cases. Start by adding components for your ingest or query needs.", + "use_case": "CUSTOM", + "version": { + "template": "1.0.0", + "compatibility": [ + "2.12.0", + "3.0.0" + ] + }, + "workflows": {} +} \ No newline at end of file diff --git a/server/routes/flow_framework_routes_service.ts b/server/routes/flow_framework_routes_service.ts index afcda1fc..19c802a5 100644 --- a/server/routes/flow_framework_routes_service.ts +++ b/server/routes/flow_framework_routes_service.ts @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +import fs from 'fs'; +import path from 'path'; import { schema } from '@osd/config-schema'; import { IRouter, @@ -14,9 +16,12 @@ import { import { CREATE_WORKFLOW_NODE_API_PATH, DELETE_WORKFLOW_NODE_API_PATH, + GET_PRESET_WORKFLOWS_NODE_API_PATH, GET_WORKFLOW_NODE_API_PATH, GET_WORKFLOW_STATE_NODE_API_PATH, SEARCH_WORKFLOWS_NODE_API_PATH, + WorkflowTemplate, + validateWorkflowTemplate, } from '../../common'; import { generateCustomError, getWorkflowsFromResponses } from './helpers'; @@ -83,6 +88,14 @@ export function registerFlowFrameworkRoutes( }, flowFrameworkRoutesService.deleteWorkflow ); + + router.get( + { + path: GET_PRESET_WORKFLOWS_NODE_API_PATH, + validate: {}, + }, + flowFrameworkRoutesService.getPresetWorkflows + ); } export class FlowFrameworkRoutesService { @@ -193,4 +206,35 @@ export class FlowFrameworkRoutesService { return generateCustomError(res, err); } }; + + getPresetWorkflows = async ( + context: RequestHandlerContext, + req: OpenSearchDashboardsRequest, + res: OpenSearchDashboardsResponseFactory + ): Promise> => { + try { + // In the future we may get these from backend via some API. For now we can + // persist a set of working presets on server-side. + const jsonTemplateDir = path.resolve(__dirname, '../resources/templates'); + const jsonTemplates = fs + .readdirSync(jsonTemplateDir) + .filter((file) => path.extname(file) === '.json'); + const workflowTemplates = [] as WorkflowTemplate[]; + jsonTemplates.forEach((jsonTemplate) => { + const templateData = fs.readFileSync( + path.join(jsonTemplateDir, jsonTemplate) + ); + const workflowTemplate = JSON.parse( + templateData.toString() + ) as WorkflowTemplate; + if (validateWorkflowTemplate(workflowTemplate)) { + workflowTemplates.push(workflowTemplate); + } + }); + + return res.ok({ body: { workflowTemplates } }); + } catch (err: any) { + return generateCustomError(res, err); + } + }; } diff --git a/server/routes/helpers.ts b/server/routes/helpers.ts index 49283de3..58b813ac 100644 --- a/server/routes/helpers.ts +++ b/server/routes/helpers.ts @@ -26,10 +26,10 @@ function toWorkflowObj(workflowHit: any): Workflow { return { id: workflowHit._id, name: hitSource.name, - useCase: hitSource.use_case, + use_case: hitSource.use_case, description: hitSource.description || '', - // TODO: update below values after frontend Workflow interface is finalized - template: {}, + version: hitSource.version, + workflows: hitSource.workflows, // TODO: this needs to be persisted by backend. Tracking issue: // https://github.com/opensearch-project/flow-framework/issues/548 lastUpdated: 1234,