Support multiple & nested flows (#119)

Signed-off-by: Tyler Ohlsen <[email protected]>
(cherry picked from commit 3eb94a5)

common/utils.ts

@@ -15,6 +15,8 @@ import {
   TemplateFlows,
   WorkflowTemplate,
   DATE_FORMAT_PATTERN,
+  COMPONENT_CATEGORY,
+  NODE_CATEGORY,
 } from './';
 
 // TODO: implement this and remove hardcoded return values
@@ -42,32 +44,92 @@ export function toTemplateFlows(
 export function toWorkspaceFlow(
   templateFlows: TemplateFlows
 ): WorkspaceFlowState {
-  const id1 = generateId('text_embedding_processor');
-  const id2 = generateId('text_embedding_processor');
-  const id3 = generateId('knn_index');
-  const dummyNodes = [
+  const ingestId1 = generateId('text_embedding_processor');
+  const ingestId2 = generateId('knn_index');
+  const ingestGroupId = generateId(COMPONENT_CATEGORY.INGEST);
+
+  const searchId1 = generateId('text_embedding_processor');
+  const searchId2 = generateId('knn_index');
+  const searchGroupId = generateId(COMPONENT_CATEGORY.SEARCH);
+
+  const ingestNodes = [
+    {
+      id: ingestGroupId,
+      position: { x: 400, y: 400 },
+      type: NODE_CATEGORY.INGEST_GROUP,
+      data: { label: COMPONENT_CATEGORY.INGEST },
+      style: {
+        width: 900,
+        height: 400,
+        overflowX: 'auto',
+        overflowY: 'auto',
+      },
+      className: 'reactflow__group-node__ingest',
+      selectable: true,
+    },
+    {
+      id: ingestId1,
+      position: { x: 100, y: 70 },
+      data: initComponentData(
+        new TextEmbeddingTransformer().toObj(),
+        ingestId1
+      ),
+      type: NODE_CATEGORY.CUSTOM,
+      parentNode: ingestGroupId,
+      extent: 'parent',
+      draggable: true,
+    },
+    {
+      id: ingestId2,
+      position: { x: 500, y: 70 },
+      data: initComponentData(new KnnIndexer().toObj(), ingestId2),
+      type: NODE_CATEGORY.CUSTOM,
+      parentNode: ingestGroupId,
+      extent: 'parent',
+      draggable: true,
+    },
+  ] as ReactFlowComponent[];
+
+  const searchNodes = [
     {
-      id: id1,
-      position: { x: 0, y: 500 },
-      data: initComponentData(new TextEmbeddingTransformer().toObj(), id1),
-      type: 'customComponent',
+      id: searchGroupId,
+      position: { x: 400, y: 1000 },
+      type: NODE_CATEGORY.SEARCH_GROUP,
+      data: { label: COMPONENT_CATEGORY.SEARCH },
+      style: {
+        width: 900,
+        height: 400,
+        overflowX: 'auto',
+        overflowY: 'auto',
+      },
+      className: 'reactflow__group-node__search',
+      selectable: true,
     },
     {
-      id: id2,
-      position: { x: 0, y: 200 },
-      data: initComponentData(new TextEmbeddingTransformer().toObj(), id2),
-      type: 'customComponent',
+      id: searchId1,
+      position: { x: 100, y: 70 },
+      data: initComponentData(
+        new TextEmbeddingTransformer().toObj(),
+        searchId1
+      ),
+      type: NODE_CATEGORY.CUSTOM,
+      parentNode: searchGroupId,
+      extent: 'parent',
+      draggable: true,
     },
     {
-      id: id3,
-      position: { x: 500, y: 500 },
-      data: initComponentData(new KnnIndexer().toObj(), id3),
-      type: 'customComponent',
+      id: searchId2,
+      position: { x: 500, y: 70 },
+      data: initComponentData(new KnnIndexer().toObj(), searchId2),
+      type: NODE_CATEGORY.CUSTOM,
+      parentNode: searchGroupId,
+      extent: 'parent',
+      draggable: true,
     },
   ] as ReactFlowComponent[];
 
   return {
-    nodes: dummyNodes,
+    nodes: [...ingestNodes, ...searchNodes],
     edges: [] as ReactFlowEdge[],
   };
 }

public/pages/workflow_detail/component_details/component_inputs.tsx

@@ -4,26 +4,55 @@
  */
 
 import React from 'react';
-import { EuiSpacer, EuiTitle } from '@elastic/eui';
+import { EuiSpacer, EuiText, EuiTitle } from '@elastic/eui';
 import { InputFieldList } from './input_field_list';
-import { ReactFlowComponent } from '../../../../common';
+import { NODE_CATEGORY, ReactFlowComponent } from '../../../../common';
 
 interface ComponentInputsProps {
   selectedComponent: ReactFlowComponent;
   onFormChange: () => void;
 }
 
 export function ComponentInputs(props: ComponentInputsProps) {
-  return (
-    <>
-      <EuiTitle size="m">
-        <h2>{props.selectedComponent.data.label || ''}</h2>
-      </EuiTitle>
-      <EuiSpacer size="s" />
-      <InputFieldList
-        selectedComponent={props.selectedComponent}
-        onFormChange={props.onFormChange}
-      />
-    </>
-  );
+  // Have custom layouts for parent/group flows
+  if (props.selectedComponent.type === NODE_CATEGORY.INGEST_GROUP) {
+    return (
+      <>
+        <EuiTitle size="m">
+          <h2>Ingest flow</h2>
+        </EuiTitle>
+        <EuiSpacer size="m" />
+        <EuiText size="s">
+          Configure a flow to transform your data as it is ingested into
+          OpenSearch.
+        </EuiText>
+      </>
+    );
+  } else if (props.selectedComponent.type === NODE_CATEGORY.SEARCH_GROUP) {
+    return (
+      <>
+        <EuiTitle size="m">
+          <h2>Search flow</h2>
+        </EuiTitle>
+        <EuiSpacer size="m" />
+        <EuiText size="s">
+          Configure a flow to transform input when searching against your
+          OpenSearch cluster.
+        </EuiText>
+      </>
+    );
+  } else {
+    return (
+      <>
+        <EuiTitle size="m">
+          <h2>{props.selectedComponent.data.label || ''}</h2>
+        </EuiTitle>
+        <EuiSpacer size="s" />
+        <InputFieldList
+          selectedComponent={props.selectedComponent}
+          onFormChange={props.onFormChange}
+        />
+      </>
+    );
+  }
 }

public/pages/workflow_detail/components/header.tsx

@@ -3,12 +3,9 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import React, { useContext } from 'react';
-import { useDispatch, useSelector } from 'react-redux';
+import React from 'react';
 import { EuiPageHeader, EuiButton, EuiLoadingSpinner } from '@elastic/eui';
 import { DEFAULT_NEW_WORKFLOW_NAME, Workflow } from '../../../../common';
-import { saveWorkflow } from '../utils';
-import { rfContext, AppState, removeDirty } from '../../../store';
 
 interface WorkflowDetailHeaderProps {
   tabs: any[];
@@ -17,10 +14,6 @@ interface WorkflowDetailHeaderProps {
 }
 
 export function WorkflowDetailHeader(props: WorkflowDetailHeaderProps) {
-  const dispatch = useDispatch();
-  const { reactFlowInstance } = useContext(rfContext);
-  const isDirty = useSelector((state: AppState) => state.workspace.isDirty);
-
   return (
     <EuiPageHeader
       pageTitle={

public/pages/workflow_detail/workflow_detail.tsx

@@ -57,7 +57,6 @@ export function WorkflowDetail(props: WorkflowDetailProps) {
   const { workflows, cachedWorkflow } = useSelector(
     (state: AppState) => state.workflows
   );
-  const { isDirty } = useSelector((state: AppState) => state.workspace);
 
   // selected workflow state
   const workflowId = props.match?.params?.workflowId;

public/pages/workflow_detail/workspace/reactflow-styles.scss

@@ -19,12 +19,28 @@ $handle-color-invalid: $euiColorDanger;
 
 .reactflow-workspace .react-flow__node {
   width: 300px;
+  height: 250px;
+}
+
+.reactflow__group-node {
+  width: 1200px;
+  height: 700px;
+  overflow-x: auto;
+  overflow-y: auto;
+  border: 'none';
+
+  &__ingest {
+    background: rgba($euiColorVis0, 0.3);
+  }
+  &__search {
+    background: rgba($euiColorVis1, 0.3);
+  }
 }
 
 // Overriding the styling for the reactflow node when it is selected.
 // We need to use important tag to override ReactFlow's wrapNode that sets the box-shadow.
 // Ref: https://github.com/wbkd/react-flow/blob/main/packages/core/src/components/Nodes/wrapNode.tsx#L187
-.reactflow-workspace .react-flow__node-customComponent.selected {
+.reactflow-workspace .react-flow__node-custom.selected {
   box-shadow: 0 0 2px 2px rgba(0, 0, 0, 0.5);
   border-radius: 5px;
   &:focus {

public/pages/workflow_detail/workspace/resizable_workspace.tsx

@@ -3,15 +3,17 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import React, { useRef, useState, useEffect, useContext } from 'react';
+import React, { useRef, useState, useEffect } from 'react';
 import { useDispatch, useSelector } from 'react-redux';
-import { useOnSelectionChange } from 'reactflow';
+import { ReactFlowProvider, useReactFlow } from 'reactflow';
 import { Form, Formik } from 'formik';
 import * as yup from 'yup';
 import { cloneDeep } from 'lodash';
 import {
   EuiButton,
   EuiCallOut,
+  EuiFlexGroup,
+  EuiFlexItem,
   EuiPageHeader,
   EuiResizableContainer,
 } from '@elastic/eui';
@@ -28,11 +30,14 @@ import {
   WorkspaceFlowState,
   toTemplateFlows,
 } from '../../../../common';
-import { AppState, removeDirty, setDirty, rfContext } from '../../../store';
+import { AppState, removeDirty, setDirty } from '../../../store';
 import { Workspace } from './workspace';
 import { ComponentDetails } from '../component_details';
 import { processNodes, saveWorkflow } from '../utils';
 
+// styling
+import './workspace-styles.scss';
+
 interface ResizableWorkspaceProps {
   isNewWorkflow: boolean;
   workflow?: Workflow;
@@ -77,29 +82,30 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) {
   };
 
   // Selected component state
-  const { reactFlowInstance } = useContext(rfContext);
+  const reactFlowInstance = useReactFlow();
   const [selectedComponent, setSelectedComponent] = useState<
     ReactFlowComponent
   >();
 
   /**
-   * Hook provided by reactflow to listen on when nodes are selected / de-selected.
+   * Custom listener on when nodes are selected / de-selected. Passed to
+   * downstream ReactFlow components you can listen using
+   * the out-of-the-box useOnSelectionChange hook.
    * - populate panel content appropriately
    * - open the panel if a node is selected and the panel is closed
    * - it is assumed that only one node can be selected at once
    */
-  useOnSelectionChange({
-    onChange: ({ nodes, edges }) => {
-      if (nodes && nodes.length > 0) {
-        setSelectedComponent(nodes[0]);
-        if (!isDetailsPanelOpen) {
-          onToggleChange();
-        }
-      } else {
-        setSelectedComponent(undefined);
+  // TODO: make more typesafe
+  function onSelectionChange({ nodes, edges }) {
+    if (nodes && nodes.length > 0) {
+      setSelectedComponent(nodes[0]);
+      if (!isDetailsPanelOpen) {
+        onToggleChange();
       }
-    },
-  });
+    } else {
+      setSelectedComponent(undefined);
+    }
+  }
 
   // Hook to update the workflow's flow state, if applicable. It may not exist if
   // it is a backend-only-created workflow, or a new, unsaved workflow. If so,
@@ -290,13 +296,26 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) {
                     minSize="50%"
                     paddingSize="s"
                   >
-                    <Workspace
-                      workflow={workflow}
-                      onNodesChange={onNodesChange}
-                    />
+                    <EuiFlexGroup
+                      direction="column"
+                      gutterSize="s"
+                      className="workspace-panel"
+                    >
+                      <EuiFlexItem>
+                        <ReactFlowProvider>
+                          <Workspace
+                            id="ingest"
+                            workflow={workflow}
+                            onNodesChange={onNodesChange}
+                            onSelectionChange={onSelectionChange}
+                          />
+                        </ReactFlowProvider>
+                      </EuiFlexItem>
+                    </EuiFlexGroup>
                   </EuiResizablePanel>
                   <EuiResizableButton />
                   <EuiResizablePanel
+                    className="workspace-panel"
                     style={{ marginRight: '-16px' }}
                     id={COMPONENT_DETAILS_PANEL_ID}
                     mode="collapsible"

public/pages/workflow_detail/workspace/workspace-styles.scss

@@ -2,5 +2,4 @@
   // this ratio will allow the workspace to render on a standard
   // laptop (3024 x 1964) without introducing overflow/scrolling
   height: 60vh;
-  padding: 0;
 }