[Backport 2.x] Fetch default options for ML processor config maps

common/constants.ts

@@ -410,6 +410,7 @@ export const DEFAULT_NEW_WORKFLOW_STATE = WORKFLOW_STATE.NOT_STARTED;
 export const DEFAULT_NEW_WORKFLOW_STATE_TYPE = ('NOT_STARTED' as any) as typeof WORKFLOW_STATE;
 export const DATE_FORMAT_PATTERN = 'MM/DD/YY hh:mm A';
 export const EMPTY_FIELD_STRING = '--';
+export const OMIT_SYSTEM_INDEX_PATTERN = '*,-.*';
 export const INDEX_NOT_FOUND_EXCEPTION = 'index_not_found_exception';
 export const ERROR_GETTING_WORKFLOW_MSG = 'Failed to retrieve template';
 export const NO_TEMPLATES_FOUND_MSG = 'There are no templates';

package.json

@@ -28,11 +28,12 @@
   },
   "dependencies": {
     "@types/jsonpath": "^0.2.4",
+    "flattie": "^1.1.1",
     "formik": "2.4.2",
     "jsonpath": "^1.1.1",
     "reactflow": "^11.8.3",
     "yup": "^1.3.2"
   },
   "devDependencies": {},
   "resolutions": {}
-}
+}

public/pages/workflow_detail/workflow_detail.test.tsx

@@ -6,13 +6,7 @@
 import React from 'react';
 import { render } from '@testing-library/react';
 import { Provider } from 'react-redux';
-import {
-  RouteComponentProps,
-  Route,
-  Switch,
-  Router,
-  Redirect,
-} from 'react-router-dom';
+import { RouteComponentProps, Route, Switch, Router } from 'react-router-dom';
 import { WorkflowDetail } from './workflow_detail';
 import { WorkflowDetailRouterProps } from '../../pages';
 import '@testing-library/jest-dom';

public/pages/workflow_detail/workflow_detail.tsx

@@ -32,6 +32,7 @@ import {
   FETCH_ALL_QUERY,
   MAX_WORKFLOW_NAME_TO_DISPLAY,
   NO_TEMPLATES_FOUND_MSG,
+  OMIT_SYSTEM_INDEX_PATTERN,
   getCharacterLimitedString,
 } from '../../../common';
 import { MountPoint } from '../../../../../src/core/public';
@@ -106,7 +107,7 @@ export function WorkflowDetail(props: WorkflowDetailProps) {
   useEffect(() => {
     dispatch(getWorkflow({ workflowId, dataSourceId }));
     dispatch(searchModels({ apiBody: FETCH_ALL_QUERY, dataSourceId }));
-    dispatch(catIndices({ pattern: '*,-.*', dataSourceId }));
+    dispatch(catIndices({ pattern: OMIT_SYSTEM_INDEX_PATTERN, dataSourceId }));
   }, []);
 
   return errorMessage.includes(ERROR_GETTING_WORKFLOW_MSG) ||

public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx

@@ -37,8 +37,8 @@ interface MapArrayFieldProps {
   valuePlaceholder?: string;
   onMapAdd?: (curArray: MapArrayFormValue) => void;
   onMapDelete?: (idxToDelete: number) => void;
-  keyOptions?: any[];
-  valueOptions?: any[];
+  keyOptions?: { label: string }[];
+  valueOptions?: { label: string }[];
 }
 
 /**

public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx

@@ -31,8 +31,8 @@ interface MapFieldProps {
   helpText?: string;
   keyPlaceholder?: string;
   valuePlaceholder?: string;
-  keyOptions?: any[];
-  valueOptions?: any[];
+  keyOptions?: { label: string }[];
+  valueOptions?: { label: string }[];
 }
 
 /**

public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx

@@ -12,7 +12,7 @@ import { WorkspaceFormValues } from '../../../../../common';
 interface SelectWithCustomOptionsProps {
   fieldPath: string;
   placeholder: string;
-  options: any[];
+  options: { label: string }[];
 }
 
 /**
@@ -50,13 +50,15 @@ export function SelectWithCustomOptions(props: SelectWithCustomOptionsProps) {
     return (
       <EuiFlexGroup direction="row" alignItems="flexStart" gutterSize="s">
         <EuiFlexItem grow={false}>
-          <EuiText size="s">{option.label}</EuiText>
-        </EuiFlexItem>
-        <EuiFlexItem grow={false}>
-          <EuiText size="xs" color="subdued" style={{ marginTop: '2px' }}>
-            {`(${option.type || 'unknown type'})`}
-          </EuiText>
+          <EuiText size="s">{option.label || ''}</EuiText>
         </EuiFlexItem>
+        {option.type && (
+          <EuiFlexItem grow={false}>
+            <EuiText size="xs" color="subdued" style={{ marginTop: '2px' }}>
+              {`(${option.type})`}
+            </EuiText>
+          </EuiFlexItem>
+        )}
       </EuiFlexGroup>
     );
   }

public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx

@@ -68,6 +68,7 @@ interface InputTransformModalProps {
   inputMapField: IConfigField;
   inputMapFieldPath: string;
   modelInterface: ModelInterface | undefined;
+  valueOptions: { label: string }[];
   onClose: () => void;
 }
 
@@ -163,7 +164,7 @@ export function InputTransformModal(props: InputTransformModalProps) {
                 Fetch some sample input data and see how it is transformed.
               </EuiText>
               <EuiSpacer size="s" />
-              <EuiText>Expected input</EuiText>
+              <EuiText>Source input</EuiText>
               <EuiSmallButton
                 style={{ width: '100px' }}
                 onClick={async () => {
@@ -266,7 +267,7 @@ export function InputTransformModal(props: InputTransformModalProps) {
                         })
                         .catch((error: any) => {
                           getCore().notifications.toasts.addDanger(
-                            `Failed to fetch input data`
+                            `Failed to fetch source input data`
                           );
                         });
                       break;
@@ -307,12 +308,13 @@ export function InputTransformModal(props: InputTransformModalProps) {
                 root object selector "${JSONPATH_ROOT_SELECTOR}"`}
                 helpLink={ML_INFERENCE_DOCS_LINK}
                 keyPlaceholder="Model input field"
+                keyOptions={parseModelInputs(props.modelInterface)}
                 valuePlaceholder={
                   props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
                     ? 'Query field'
                     : 'Document field'
                 }
-                keyOptions={parseModelInputs(props.modelInterface)}
+                valueOptions={props.valueOptions}
                 // If the map we are adding is the first one, populate the selected option to index 0
                 onMapAdd={(curArray) => {
                   if (isEmpty(curArray)) {
@@ -356,10 +358,10 @@ export function InputTransformModal(props: InputTransformModalProps) {
                 )}
                 <EuiFlexItem grow={true}>
                   {outputOptions.length === 1 ? (
-                    <EuiText>Expected output</EuiText>
+                    <EuiText>Transformed input</EuiText>
                   ) : (
                     <EuiCompressedSelect
-                      prepend={<EuiText>Expected output for</EuiText>}
+                      prepend={<EuiText>Transformed input for</EuiText>}
                       options={outputOptions}
                       value={selectedOutputOption}
                       onChange={(e) => {

public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx

@@ -5,7 +5,9 @@
 
 import React, { useState, useEffect } from 'react';
 import { getIn, useFormikContext } from 'formik';
+import { isEmpty } from 'lodash';
 import { useSelector } from 'react-redux';
+import { flattie } from 'flattie';
 import {
   EuiAccordion,
   EuiSmallButtonEmpty,
@@ -25,14 +27,15 @@ import {
   ML_INFERENCE_DOCS_LINK,
   WorkflowFormValues,
   ModelInterface,
+  IndexMappings,
 } from '../../../../../common';
 import { MapArrayField, ModelField } from '../input_fields';
-import { isEmpty } from 'lodash';
 import { InputTransformModal } from './input_transform_modal';
 import { OutputTransformModal } from './output_transform_modal';
-import { AppState } from '../../../../store';
+import { AppState, getMappings, useAppDispatch } from '../../../../store';
 import {
   formikToPartialPipeline,
+  getDataSourceId,
   parseModelInputs,
   parseModelOutputs,
 } from '../../../../utils';
@@ -52,7 +55,10 @@ interface MLProcessorInputsProps {
  * output map configuration forms, respectively.
  */
 export function MLProcessorInputs(props: MLProcessorInputsProps) {
+  const dispatch = useAppDispatch();
+  const dataSourceId = getDataSourceId();
   const models = useSelector((state: AppState) => state.ml.models);
+  const indices = useSelector((state: AppState) => state.opensearch.indices);
   const { values, setFieldValue, setFieldTouched } = useFormikContext<
     WorkflowFormValues
   >();
@@ -115,7 +121,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
   // 1: update model interface states
   // 2. clear out any persisted input_map/output_map form values, as those would now be invalid
   function onModelChange(modelId: string) {
-    updateModelInterfaceStates(modelId);
+    setModelInterface(models[modelId]?.interface);
     setFieldValue(inputMapFieldPath, []);
     setFieldValue(outputMapFieldPath, []);
     setFieldTouched(inputMapFieldPath, false);
@@ -127,16 +133,75 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
     if (!isEmpty(models)) {
       const modelId = getIn(values, `${modelFieldPath}.id`);
       if (modelId) {
-        updateModelInterfaceStates(modelId);
+        setModelInterface(models[modelId]?.interface);
       }
     }
   }, [models]);
 
-  // reusable function to update interface states based on the model ID
-  function updateModelInterfaceStates(modelId: string) {
-    const newSelectedModel = models[modelId];
-    setModelInterface(newSelectedModel?.interface);
-  }
+  // persisting doc/query/index mapping fields to collect a list
+  // of options to display in the dropdowns when configuring input / output maps
+  const [docFields, setDocFields] = useState<{ label: string }[]>([]);
+  const [queryFields, setQueryFields] = useState<{ label: string }[]>([]);
+  const [indexMappingFields, setIndexMappingFields] = useState<
+    { label: string }[]
+  >([]);
+  useEffect(() => {
+    try {
+      const docObjKeys = Object.keys(
+        flattie((JSON.parse(values.ingest.docs) as {}[])[0])
+      );
+      if (docObjKeys.length > 0) {
+        setDocFields(
+          docObjKeys.map((key) => {
+            return {
+              label: key,
+            };
+          })
+        );
+      }
+    } catch {}
+  }, [values?.ingest?.docs]);
+  useEffect(() => {
+    try {
+      const queryObjKeys = Object.keys(
+        flattie(JSON.parse(values.search.request))
+      );
+      if (queryObjKeys.length > 0) {
+        setQueryFields(
+          queryObjKeys.map((key) => {
+            return {
+              label: key,
+            };
+          })
+        );
+      }
+    } catch {}
+  }, [values?.search?.request]);
+  useEffect(() => {
+    const indexName = values?.search?.index?.name as string | undefined;
+    if (indexName !== undefined && indices[indexName] !== undefined) {
+      dispatch(
+        getMappings({
+          index: indexName,
+          dataSourceId,
+        })
+      )
+        .unwrap()
+        .then((resp: IndexMappings) => {
+          const mappingsObjKeys = Object.keys(resp.properties);
+          if (mappingsObjKeys.length > 0) {
+            setIndexMappingFields(
+              mappingsObjKeys.map((key) => {
+                return {
+                  label: key,
+                  type: resp.properties[key]?.type,
+                };
+              })
+            );
+          }
+        });
+    }
+  }, [values?.search?.index?.name]);
 
   return (
     <>
@@ -148,6 +213,13 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
           inputMapField={inputMapField}
           inputMapFieldPath={inputMapFieldPath}
           modelInterface={modelInterface}
+          valueOptions={
+            props.context === PROCESSOR_CONTEXT.INGEST
+              ? docFields
+              : props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
+              ? queryFields
+              : indexMappingFields
+          }
           onClose={() => setIsInputTransformModalOpen(false)}
         />
       )}
@@ -212,12 +284,19 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
             root object selector "${JSONPATH_ROOT_SELECTOR}"`}
             helpLink={ML_INFERENCE_DOCS_LINK}
             keyPlaceholder="Model input field"
+            keyOptions={parseModelInputs(modelInterface)}
             valuePlaceholder={
               props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
                 ? 'Query field'
                 : 'Document field'
             }
-            keyOptions={parseModelInputs(modelInterface)}
+            valueOptions={
+              props.context === PROCESSOR_CONTEXT.INGEST
+                ? docFields
+                : props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
+                ? queryFields
+                : indexMappingFields
+            }
           />
           <EuiSpacer size="l" />
           <EuiFlexGroup direction="row">
@@ -262,7 +341,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
             keyPlaceholder={
               props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
                 ? 'Query field'
-                : 'Document field'
+                : 'New document field'
             }
             valuePlaceholder="Model output field"
             valueOptions={parseModelOutputs(modelInterface)}

public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx

@@ -126,7 +126,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
                 Fetch some sample output data and see how it is transformed.
               </EuiText>
               <EuiSpacer size="s" />
-              <EuiText>Expected input</EuiText>
+              <EuiText>Source output</EuiText>
               <EuiSmallButton
                 style={{ width: '100px' }}
                 onClick={async () => {
@@ -219,7 +219,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
                         })
                         .catch((error: any) => {
                           getCore().notifications.toasts.addDanger(
-                            `Failed to fetch input data`
+                            `Failed to fetch source output data`
                           );
                         });
                       break;
@@ -282,10 +282,10 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
           <EuiFlexItem>
             <>
               {outputOptions.length === 1 ? (
-                <EuiText>Expected output</EuiText>
+                <EuiText>Transformed output</EuiText>
               ) : (
                 <EuiCompressedSelect
-                  prepend={<EuiText>Expected output for</EuiText>}
+                  prepend={<EuiText>Transformed output for</EuiText>}
                   options={outputOptions}
                   value={selectedOutputOption}
                   onChange={(e) => {