fetch and set presets for input / output maps where applicable

Signed-off-by: Tyler Ohlsen <[email protected]>
ohltyler · Sep 9, 2024 · bf89503 · bf89503
1 parent 9b644de
commit bf89503
Show file tree

Hide file tree

Showing 11 changed files with 136 additions and 43 deletions.
diff --git a/common/constants.ts b/common/constants.ts
@@ -410,6 +410,7 @@ export const DEFAULT_NEW_WORKFLOW_STATE = WORKFLOW_STATE.NOT_STARTED;
 export const DEFAULT_NEW_WORKFLOW_STATE_TYPE = ('NOT_STARTED' as any) as typeof WORKFLOW_STATE;
 export const DATE_FORMAT_PATTERN = 'MM/DD/YY hh:mm A';
 export const EMPTY_FIELD_STRING = '--';
+export const OMIT_SYSTEM_INDEX_PATTERN = '*,-.*';
 export const INDEX_NOT_FOUND_EXCEPTION = 'index_not_found_exception';
 export const ERROR_GETTING_WORKFLOW_MSG = 'Failed to retrieve template';
 export const NO_TEMPLATES_FOUND_MSG = 'There are no templates';

diff --git a/package.json b/package.json
@@ -28,11 +28,12 @@
   },
   "dependencies": {
     "@types/jsonpath": "^0.2.4",
+    "flattie": "^1.1.1",
     "formik": "2.4.2",
     "jsonpath": "^1.1.1",
     "reactflow": "^11.8.3",
     "yup": "^1.3.2"
   },
   "devDependencies": {},
   "resolutions": {}
-}
+}
diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx
@@ -32,6 +32,7 @@ import {
   FETCH_ALL_QUERY,
   MAX_WORKFLOW_NAME_TO_DISPLAY,
   NO_TEMPLATES_FOUND_MSG,
+  OMIT_SYSTEM_INDEX_PATTERN,
   getCharacterLimitedString,
 } from '../../../common';
 import { MountPoint } from '../../../../../src/core/public';
@@ -106,7 +107,7 @@ export function WorkflowDetail(props: WorkflowDetailProps) {
   useEffect(() => {
     dispatch(getWorkflow({ workflowId, dataSourceId }));
     dispatch(searchModels({ apiBody: FETCH_ALL_QUERY, dataSourceId }));
-    dispatch(catIndices({ pattern: '*,-.*', dataSourceId }));
+    dispatch(catIndices({ pattern: OMIT_SYSTEM_INDEX_PATTERN, dataSourceId }));
   }, []);
 
   return errorMessage.includes(ERROR_GETTING_WORKFLOW_MSG) ||

diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx
@@ -37,8 +37,8 @@ interface MapArrayFieldProps {
   valuePlaceholder?: string;
   onMapAdd?: (curArray: MapArrayFormValue) => void;
   onMapDelete?: (idxToDelete: number) => void;
-  keyOptions?: any[];
-  valueOptions?: any[];
+  keyOptions?: { label: string }[];
+  valueOptions?: { label: string }[];
 }
 
 /**

diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx
@@ -31,8 +31,8 @@ interface MapFieldProps {
   helpText?: string;
   keyPlaceholder?: string;
   valuePlaceholder?: string;
-  keyOptions?: any[];
-  valueOptions?: any[];
+  keyOptions?: { label: string }[];
+  valueOptions?: { label: string }[];
 }
 
 /**

diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx
@@ -12,7 +12,7 @@ import { WorkspaceFormValues } from '../../../../../common';
 interface SelectWithCustomOptionsProps {
   fieldPath: string;
   placeholder: string;
-  options: any[];
+  options: { label: string }[];
 }
 
 /**
@@ -50,13 +50,15 @@ export function SelectWithCustomOptions(props: SelectWithCustomOptionsProps) {
     return (
       <EuiFlexGroup direction="row" alignItems="flexStart" gutterSize="s">
         <EuiFlexItem grow={false}>
-          <EuiText size="s">{option.label}</EuiText>
-        </EuiFlexItem>
-        <EuiFlexItem grow={false}>
-          <EuiText size="xs" color="subdued" style={{ marginTop: '2px' }}>
-            {`(${option.type || 'unknown type'})`}
-          </EuiText>
+          <EuiText size="s">{option.label || ''}</EuiText>
         </EuiFlexItem>
+        {option.type && (
+          <EuiFlexItem grow={false}>
+            <EuiText size="xs" color="subdued" style={{ marginTop: '2px' }}>
+              {`(${option.type})`}
+            </EuiText>
+          </EuiFlexItem>
+        )}
       </EuiFlexGroup>
     );
   }

diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx
@@ -5,7 +5,9 @@
 
 import React, { useState, useEffect } from 'react';
 import { getIn, useFormikContext } from 'formik';
+import { isEmpty } from 'lodash';
 import { useSelector } from 'react-redux';
+import { flattie } from 'flattie';
 import {
   EuiAccordion,
   EuiSmallButtonEmpty,
@@ -25,14 +27,15 @@ import {
   ML_INFERENCE_DOCS_LINK,
   WorkflowFormValues,
   ModelInterface,
+  IndexMappings,
 } from '../../../../../common';
 import { MapArrayField, ModelField } from '../input_fields';
-import { isEmpty } from 'lodash';
 import { InputTransformModal } from './input_transform_modal';
 import { OutputTransformModal } from './output_transform_modal';
-import { AppState } from '../../../../store';
+import { AppState, getMappings, useAppDispatch } from '../../../../store';
 import {
   formikToPartialPipeline,
+  getDataSourceId,
   parseModelInputs,
   parseModelOutputs,
 } from '../../../../utils';
@@ -52,7 +55,10 @@ interface MLProcessorInputsProps {
  * output map configuration forms, respectively.
  */
 export function MLProcessorInputs(props: MLProcessorInputsProps) {
+  const dispatch = useAppDispatch();
+  const dataSourceId = getDataSourceId();
   const models = useSelector((state: AppState) => state.ml.models);
+  const indices = useSelector((state: AppState) => state.opensearch.indices);
   const { values, setFieldValue, setFieldTouched } = useFormikContext<
     WorkflowFormValues
   >();
@@ -115,7 +121,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
   // 1: update model interface states
   // 2. clear out any persisted input_map/output_map form values, as those would now be invalid
   function onModelChange(modelId: string) {
-    updateModelInterfaceStates(modelId);
+    setModelInterface(models[modelId]?.interface);
     setFieldValue(inputMapFieldPath, []);
     setFieldValue(outputMapFieldPath, []);
     setFieldTouched(inputMapFieldPath, false);
@@ -127,16 +133,75 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
     if (!isEmpty(models)) {
       const modelId = getIn(values, `${modelFieldPath}.id`);
       if (modelId) {
-        updateModelInterfaceStates(modelId);
+        setModelInterface(models[modelId]?.interface);
       }
     }
   }, [models]);
 
-  // reusable function to update interface states based on the model ID
-  function updateModelInterfaceStates(modelId: string) {
-    const newSelectedModel = models[modelId];
-    setModelInterface(newSelectedModel?.interface);
-  }
+  // persisting doc/query/index mapping fields to collect a list
+  // of options to display in the dropdowns when configuring input / output maps
+  const [docFields, setDocFields] = useState<{ label: string }[]>([]);
+  const [queryFields, setQueryFields] = useState<{ label: string }[]>([]);
+  const [indexMappingFields, setIndexMappingFields] = useState<
+    { label: string }[]
+  >([]);
+  useEffect(() => {
+    try {
+      const docObjKeys = Object.keys(
+        flattie((JSON.parse(values.ingest.docs) as {}[])[0])
+      );
+      if (docObjKeys.length > 0) {
+        setDocFields(
+          docObjKeys.map((key) => {
+            return {
+              label: key,
+            };
+          })
+        );
+      }
+    } catch {}
+  }, [values?.ingest?.docs]);
+  useEffect(() => {
+    try {
+      const queryObjKeys = Object.keys(
+        flattie(JSON.parse(values.search.request))
+      );
+      if (queryObjKeys.length > 0) {
+        setQueryFields(
+          queryObjKeys.map((key) => {
+            return {
+              label: key,
+            };
+          })
+        );
+      }
+    } catch {}
+  }, [values?.search?.request]);
+  useEffect(() => {
+    const indexName = values?.ingest?.index?.name as string | undefined;
+    if (indexName !== undefined && indices[indexName] !== undefined) {
+      dispatch(
+        getMappings({
+          index: indexName,
+          dataSourceId,
+        })
+      )
+        .unwrap()
+        .then((resp: IndexMappings) => {
+          const mappingsObjKeys = Object.keys(resp.properties);
+          if (mappingsObjKeys.length > 0) {
+            setIndexMappingFields(
+              mappingsObjKeys.map((key) => {
+                return {
+                  label: key,
+                  type: resp.properties[key]?.type,
+                };
+              })
+            );
+          }
+        });
+    }
+  }, [values?.ingest?.index?.name]);
 
   return (
     <>
@@ -212,12 +277,19 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
             root object selector "${JSONPATH_ROOT_SELECTOR}"`}
             helpLink={ML_INFERENCE_DOCS_LINK}
             keyPlaceholder="Model input field"
+            keyOptions={parseModelInputs(modelInterface)}
             valuePlaceholder={
               props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
                 ? 'Query field'
                 : 'Document field'
             }
-            keyOptions={parseModelInputs(modelInterface)}
+            valueOptions={
+              props.context === PROCESSOR_CONTEXT.INGEST
+                ? docFields
+                : props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
+                ? queryFields
+                : indexMappingFields
+            }
           />
           <EuiSpacer size="l" />
           <EuiFlexGroup direction="row">
@@ -262,7 +334,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
             keyPlaceholder={
               props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
                 ? 'Query field'
-                : 'Document field'
+                : 'New document field'
             }
             valuePlaceholder="Model output field"
             valueOptions={parseModelOutputs(modelInterface)}

diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx
@@ -3,12 +3,17 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import React from 'react';
+import React, { useEffect } from 'react';
 import { EuiFlexGroup, EuiFlexItem, EuiHorizontalRule } from '@elastic/eui';
 import { ConfigureSearchRequest } from './configure_search_request';
 import { EnrichSearchRequest } from './enrich_search_request';
 import { EnrichSearchResponse } from './enrich_search_response';
-import { WorkflowConfig } from '../../../../../common';
+import {
+  OMIT_SYSTEM_INDEX_PATTERN,
+  WorkflowConfig,
+} from '../../../../../common';
+import { catIndices, useAppDispatch } from '../../../../store';
+import { getDataSourceId } from '../../../../utils';
 
 interface SearchInputsProps {
   uiConfig: WorkflowConfig;
@@ -21,6 +26,15 @@ interface SearchInputsProps {
  * The base component containing all of the search-related inputs
  */
 export function SearchInputs(props: SearchInputsProps) {
+  const dispatch = useAppDispatch();
+  const dataSourceId = getDataSourceId();
+  // re-fetch indices on initial load. When users are first creating,
+  // they may enter this page without getting the updated index info
+  // for a newly-created index, so we re-fetch that here.
+  useEffect(() => {
+    dispatch(catIndices({ pattern: OMIT_SYSTEM_INDEX_PATTERN, dataSourceId }));
+  }, []);
+
   return (
     <EuiFlexGroup direction="column">
       <EuiFlexItem grow={false}>

diff --git a/public/pages/workflows/new_workflow/quick_configure_modal.tsx b/public/pages/workflows/new_workflow/quick_configure_modal.tsx
@@ -5,6 +5,9 @@
 
 import React, { useState, useEffect } from 'react';
 import { useHistory } from 'react-router-dom';
+import { isEmpty } from 'lodash';
+import { useSelector } from 'react-redux';
+import { flattie } from 'flattie';
 import {
   EuiSmallButton,
   EuiModal,
@@ -17,8 +20,6 @@ import {
   EuiCompressedFormRow,
 } from '@elastic/eui';
 import {
-  DEFAULT_LABEL_FIELD,
-  DEFAULT_TEXT_FIELD,
   IMAGE_FIELD_PATTERN,
   IndexMappings,
   LABEL_FIELD_PATTERN,
@@ -47,8 +48,6 @@ import {
   parseModelOutputs,
 } from '../../../utils/utils';
 import { QuickConfigureInputs } from './quick_configure_inputs';
-import { isEmpty } from 'lodash';
-import { useSelector } from 'react-redux';
 
 interface QuickConfigureModalProps {
   workflow: Workflow;
@@ -292,37 +291,34 @@ function updateSearchRequestProcessorConfig(
   modelInterface: ModelInterface | undefined,
   isVectorSearchUseCase: boolean
 ): WorkflowConfig {
+  let defaultQueryValue = '' as string;
+  try {
+    defaultQueryValue = Object.keys(
+      flattie(JSON.parse(config.search?.request?.value as string))
+    )[0];
+  } catch {}
   config.search.enrichRequest.processors[0].fields.forEach((field) => {
     if (field.id === 'model' && fields.modelId) {
       field.value = { id: fields.modelId };
     }
     if (field.id === 'input_map') {
       const inputMap = generateMapFromModelInputs(modelInterface);
-      // TODO: may change in the future. This is assuming the default query is a
-      // basic term query.
-      const defaultValue = `query.term.${
-        isVectorSearchUseCase ? DEFAULT_TEXT_FIELD : DEFAULT_LABEL_FIELD
-      }.value`;
       if (inputMap.length > 0) {
         inputMap[0] = {
           ...inputMap[0],
-          value: defaultValue,
+          value: defaultQueryValue,
         };
       } else {
         inputMap.push({
           key: '',
-          value: defaultValue,
+          value: defaultQueryValue,
         });
       }
       field.value = [inputMap] as MapArrayFormValue;
     }
     if (field.id === 'output_map') {
       const outputMap = generateMapFromModelOutputs(modelInterface);
-      // TODO: may change in the future. This is assuming the default query is a
-      // basic term query.
-      const defaultKey = isVectorSearchUseCase
-        ? VECTOR
-        : `query.term.${DEFAULT_LABEL_FIELD}.value`;
+      const defaultKey = isVectorSearchUseCase ? VECTOR : defaultQueryValue;
       if (outputMap.length > 0) {
         outputMap[0] = {
           ...outputMap[0],

diff --git a/public/store/reducers/opensearch_reducer.ts b/public/store/reducers/opensearch_reducer.ts
@@ -8,6 +8,7 @@ import { getRouteService } from '../../services';
 import {
   Index,
   IngestPipelineConfig,
+  OMIT_SYSTEM_INDEX_PATTERN,
   SimulateIngestPipelineDoc,
 } from '../../../common';
 import { HttpFetchError } from '../../../../../src/core/public';
@@ -33,7 +34,7 @@ export const catIndices = createAsyncThunk(
     { rejectWithValue }
   ) => {
     // defaulting to fetch everything except system indices (starting with '.')
-    const patternString = pattern || '*,-.*';
+    const patternString = pattern || OMIT_SYSTEM_INDEX_PATTERN;
     const response: any | HttpFetchError = await getRouteService().catIndices(
       patternString,
       dataSourceId

diff --git a/yarn.lock b/yarn.lock
@@ -402,6 +402,11 @@ fast-levenshtein@~2.0.6:
   resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917"
   integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==
 
+flattie@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/flattie/-/flattie-1.1.1.tgz#88182235723113667d36217fec55359275d6fe3d"
+  integrity sha512-9UbaD6XdAL97+k/n+N7JwX46K/M6Zc6KcFYskrYL8wbBV/Uyk0CTAMY0VT+qiK5PM7AIc9aTWYtq65U7T+aCNQ==
+
 [email protected]:
   version "2.4.2"
   resolved "https://registry.yarnpkg.com/formik/-/formik-2.4.2.tgz#a1115457cfb012a5c782cea3ad4b40b2fe36fa18"