From bf895034a89611cac2e6ea3867a51da71853e96a Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 9 Sep 2024 12:07:36 -0700 Subject: [PATCH] fetch and set presets for input / output maps where applicable Signed-off-by: Tyler Ohlsen --- common/constants.ts | 1 + package.json | 3 +- .../pages/workflow_detail/workflow_detail.tsx | 3 +- .../input_fields/map_array_field.tsx | 4 +- .../input_fields/map_field.tsx | 4 +- .../select_with_custom_options.tsx | 16 ++-- .../processor_inputs/ml_processor_inputs.tsx | 94 ++++++++++++++++--- .../search_inputs/search_inputs.tsx | 18 +++- .../new_workflow/quick_configure_modal.tsx | 28 +++--- public/store/reducers/opensearch_reducer.ts | 3 +- yarn.lock | 5 + 11 files changed, 136 insertions(+), 43 deletions(-) diff --git a/common/constants.ts b/common/constants.ts index 373d5286..52d26775 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -410,6 +410,7 @@ export const DEFAULT_NEW_WORKFLOW_STATE = WORKFLOW_STATE.NOT_STARTED; export const DEFAULT_NEW_WORKFLOW_STATE_TYPE = ('NOT_STARTED' as any) as typeof WORKFLOW_STATE; export const DATE_FORMAT_PATTERN = 'MM/DD/YY hh:mm A'; export const EMPTY_FIELD_STRING = '--'; +export const OMIT_SYSTEM_INDEX_PATTERN = '*,-.*'; export const INDEX_NOT_FOUND_EXCEPTION = 'index_not_found_exception'; export const ERROR_GETTING_WORKFLOW_MSG = 'Failed to retrieve template'; export const NO_TEMPLATES_FOUND_MSG = 'There are no templates'; diff --git a/package.json b/package.json index 48e918c3..4f222af0 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ }, "dependencies": { "@types/jsonpath": "^0.2.4", + "flattie": "^1.1.1", "formik": "2.4.2", "jsonpath": "^1.1.1", "reactflow": "^11.8.3", @@ -35,4 +36,4 @@ }, "devDependencies": {}, "resolutions": {} -} \ No newline at end of file +} diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx index a0ec10df..13d6468f 100644 --- a/public/pages/workflow_detail/workflow_detail.tsx +++ b/public/pages/workflow_detail/workflow_detail.tsx @@ -32,6 +32,7 @@ import { FETCH_ALL_QUERY, MAX_WORKFLOW_NAME_TO_DISPLAY, NO_TEMPLATES_FOUND_MSG, + OMIT_SYSTEM_INDEX_PATTERN, getCharacterLimitedString, } from '../../../common'; import { MountPoint } from '../../../../../src/core/public'; @@ -106,7 +107,7 @@ export function WorkflowDetail(props: WorkflowDetailProps) { useEffect(() => { dispatch(getWorkflow({ workflowId, dataSourceId })); dispatch(searchModels({ apiBody: FETCH_ALL_QUERY, dataSourceId })); - dispatch(catIndices({ pattern: '*,-.*', dataSourceId })); + dispatch(catIndices({ pattern: OMIT_SYSTEM_INDEX_PATTERN, dataSourceId })); }, []); return errorMessage.includes(ERROR_GETTING_WORKFLOW_MSG) || diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx index 11cb8cd3..6971e65e 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx @@ -37,8 +37,8 @@ interface MapArrayFieldProps { valuePlaceholder?: string; onMapAdd?: (curArray: MapArrayFormValue) => void; onMapDelete?: (idxToDelete: number) => void; - keyOptions?: any[]; - valueOptions?: any[]; + keyOptions?: { label: string }[]; + valueOptions?: { label: string }[]; } /** diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx index be9b43cd..8d026d83 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx @@ -31,8 +31,8 @@ interface MapFieldProps { helpText?: string; keyPlaceholder?: string; valuePlaceholder?: string; - keyOptions?: any[]; - valueOptions?: any[]; + keyOptions?: { label: string }[]; + valueOptions?: { label: string }[]; } /** diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx index efa11cc6..ac698805 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx @@ -12,7 +12,7 @@ import { WorkspaceFormValues } from '../../../../../common'; interface SelectWithCustomOptionsProps { fieldPath: string; placeholder: string; - options: any[]; + options: { label: string }[]; } /** @@ -50,13 +50,15 @@ export function SelectWithCustomOptions(props: SelectWithCustomOptionsProps) { return ( - {option.label} - - - - {`(${option.type || 'unknown type'})`} - + {option.label || ''} + {option.type && ( + + + {`(${option.type})`} + + + )} ); } diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx index 7baf2aee..4a231cfd 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx @@ -5,7 +5,9 @@ import React, { useState, useEffect } from 'react'; import { getIn, useFormikContext } from 'formik'; +import { isEmpty } from 'lodash'; import { useSelector } from 'react-redux'; +import { flattie } from 'flattie'; import { EuiAccordion, EuiSmallButtonEmpty, @@ -25,14 +27,15 @@ import { ML_INFERENCE_DOCS_LINK, WorkflowFormValues, ModelInterface, + IndexMappings, } from '../../../../../common'; import { MapArrayField, ModelField } from '../input_fields'; -import { isEmpty } from 'lodash'; import { InputTransformModal } from './input_transform_modal'; import { OutputTransformModal } from './output_transform_modal'; -import { AppState } from '../../../../store'; +import { AppState, getMappings, useAppDispatch } from '../../../../store'; import { formikToPartialPipeline, + getDataSourceId, parseModelInputs, parseModelOutputs, } from '../../../../utils'; @@ -52,7 +55,10 @@ interface MLProcessorInputsProps { * output map configuration forms, respectively. */ export function MLProcessorInputs(props: MLProcessorInputsProps) { + const dispatch = useAppDispatch(); + const dataSourceId = getDataSourceId(); const models = useSelector((state: AppState) => state.ml.models); + const indices = useSelector((state: AppState) => state.opensearch.indices); const { values, setFieldValue, setFieldTouched } = useFormikContext< WorkflowFormValues >(); @@ -115,7 +121,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { // 1: update model interface states // 2. clear out any persisted input_map/output_map form values, as those would now be invalid function onModelChange(modelId: string) { - updateModelInterfaceStates(modelId); + setModelInterface(models[modelId]?.interface); setFieldValue(inputMapFieldPath, []); setFieldValue(outputMapFieldPath, []); setFieldTouched(inputMapFieldPath, false); @@ -127,16 +133,75 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { if (!isEmpty(models)) { const modelId = getIn(values, `${modelFieldPath}.id`); if (modelId) { - updateModelInterfaceStates(modelId); + setModelInterface(models[modelId]?.interface); } } }, [models]); - // reusable function to update interface states based on the model ID - function updateModelInterfaceStates(modelId: string) { - const newSelectedModel = models[modelId]; - setModelInterface(newSelectedModel?.interface); - } + // persisting doc/query/index mapping fields to collect a list + // of options to display in the dropdowns when configuring input / output maps + const [docFields, setDocFields] = useState<{ label: string }[]>([]); + const [queryFields, setQueryFields] = useState<{ label: string }[]>([]); + const [indexMappingFields, setIndexMappingFields] = useState< + { label: string }[] + >([]); + useEffect(() => { + try { + const docObjKeys = Object.keys( + flattie((JSON.parse(values.ingest.docs) as {}[])[0]) + ); + if (docObjKeys.length > 0) { + setDocFields( + docObjKeys.map((key) => { + return { + label: key, + }; + }) + ); + } + } catch {} + }, [values?.ingest?.docs]); + useEffect(() => { + try { + const queryObjKeys = Object.keys( + flattie(JSON.parse(values.search.request)) + ); + if (queryObjKeys.length > 0) { + setQueryFields( + queryObjKeys.map((key) => { + return { + label: key, + }; + }) + ); + } + } catch {} + }, [values?.search?.request]); + useEffect(() => { + const indexName = values?.ingest?.index?.name as string | undefined; + if (indexName !== undefined && indices[indexName] !== undefined) { + dispatch( + getMappings({ + index: indexName, + dataSourceId, + }) + ) + .unwrap() + .then((resp: IndexMappings) => { + const mappingsObjKeys = Object.keys(resp.properties); + if (mappingsObjKeys.length > 0) { + setIndexMappingFields( + mappingsObjKeys.map((key) => { + return { + label: key, + type: resp.properties[key]?.type, + }; + }) + ); + } + }); + } + }, [values?.ingest?.index?.name]); return ( <> @@ -212,12 +277,19 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { root object selector "${JSONPATH_ROOT_SELECTOR}"`} helpLink={ML_INFERENCE_DOCS_LINK} keyPlaceholder="Model input field" + keyOptions={parseModelInputs(modelInterface)} valuePlaceholder={ props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST ? 'Query field' : 'Document field' } - keyOptions={parseModelInputs(modelInterface)} + valueOptions={ + props.context === PROCESSOR_CONTEXT.INGEST + ? docFields + : props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST + ? queryFields + : indexMappingFields + } /> @@ -262,7 +334,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { keyPlaceholder={ props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST ? 'Query field' - : 'Document field' + : 'New document field' } valuePlaceholder="Model output field" valueOptions={parseModelOutputs(modelInterface)} diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx index 14a9cde9..9ac13b15 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx @@ -3,12 +3,17 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React from 'react'; +import React, { useEffect } from 'react'; import { EuiFlexGroup, EuiFlexItem, EuiHorizontalRule } from '@elastic/eui'; import { ConfigureSearchRequest } from './configure_search_request'; import { EnrichSearchRequest } from './enrich_search_request'; import { EnrichSearchResponse } from './enrich_search_response'; -import { WorkflowConfig } from '../../../../../common'; +import { + OMIT_SYSTEM_INDEX_PATTERN, + WorkflowConfig, +} from '../../../../../common'; +import { catIndices, useAppDispatch } from '../../../../store'; +import { getDataSourceId } from '../../../../utils'; interface SearchInputsProps { uiConfig: WorkflowConfig; @@ -21,6 +26,15 @@ interface SearchInputsProps { * The base component containing all of the search-related inputs */ export function SearchInputs(props: SearchInputsProps) { + const dispatch = useAppDispatch(); + const dataSourceId = getDataSourceId(); + // re-fetch indices on initial load. When users are first creating, + // they may enter this page without getting the updated index info + // for a newly-created index, so we re-fetch that here. + useEffect(() => { + dispatch(catIndices({ pattern: OMIT_SYSTEM_INDEX_PATTERN, dataSourceId })); + }, []); + return ( diff --git a/public/pages/workflows/new_workflow/quick_configure_modal.tsx b/public/pages/workflows/new_workflow/quick_configure_modal.tsx index 537fa5e7..62a3cf1d 100644 --- a/public/pages/workflows/new_workflow/quick_configure_modal.tsx +++ b/public/pages/workflows/new_workflow/quick_configure_modal.tsx @@ -5,6 +5,9 @@ import React, { useState, useEffect } from 'react'; import { useHistory } from 'react-router-dom'; +import { isEmpty } from 'lodash'; +import { useSelector } from 'react-redux'; +import { flattie } from 'flattie'; import { EuiSmallButton, EuiModal, @@ -17,8 +20,6 @@ import { EuiCompressedFormRow, } from '@elastic/eui'; import { - DEFAULT_LABEL_FIELD, - DEFAULT_TEXT_FIELD, IMAGE_FIELD_PATTERN, IndexMappings, LABEL_FIELD_PATTERN, @@ -47,8 +48,6 @@ import { parseModelOutputs, } from '../../../utils/utils'; import { QuickConfigureInputs } from './quick_configure_inputs'; -import { isEmpty } from 'lodash'; -import { useSelector } from 'react-redux'; interface QuickConfigureModalProps { workflow: Workflow; @@ -292,37 +291,34 @@ function updateSearchRequestProcessorConfig( modelInterface: ModelInterface | undefined, isVectorSearchUseCase: boolean ): WorkflowConfig { + let defaultQueryValue = '' as string; + try { + defaultQueryValue = Object.keys( + flattie(JSON.parse(config.search?.request?.value as string)) + )[0]; + } catch {} config.search.enrichRequest.processors[0].fields.forEach((field) => { if (field.id === 'model' && fields.modelId) { field.value = { id: fields.modelId }; } if (field.id === 'input_map') { const inputMap = generateMapFromModelInputs(modelInterface); - // TODO: may change in the future. This is assuming the default query is a - // basic term query. - const defaultValue = `query.term.${ - isVectorSearchUseCase ? DEFAULT_TEXT_FIELD : DEFAULT_LABEL_FIELD - }.value`; if (inputMap.length > 0) { inputMap[0] = { ...inputMap[0], - value: defaultValue, + value: defaultQueryValue, }; } else { inputMap.push({ key: '', - value: defaultValue, + value: defaultQueryValue, }); } field.value = [inputMap] as MapArrayFormValue; } if (field.id === 'output_map') { const outputMap = generateMapFromModelOutputs(modelInterface); - // TODO: may change in the future. This is assuming the default query is a - // basic term query. - const defaultKey = isVectorSearchUseCase - ? VECTOR - : `query.term.${DEFAULT_LABEL_FIELD}.value`; + const defaultKey = isVectorSearchUseCase ? VECTOR : defaultQueryValue; if (outputMap.length > 0) { outputMap[0] = { ...outputMap[0], diff --git a/public/store/reducers/opensearch_reducer.ts b/public/store/reducers/opensearch_reducer.ts index d40e9396..66dd4d12 100644 --- a/public/store/reducers/opensearch_reducer.ts +++ b/public/store/reducers/opensearch_reducer.ts @@ -8,6 +8,7 @@ import { getRouteService } from '../../services'; import { Index, IngestPipelineConfig, + OMIT_SYSTEM_INDEX_PATTERN, SimulateIngestPipelineDoc, } from '../../../common'; import { HttpFetchError } from '../../../../../src/core/public'; @@ -33,7 +34,7 @@ export const catIndices = createAsyncThunk( { rejectWithValue } ) => { // defaulting to fetch everything except system indices (starting with '.') - const patternString = pattern || '*,-.*'; + const patternString = pattern || OMIT_SYSTEM_INDEX_PATTERN; const response: any | HttpFetchError = await getRouteService().catIndices( patternString, dataSourceId diff --git a/yarn.lock b/yarn.lock index 93aa70c0..8972f3bf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -402,6 +402,11 @@ fast-levenshtein@~2.0.6: resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== +flattie@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/flattie/-/flattie-1.1.1.tgz#88182235723113667d36217fec55359275d6fe3d" + integrity sha512-9UbaD6XdAL97+k/n+N7JwX46K/M6Zc6KcFYskrYL8wbBV/Uyk0CTAMY0VT+qiK5PM7AIc9aTWYtq65U7T+aCNQ== + formik@2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/formik/-/formik-2.4.2.tgz#a1115457cfb012a5c782cea3ad4b40b2fe36fa18"