From 037ad643222301dea8a12dab6956e0371760f282 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Fri, 26 Jul 2024 16:17:47 -0700 Subject: [PATCH] Support multiple input/output maps for ML processors (#244) Signed-off-by: Tyler Ohlsen (cherry picked from commit e9080237c9a15887a4735df718803f12a7d51a65) --- common/interfaces.ts | 5 +- public/configs/ml_processor.ts | 4 +- .../workflow_inputs/input_fields/index.ts | 1 + .../input_fields/map_array_field.tsx | 149 ++++++++++++++++++ .../input_fields/map_field.tsx | 12 +- .../input_transform_modal.tsx | 60 ++++++- .../processor_inputs/ml_processor_inputs.tsx | 19 ++- .../output_transform_modal.tsx | 63 ++++++-- public/utils/config_to_form_utils.ts | 4 +- public/utils/config_to_schema_utils.ts | 20 ++- public/utils/config_to_template_utils.ts | 35 ++-- public/utils/utils.ts | 6 +- 12 files changed, 331 insertions(+), 47 deletions(-) create mode 100644 public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx diff --git a/common/interfaces.ts b/common/interfaces.ts index e5bc1914..1ab38b5b 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -23,7 +23,8 @@ export type ConfigFieldType = | 'jsonArray' | 'select' | 'model' - | 'map'; + | 'map' + | 'mapArray'; export type ConfigFieldValue = string | {}; export interface IConfigField { type: ConfigFieldType; @@ -81,6 +82,8 @@ export type MapEntry = { export type MapFormValue = MapEntry[]; +export type MapArrayFormValue = MapFormValue[]; + export type WorkflowFormValues = { ingest: FormikValues; search: FormikValues; diff --git a/public/configs/ml_processor.ts b/public/configs/ml_processor.ts index 53a6bef2..ed2065a9 100644 --- a/public/configs/ml_processor.ts +++ b/public/configs/ml_processor.ts @@ -22,11 +22,11 @@ export abstract class MLProcessor extends Processor { }, { id: 'inputMap', - type: 'map', + type: 'mapArray', }, { id: 'outputMap', - type: 'map', + type: 'mapArray', }, ]; } diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/index.ts b/public/pages/workflow_detail/workflow_inputs/input_fields/index.ts index ab356acc..75e2f859 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/index.ts +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/index.ts @@ -7,4 +7,5 @@ export { TextField } from './text_field'; export { JsonField } from './json_field'; export { ModelField } from './model_field'; export { MapField } from './map_field'; +export { MapArrayField } from './map_array_field'; export { BooleanField } from './boolean_field'; diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx new file mode 100644 index 00000000..4e2f8fed --- /dev/null +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx @@ -0,0 +1,149 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { + EuiAccordion, + EuiButton, + EuiButtonIcon, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, + EuiLink, + EuiPanel, + EuiText, +} from '@elastic/eui'; +import { Field, FieldProps, getIn, useFormikContext } from 'formik'; +import { + IConfigField, + MapArrayFormValue, + MapEntry, + WorkflowFormValues, +} from '../../../../../common'; +import { MapField } from './map_field'; + +interface MapArrayFieldProps { + field: IConfigField; + fieldPath: string; // the full path in string-form to the field (e.g., 'ingest.enrich.processors.text_embedding_processor.inputField') + label?: string; + helpLink?: string; + helpText?: string; + keyPlaceholder?: string; + valuePlaceholder?: string; + onFormChange: () => void; + onMapAdd?: (curArray: MapArrayFormValue) => void; + onMapDelete?: (idxToDelete: number) => void; +} + +/** + * Input component for configuring an array of field mappings + */ +export function MapArrayField(props: MapArrayFieldProps) { + const { setFieldValue, setFieldTouched, errors, touched } = useFormikContext< + WorkflowFormValues + >(); + + // Adding a map to the end of the existing arr + function addMap(curMapArray: MapArrayFormValue): void { + setFieldValue(props.fieldPath, [...curMapArray, []]); + setFieldTouched(props.fieldPath, true); + props.onFormChange(); + if (props.onMapAdd) { + props.onMapAdd(curMapArray); + } + } + + // Deleting a map + function deleteMap( + curMapArray: MapArrayFormValue, + entryIndexToDelete: number + ): void { + const updatedMapArray = [...curMapArray]; + updatedMapArray.splice(entryIndexToDelete, 1); + setFieldValue(props.fieldPath, updatedMapArray); + setFieldTouched(props.fieldPath, true); + props.onFormChange(); + if (props.onMapDelete) { + props.onMapDelete(entryIndexToDelete); + } + } + + return ( + + {({ field, form }: FieldProps) => { + return ( + + + Learn more + + + ) : undefined + } + helpText={props.helpText || undefined} + isInvalid={ + getIn(errors, field.name) !== undefined && + getIn(errors, field.name).length > 0 && + getIn(touched, field.name) !== undefined && + getIn(touched, field.name).length > 0 + } + > + + {field.value?.map((mapping: MapEntry, idx: number) => { + return ( + + { + deleteMap(field.value, idx); + }} + /> + } + > + + + + + + ); + })} + +
+ { + addMap(field.value); + }} + > + {field.value?.length > 0 ? 'Add another map' : 'Add map'} + +
+ + + + ); + }} + + ); +} diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx index a51b761a..3959170c 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx @@ -16,16 +16,14 @@ import { } from '@elastic/eui'; import { Field, FieldProps, getIn, useFormikContext } from 'formik'; import { - IConfigField, MapEntry, MapFormValue, WorkflowFormValues, } from '../../../../../common'; interface MapFieldProps { - field: IConfigField; fieldPath: string; // the full path in string-form to the field (e.g., 'ingest.enrich.processors.text_embedding_processor.inputField') - label: string; + label?: string; helpLink?: string; helpText?: string; keyPlaceholder?: string; @@ -62,10 +60,11 @@ export function MapField(props: MapFieldProps) { } return ( - + {({ field, form }: FieldProps) => { return ( { return ( - - + + void; } +// TODO: InputTransformModal and OutputTransformModal are very similar, and can +// likely be refactored and have more reusable components. Leave as-is until the +// UI is more finalized. + /** * A modal to configure advanced JSON-to-JSON transforms into a model's expected input */ @@ -59,10 +66,19 @@ export function InputTransformModal(props: InputTransformModalProps) { // source input / transformed output state const [sourceInput, setSourceInput] = useState('[]'); - const [transformedOutput, setTransformedOutput] = useState('[]'); + const [transformedOutput, setTransformedOutput] = useState('{}'); // get the current input map - const map = getIn(values, `ingest.enrich.${props.config.id}.inputMap`); + const map = getIn(values, props.inputMapFieldPath) as MapArrayFormValue; + + // selected output state + const outputOptions = map.map((_, idx) => ({ + value: idx, + text: `Prediction ${idx + 1}`, + })) as EuiSelectOption[]; + const [selectedOutputOption, setSelectedOutputOption] = useState< + number | undefined + >((outputOptions[0]?.value as number) ?? undefined); return ( @@ -149,34 +165,62 @@ export function InputTransformModal(props: InputTransformModalProps) { root object selector "${JSONPATH_ROOT_SELECTOR}"`} - { + if (isEmpty(curArray)) { + setSelectedOutputOption(0); + } + }} + // If the map we are deleting is the one we last used to test, reset the state and + // default to the first map in the list. + onMapDelete={(idxToDelete) => { + if (selectedOutputOption === idxToDelete) { + setSelectedOutputOption(0); + setTransformedOutput('{}'); + } + }} /> <> - Expected output + Expected output for} + compressed={true} + options={outputOptions} + value={selectedOutputOption} + onChange={(e) => { + setSelectedOutputOption(Number(e.target.value)); + setTransformedOutput('{}'); + }} + /> + { switch (props.context) { case PROCESSOR_CONTEXT.INGEST: { - if (!isEmpty(map) && !isEmpty(JSON.parse(sourceInput))) { + if ( + !isEmpty(map) && + !isEmpty(JSON.parse(sourceInput)) && + selectedOutputOption !== undefined + ) { let sampleSourceInput = {}; try { sampleSourceInput = JSON.parse(sourceInput)[0]; const output = generateTransform( sampleSourceInput, - map + map[selectedOutputOption] ); setTransformedOutput( JSON.stringify(output, undefined, 2) diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx index adf7ca89..ba6640e2 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx @@ -7,6 +7,7 @@ import React, { useState } from 'react'; import { getIn, useFormikContext } from 'formik'; import { EuiButtonEmpty, + EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiSpacer, @@ -21,7 +22,7 @@ import { JSONPATH_ROOT_SELECTOR, ML_INFERENCE_DOCS_LINK, } from '../../../../../common'; -import { MapField, ModelField } from '../input_fields'; +import { MapArrayField, ModelField } from '../input_fields'; import { isEmpty } from 'lodash'; import { InputTransformModal } from './input_transform_modal'; import { OutputTransformModal } from './output_transform_modal'; @@ -51,10 +52,12 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { (field) => field.id === 'inputMap' ) as IConfigField; const inputMapFieldPath = `${props.baseConfigPath}.${props.config.id}.${inputMapField.id}`; + const inputMapValue = getIn(values, inputMapFieldPath); const outputMapField = props.config.fields.find( (field) => field.id === 'outputMap' ) as IConfigField; const outputMapFieldPath = `${props.baseConfigPath}.${props.config.id}.${outputMapField.id}`; + const outputMapValue = getIn(values, outputMapFieldPath); // advanced transformations modal state const [isInputTransformModalOpen, setIsInputTransformModalOpen] = useState< @@ -121,7 +124,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { root object selector "${JSONPATH_ROOT_SELECTOR}"`} - - + {inputMapValue.length !== outputMapValue.length && + inputMapValue.length > 0 && + outputMapValue.length > 0 && ( + + )} )} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx index f7f6b21b..d2232115 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx @@ -16,6 +16,8 @@ import { EuiModalFooter, EuiModalHeader, EuiModalHeaderTitle, + EuiSelect, + EuiSelectOption, EuiSpacer, EuiText, } from '@elastic/eui'; @@ -24,6 +26,8 @@ import { IProcessorConfig, IngestPipelineConfig, JSONPATH_ROOT_SELECTOR, + ML_INFERENCE_DOCS_LINK, + MapArrayFormValue, PROCESSOR_CONTEXT, SimulateIngestPipelineResponse, WorkflowConfig, @@ -37,7 +41,7 @@ import { } from '../../../../utils'; import { simulatePipeline, useAppDispatch } from '../../../../store'; import { getCore } from '../../../../services'; -import { MapField } from '../input_fields'; +import { MapArrayField } from '../input_fields'; interface OutputTransformModalProps { uiConfig: WorkflowConfig; @@ -58,10 +62,19 @@ export function OutputTransformModal(props: OutputTransformModalProps) { // source input / transformed output state const [sourceInput, setSourceInput] = useState('[]'); - const [transformedOutput, setTransformedOutput] = useState('[]'); + const [transformedOutput, setTransformedOutput] = useState('{}'); // get the current output map - const map = getIn(values, `ingest.enrich.${props.config.id}.outputMap`); + const map = getIn(values, props.outputMapFieldPath) as MapArrayFormValue; + + // selected output state + const outputOptions = map.map((_, idx) => ({ + value: idx, + text: `Prediction output ${idx + 1}`, + })) as EuiSelectOption[]; + const [selectedOutputOption, setSelectedOutputOption] = useState< + number | undefined + >((outputOptions[0]?.value as number) ?? undefined); return ( @@ -149,36 +162,62 @@ export function OutputTransformModal(props: OutputTransformModalProps) { root object selector "${JSONPATH_ROOT_SELECTOR}"`} - { + if (isEmpty(curArray)) { + setSelectedOutputOption(0); + } + }} + // If the map we are deleting is the one we last used to test, reset the state and + // default to the first map in the list. + onMapDelete={(idxToDelete) => { + if (selectedOutputOption === idxToDelete) { + setSelectedOutputOption(0); + setTransformedOutput('{}'); + } + }} /> <> - Expected output + Expected output for} + compressed={true} + options={outputOptions} + value={selectedOutputOption} + onChange={(e) => { + setSelectedOutputOption(Number(e.target.value)); + setTransformedOutput('{}'); + }} + /> + { switch (props.context) { case PROCESSOR_CONTEXT.INGEST: { - if (!isEmpty(map) && !isEmpty(JSON.parse(sourceInput))) { + if ( + !isEmpty(map) && + !isEmpty(JSON.parse(sourceInput)) && + selectedOutputOption !== undefined + ) { let sampleSourceInput = {}; try { sampleSourceInput = JSON.parse(sourceInput)[0]; const output = generateTransform( sampleSourceInput, - map + map[selectedOutputOption] ); setTransformedOutput( JSON.stringify(output, undefined, 2) diff --git a/public/utils/config_to_form_utils.ts b/public/utils/config_to_form_utils.ts index 57c1a73d..3fdcfcab 100644 --- a/public/utils/config_to_form_utils.ts +++ b/public/utils/config_to_form_utils.ts @@ -15,7 +15,6 @@ import { ConfigFieldType, ConfigFieldValue, ModelFormValue, - FETCH_ALL_QUERY_BODY, } from '../../common'; /* @@ -123,5 +122,8 @@ export function getInitialValue(fieldType: ConfigFieldType): ConfigFieldValue { case 'jsonArray': { return '[]'; } + case 'mapArray': { + return []; + } } } diff --git a/public/utils/config_to_schema_utils.ts b/public/utils/config_to_schema_utils.ts index 9ccb60c0..b5fbd1b1 100644 --- a/public/utils/config_to_schema_utils.ts +++ b/public/utils/config_to_schema_utils.ts @@ -127,7 +127,6 @@ function getFieldSchema(fieldType: ConfigFieldType): Schema { try { // @ts-ignore return Array.isArray(JSON.parse(value)); - return true; } catch (error) { return false; } @@ -135,6 +134,25 @@ function getFieldSchema(fieldType: ConfigFieldType): Schema { break; } + case 'mapArray': { + baseSchema = yup.array().of( + yup.array().of( + yup.object().shape({ + key: yup + .string() + .min(1, 'Too short') + .max(70, 'Too long') + .required(), + value: yup + .string() + .min(1, 'Too short') + .max(70, 'Too long') + .required(), + }) + ) + ); + break; + } } // TODO: make optional schema if we support optional fields in the future diff --git a/public/utils/config_to_template_utils.ts b/public/utils/config_to_template_utils.ts index 641ffa65..f172c13a 100644 --- a/public/utils/config_to_template_utils.ts +++ b/public/utils/config_to_template_utils.ts @@ -17,7 +17,7 @@ import { IndexConfig, IProcessorConfig, MLInferenceProcessor, - MapFormValue, + MapArrayFormValue, IngestProcessor, Workflow, WorkflowTemplate, @@ -25,6 +25,8 @@ import { SearchProcessor, IngestConfig, SearchConfig, + MapFormValue, + MapEntry, } from '../../common'; import { processorConfigToFormik } from './config_to_form_utils'; import { generateId } from './utils'; @@ -145,8 +147,8 @@ export function processorConfigsToTemplateProcessors( processorConfig ) as { model: ModelFormValue; - inputMap: MapFormValue; - outputMap: MapFormValue; + inputMap: MapArrayFormValue; + outputMap: MapArrayFormValue; }; let processor = { @@ -155,14 +157,15 @@ export function processorConfigsToTemplateProcessors( }, } as MLInferenceProcessor; if (inputMap?.length > 0) { - processor.ml_inference.input_map = inputMap.map((mapEntry) => ({ - [mapEntry.key]: mapEntry.value, - })); + processor.ml_inference.input_map = inputMap.map((mapFormValue) => + mergeMapIntoSingleObj(mapFormValue) + ); } + if (outputMap?.length > 0) { - processor.ml_inference.output_map = outputMap.map((mapEntry) => ({ - [mapEntry.key]: mapEntry.value, - })); + processor.ml_inference.output_map = outputMap.map((mapFormValue) => + mergeMapIntoSingleObj(mapFormValue) + ); } processorsList.push(processor); @@ -249,3 +252,17 @@ export function reduceToTemplate(workflow: Workflow): WorkflowTemplate { } = workflow; return workflowTemplate; } + +// Helper fn to merge the form map (an arr of objs) into a single obj, such that each key +// is an obj property, and each value is a property value. Used to format into the +// expected inputs for input_maps and output_maps of the ML inference processors. +function mergeMapIntoSingleObj(mapFormValue: MapFormValue): {} { + let curMap = {} as MapEntry; + mapFormValue.forEach((mapEntry) => { + curMap = { + ...curMap, + [mapEntry.key]: mapEntry.value, + }; + }); + return curMap; +} diff --git a/public/utils/utils.ts b/public/utils/utils.ts index 4bd2b8b3..3473f495 100644 --- a/public/utils/utils.ts +++ b/public/utils/utils.ts @@ -8,6 +8,7 @@ import jsonpath from 'jsonpath'; import { get } from 'lodash'; import { JSONPATH_ROOT_SELECTOR, + MapFormValue, SimulateIngestPipelineDoc, SimulateIngestPipelineResponse, WORKFLOW_RESOURCE_TYPE, @@ -157,10 +158,7 @@ export function unwrapTransformedDocs( // ML inference processors will use standard dot notation or JSONPath depending on the input. // We follow the same logic here to generate consistent results. -export function generateTransform( - input: {}, - map: { key: string; value: string }[] -): {} { +export function generateTransform(input: {}, map: MapFormValue): {} { let output = {}; map.forEach((mapEntry) => { const path = mapEntry.value;