From e40f057b55a730289b606d39d9a26fdb6189797b Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 26 Aug 2024 12:32:36 -0700 Subject: [PATCH] onboard normalization processor (#311) Signed-off-by: Tyler Ohlsen (cherry picked from commit e913e410307471e91e779699ae4c8f91fefe8e3a) --- common/constants.ts | 3 + .../search_response_processors/index.ts | 1 + .../normalization_processor.ts | 43 ++++++++++++ .../normalization_processor_inputs.tsx | 65 +++++++++++++++++++ .../processor_inputs/processor_inputs.tsx | 15 +++++ .../workflow_inputs/processors_list.tsx | 10 +++ .../new_workflow/quick_configure_inputs.tsx | 2 +- public/utils/config_to_template_utils.ts | 63 +++++++++++++++++- 8 files changed, 199 insertions(+), 3 deletions(-) create mode 100644 public/configs/search_response_processors/normalization_processor.ts create mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/normalization_processor_inputs.tsx diff --git a/common/constants.ts b/common/constants.ts index b30be6bd..cf9caeac 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -73,6 +73,7 @@ export enum PROCESSOR_TYPE { SPLIT = 'split', SORT = 'sort', TEXT_CHUNKING = 'text_chunking', + NORMALIZATION = 'normalization-processor', } export enum MODEL_TYPE { @@ -129,6 +130,8 @@ export const TEXT_CHUNKING_PROCESSOR_LINK = 'https://opensearch.org/docs/latest/ingest-pipelines/processors/text-chunking/'; export const CREATE_WORKFLOW_LINK = 'https://opensearch.org/docs/latest/automating-configurations/api/create-workflow/'; +export const NORMALIZATION_PROCESSOR_LINK = + 'https://opensearch.org/docs/latest/search-plugins/search-pipelines/normalization-processor/'; /** * Text chunking algorithm constants diff --git a/public/configs/search_response_processors/index.ts b/public/configs/search_response_processors/index.ts index e8205e7d..71a5ece5 100644 --- a/public/configs/search_response_processors/index.ts +++ b/public/configs/search_response_processors/index.ts @@ -6,3 +6,4 @@ export * from './ml_search_response_processor'; export * from './split_search_response_processor'; export * from './sort_search_response_processor'; +export * from './normalization_processor'; diff --git a/public/configs/search_response_processors/normalization_processor.ts b/public/configs/search_response_processors/normalization_processor.ts new file mode 100644 index 00000000..c56cb1a7 --- /dev/null +++ b/public/configs/search_response_processors/normalization_processor.ts @@ -0,0 +1,43 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { PROCESSOR_TYPE } from '../../../common'; +import { Processor } from '../processor'; + +/** + * The normalization processor config. Used in search flows. + */ +export class NormalizationProcessor extends Processor { + constructor() { + super(); + this.type = PROCESSOR_TYPE.NORMALIZATION; + this.name = 'Normalization Processor'; + this.fields = []; + this.optionalFields = [ + { + id: 'weights', + type: 'string', + }, + { + id: 'normalization_technique', + type: 'select', + selectOptions: ['min_max', 'l2'], + }, + { + id: 'combination_technique', + type: 'select', + selectOptions: ['arithmetic_mean', 'geometric_mean', 'harmonic_mean'], + }, + { + id: 'description', + type: 'string', + }, + { + id: 'tag', + type: 'string', + }, + ]; + } +} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/normalization_processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/normalization_processor_inputs.tsx new file mode 100644 index 00000000..253952e6 --- /dev/null +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/normalization_processor_inputs.tsx @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { EuiAccordion, EuiFlexItem, EuiSpacer } from '@elastic/eui'; +import { + IProcessorConfig, + PROCESSOR_CONTEXT, + WorkflowConfig, + NORMALIZATION_PROCESSOR_LINK, +} from '../../../../../common'; +import { TextField } from '../input_fields'; +import { ConfigFieldList } from '../config_field_list'; + +interface NormalizationProcessorInputsProps { + uiConfig: WorkflowConfig; + config: IProcessorConfig; + baseConfigPath: string; // the base path of the nested config, if applicable. e.g., 'ingest.enrich' + context: PROCESSOR_CONTEXT; +} + +/** + * Specialized component to render the normalization processor. Adds some helper text around weights field. + * In the future, may have a more customizable / guided way for specifying the array of weights. + * For example, could have some visual way of linking it to the underlying sub-queries in the query field, + * enforce its length = the number of queries, etc. + */ +export function NormalizationProcessorInputs( + props: NormalizationProcessorInputsProps +) { + // extracting field info from the config + const optionalFields = props.config.optionalFields || []; + const weightsFieldPath = `${props.baseConfigPath}.${props.config.id}.weights`; + const optionalFieldsWithoutWeights = optionalFields.filter( + (field) => field.id !== 'weights' + ); + + return ( + // We only have optional fields for this processor, so everything is nested under the accordion + + + + + + + + + ); +} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx index 0f941340..99d7dffd 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx @@ -15,6 +15,7 @@ import { import { MLProcessorInputs } from './ml_processor_inputs'; import { ConfigFieldList } from '../config_field_list'; import { TextChunkingProcessorInputs } from './text_chunking_processor_inputs'; +import { NormalizationProcessorInputs } from './normalization_processor_inputs'; /** * Base component for rendering processor form inputs based on the processor type @@ -69,6 +70,20 @@ export function ProcessorInputs(props: ProcessorInputsProps) { ); break; } + case PROCESSOR_TYPE.NORMALIZATION: { + el = ( + + + + + ); + break; + } default: { el = ( diff --git a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx index 64ac0be0..a69d1a4f 100644 --- a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx @@ -28,6 +28,7 @@ import { MLIngestProcessor, MLSearchRequestProcessor, MLSearchResponseProcessor, + NormalizationProcessor, SortIngestProcessor, SortSearchResponseProcessor, SplitIngestProcessor, @@ -272,6 +273,15 @@ export function ProcessorsList(props: ProcessorsListProps) { ); }, }, + { + name: 'Normalization Processor', + onClick: () => { + closePopover(); + addProcessor( + new NormalizationProcessor().toObj() + ); + }, + }, ], }, ]} diff --git a/public/pages/workflows/new_workflow/quick_configure_inputs.tsx b/public/pages/workflows/new_workflow/quick_configure_inputs.tsx index 5233ae6f..6aa8c49f 100644 --- a/public/pages/workflows/new_workflow/quick_configure_inputs.tsx +++ b/public/pages/workflows/new_workflow/quick_configure_inputs.tsx @@ -65,7 +65,7 @@ export function QuickConfigureInputs(props: QuickConfigureInputsProps) { processor.type === PROCESSOR_TYPE.NORMALIZATION + ); + const phaseResultsProcessors = processorConfigsToTemplateProcessors( + normalizationProcessor ? [normalizationProcessor] : [] + ); const searchResponseProcessors = processorConfigsToTemplateProcessors( - searchConfig.enrichResponse.processors + searchConfig.enrichResponse.processors.filter( + (processor) => processor.type !== PROCESSOR_TYPE.NORMALIZATION + ) ); const hasProcessors = - searchRequestProcessors.length > 0 || searchResponseProcessors.length > 0; + searchRequestProcessors.length > 0 || + searchResponseProcessors.length > 0 || + phaseResultsProcessors.length > 0; return hasProcessors ? [ @@ -133,6 +148,7 @@ function searchConfigToTemplateNodes( configurations: JSON.stringify({ request_processors: searchRequestProcessors, response_processors: searchResponseProcessors, + phase_results_processors: phaseResultsProcessors, } as SearchPipelineConfig), }, } as CreateSearchPipelineNode, @@ -254,6 +270,49 @@ export function processorConfigsToTemplateProcessors( }); break; } + // optionally add any parameters specified, in the expected nested format + // for the normalization processor + case PROCESSOR_TYPE.NORMALIZATION: { + const { + normalization_technique, + combination_technique, + weights, + } = processorConfigToFormik(processorConfig); + + let finalConfig = {} as any; + if (!isEmpty(normalization_technique)) { + finalConfig = { + ...finalConfig, + normalization: { + technique: normalization_technique, + }, + }; + } + if (!isEmpty(combination_technique)) { + finalConfig = { + ...finalConfig, + combination: { + ...(finalConfig?.combination || {}), + technique: combination_technique, + }, + }; + } + if (!isEmpty(weights) && weights.split(',').length > 0) { + finalConfig = { + ...finalConfig, + combination: { + ...(finalConfig?.combination || {}), + parameters: { + weights: weights.split(',').map(Number), + }, + }, + }; + } + processorsList.push({ + [processorConfig.type]: finalConfig, + }); + break; + } case PROCESSOR_TYPE.SPLIT: case PROCESSOR_TYPE.SORT: default: {