diff --git a/common/constants.ts b/common/constants.ts index 2d0bf50d..52d26775 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -90,6 +90,7 @@ export enum WORKFLOW_TYPE { SEMANTIC_SEARCH = 'Semantic search', MULTIMODAL_SEARCH = 'Multimodal search', HYBRID_SEARCH = 'Hybrid search', + SENTIMENT_ANALYSIS = 'Sentiment analysis', CUSTOM = 'Custom', UNKNOWN = 'Unknown', } @@ -181,9 +182,14 @@ export const SHARED_OPTIONAL_FIELDS = ['max_chunk_limit', 'description', 'tag']; /** * QUERY PRESETS */ +export const DEFAULT_TEXT_FIELD = 'my_text'; +export const DEFAULT_VECTOR_FIELD = 'my_embedding'; +export const DEFAULT_IMAGE_FIELD = 'my_image'; +export const DEFAULT_LABEL_FIELD = 'label'; export const VECTOR_FIELD_PATTERN = `{{vector_field}}`; export const TEXT_FIELD_PATTERN = `{{text_field}}`; export const IMAGE_FIELD_PATTERN = `{{image_field}}`; +export const LABEL_FIELD_PATTERN = `{{label_field}}`; export const QUERY_TEXT_PATTERN = `{{query_text}}`; export const QUERY_IMAGE_PATTERN = `{{query_image}}`; export const MODEL_ID_PATTERN = `{{model_id}}`; @@ -198,7 +204,7 @@ export const FETCH_ALL_QUERY = { }, size: 1000, }; -export const TERM_QUERY = { +export const TERM_QUERY_TEXT = { query: { term: { [TEXT_FIELD_PATTERN]: { @@ -207,6 +213,15 @@ export const TERM_QUERY = { }, }, }; +export const TERM_QUERY_LABEL = { + query: { + term: { + [LABEL_FIELD_PATTERN]: { + value: QUERY_TEXT_PATTERN, + }, + }, + }, +}; export const KNN_QUERY = { _source: { excludes: [VECTOR_FIELD_PATTERN], @@ -353,7 +368,7 @@ export const QUERY_PRESETS = [ }, { name: 'Term', - query: customStringify(TERM_QUERY), + query: customStringify(TERM_QUERY_TEXT), }, { name: 'Basic k-NN', @@ -395,6 +410,7 @@ export const DEFAULT_NEW_WORKFLOW_STATE = WORKFLOW_STATE.NOT_STARTED; export const DEFAULT_NEW_WORKFLOW_STATE_TYPE = ('NOT_STARTED' as any) as typeof WORKFLOW_STATE; export const DATE_FORMAT_PATTERN = 'MM/DD/YY hh:mm A'; export const EMPTY_FIELD_STRING = '--'; +export const OMIT_SYSTEM_INDEX_PATTERN = '*,-.*'; export const INDEX_NOT_FOUND_EXCEPTION = 'index_not_found_exception'; export const ERROR_GETTING_WORKFLOW_MSG = 'Failed to retrieve template'; export const NO_TEMPLATES_FOUND_MSG = 'There are no templates'; diff --git a/common/interfaces.ts b/common/interfaces.ts index 2d46da26..4fe68f4b 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -483,10 +483,11 @@ export type QueryPreset = { }; export type QuickConfigureFields = { - embeddingModelId?: string; + modelId?: string; vectorField?: string; textField?: string; imageField?: string; + labelField?: string; embeddingLength?: number; }; diff --git a/package.json b/package.json index 31ab2856..44394399 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ }, "dependencies": { "@types/jsonpath": "^0.2.4", + "flattie": "^1.1.1", "formik": "2.4.2", "jsonpath": "^1.1.1", "reactflow": "^11.8.3", diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx index a0ec10df..13d6468f 100644 --- a/public/pages/workflow_detail/workflow_detail.tsx +++ b/public/pages/workflow_detail/workflow_detail.tsx @@ -32,6 +32,7 @@ import { FETCH_ALL_QUERY, MAX_WORKFLOW_NAME_TO_DISPLAY, NO_TEMPLATES_FOUND_MSG, + OMIT_SYSTEM_INDEX_PATTERN, getCharacterLimitedString, } from '../../../common'; import { MountPoint } from '../../../../../src/core/public'; @@ -106,7 +107,7 @@ export function WorkflowDetail(props: WorkflowDetailProps) { useEffect(() => { dispatch(getWorkflow({ workflowId, dataSourceId })); dispatch(searchModels({ apiBody: FETCH_ALL_QUERY, dataSourceId })); - dispatch(catIndices({ pattern: '*,-.*', dataSourceId })); + dispatch(catIndices({ pattern: OMIT_SYSTEM_INDEX_PATTERN, dataSourceId })); }, []); return errorMessage.includes(ERROR_GETTING_WORKFLOW_MSG) || diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx index 11cb8cd3..6971e65e 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx @@ -37,8 +37,8 @@ interface MapArrayFieldProps { valuePlaceholder?: string; onMapAdd?: (curArray: MapArrayFormValue) => void; onMapDelete?: (idxToDelete: number) => void; - keyOptions?: any[]; - valueOptions?: any[]; + keyOptions?: { label: string }[]; + valueOptions?: { label: string }[]; } /** diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx index be9b43cd..8d026d83 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx @@ -31,8 +31,8 @@ interface MapFieldProps { helpText?: string; keyPlaceholder?: string; valuePlaceholder?: string; - keyOptions?: any[]; - valueOptions?: any[]; + keyOptions?: { label: string }[]; + valueOptions?: { label: string }[]; } /** diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx index efa11cc6..ac698805 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx @@ -12,7 +12,7 @@ import { WorkspaceFormValues } from '../../../../../common'; interface SelectWithCustomOptionsProps { fieldPath: string; placeholder: string; - options: any[]; + options: { label: string }[]; } /** @@ -50,13 +50,15 @@ export function SelectWithCustomOptions(props: SelectWithCustomOptionsProps) { return ( - {option.label} - - - - {`(${option.type || 'unknown type'})`} - + {option.label || ''} + {option.type && ( + + + {`(${option.type})`} + + + )} ); } diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx index 7e4033af..2aac3a98 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx @@ -68,6 +68,7 @@ interface InputTransformModalProps { inputMapField: IConfigField; inputMapFieldPath: string; modelInterface: ModelInterface | undefined; + valueOptions: { label: string }[]; onClose: () => void; } @@ -163,7 +164,7 @@ export function InputTransformModal(props: InputTransformModalProps) { Fetch some sample input data and see how it is transformed. - Expected input + Source input { @@ -266,7 +267,7 @@ export function InputTransformModal(props: InputTransformModalProps) { }) .catch((error: any) => { getCore().notifications.toasts.addDanger( - `Failed to fetch input data` + `Failed to fetch source input data` ); }); break; @@ -307,12 +308,13 @@ export function InputTransformModal(props: InputTransformModalProps) { root object selector "${JSONPATH_ROOT_SELECTOR}"`} helpLink={ML_INFERENCE_DOCS_LINK} keyPlaceholder="Model input field" + keyOptions={parseModelInputs(props.modelInterface)} valuePlaceholder={ props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST ? 'Query field' : 'Document field' } - keyOptions={parseModelInputs(props.modelInterface)} + valueOptions={props.valueOptions} // If the map we are adding is the first one, populate the selected option to index 0 onMapAdd={(curArray) => { if (isEmpty(curArray)) { @@ -356,10 +358,10 @@ export function InputTransformModal(props: InputTransformModalProps) { )} {outputOptions.length === 1 ? ( - Expected output + Transformed input ) : ( Expected output for} + prepend={Transformed input for} options={outputOptions} value={selectedOutputOption} onChange={(e) => { diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx index 7baf2aee..bc6fa033 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx @@ -5,7 +5,9 @@ import React, { useState, useEffect } from 'react'; import { getIn, useFormikContext } from 'formik'; +import { isEmpty } from 'lodash'; import { useSelector } from 'react-redux'; +import { flattie } from 'flattie'; import { EuiAccordion, EuiSmallButtonEmpty, @@ -25,14 +27,15 @@ import { ML_INFERENCE_DOCS_LINK, WorkflowFormValues, ModelInterface, + IndexMappings, } from '../../../../../common'; import { MapArrayField, ModelField } from '../input_fields'; -import { isEmpty } from 'lodash'; import { InputTransformModal } from './input_transform_modal'; import { OutputTransformModal } from './output_transform_modal'; -import { AppState } from '../../../../store'; +import { AppState, getMappings, useAppDispatch } from '../../../../store'; import { formikToPartialPipeline, + getDataSourceId, parseModelInputs, parseModelOutputs, } from '../../../../utils'; @@ -52,7 +55,10 @@ interface MLProcessorInputsProps { * output map configuration forms, respectively. */ export function MLProcessorInputs(props: MLProcessorInputsProps) { + const dispatch = useAppDispatch(); + const dataSourceId = getDataSourceId(); const models = useSelector((state: AppState) => state.ml.models); + const indices = useSelector((state: AppState) => state.opensearch.indices); const { values, setFieldValue, setFieldTouched } = useFormikContext< WorkflowFormValues >(); @@ -115,7 +121,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { // 1: update model interface states // 2. clear out any persisted input_map/output_map form values, as those would now be invalid function onModelChange(modelId: string) { - updateModelInterfaceStates(modelId); + setModelInterface(models[modelId]?.interface); setFieldValue(inputMapFieldPath, []); setFieldValue(outputMapFieldPath, []); setFieldTouched(inputMapFieldPath, false); @@ -127,16 +133,75 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { if (!isEmpty(models)) { const modelId = getIn(values, `${modelFieldPath}.id`); if (modelId) { - updateModelInterfaceStates(modelId); + setModelInterface(models[modelId]?.interface); } } }, [models]); - // reusable function to update interface states based on the model ID - function updateModelInterfaceStates(modelId: string) { - const newSelectedModel = models[modelId]; - setModelInterface(newSelectedModel?.interface); - } + // persisting doc/query/index mapping fields to collect a list + // of options to display in the dropdowns when configuring input / output maps + const [docFields, setDocFields] = useState<{ label: string }[]>([]); + const [queryFields, setQueryFields] = useState<{ label: string }[]>([]); + const [indexMappingFields, setIndexMappingFields] = useState< + { label: string }[] + >([]); + useEffect(() => { + try { + const docObjKeys = Object.keys( + flattie((JSON.parse(values.ingest.docs) as {}[])[0]) + ); + if (docObjKeys.length > 0) { + setDocFields( + docObjKeys.map((key) => { + return { + label: key, + }; + }) + ); + } + } catch {} + }, [values?.ingest?.docs]); + useEffect(() => { + try { + const queryObjKeys = Object.keys( + flattie(JSON.parse(values.search.request)) + ); + if (queryObjKeys.length > 0) { + setQueryFields( + queryObjKeys.map((key) => { + return { + label: key, + }; + }) + ); + } + } catch {} + }, [values?.search?.request]); + useEffect(() => { + const indexName = values?.search?.index?.name as string | undefined; + if (indexName !== undefined && indices[indexName] !== undefined) { + dispatch( + getMappings({ + index: indexName, + dataSourceId, + }) + ) + .unwrap() + .then((resp: IndexMappings) => { + const mappingsObjKeys = Object.keys(resp.properties); + if (mappingsObjKeys.length > 0) { + setIndexMappingFields( + mappingsObjKeys.map((key) => { + return { + label: key, + type: resp.properties[key]?.type, + }; + }) + ); + } + }); + } + }, [values?.search?.index?.name]); return ( <> @@ -148,6 +213,13 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { inputMapField={inputMapField} inputMapFieldPath={inputMapFieldPath} modelInterface={modelInterface} + valueOptions={ + props.context === PROCESSOR_CONTEXT.INGEST + ? docFields + : props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST + ? queryFields + : indexMappingFields + } onClose={() => setIsInputTransformModalOpen(false)} /> )} @@ -212,12 +284,19 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { root object selector "${JSONPATH_ROOT_SELECTOR}"`} helpLink={ML_INFERENCE_DOCS_LINK} keyPlaceholder="Model input field" + keyOptions={parseModelInputs(modelInterface)} valuePlaceholder={ props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST ? 'Query field' : 'Document field' } - keyOptions={parseModelInputs(modelInterface)} + valueOptions={ + props.context === PROCESSOR_CONTEXT.INGEST + ? docFields + : props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST + ? queryFields + : indexMappingFields + } /> @@ -262,7 +341,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { keyPlaceholder={ props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST ? 'Query field' - : 'Document field' + : 'New document field' } valuePlaceholder="Model output field" valueOptions={parseModelOutputs(modelInterface)} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx index 393459c1..56a7408e 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx @@ -126,7 +126,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) { Fetch some sample output data and see how it is transformed. - Expected input + Source output { @@ -219,7 +219,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) { }) .catch((error: any) => { getCore().notifications.toasts.addDanger( - `Failed to fetch input data` + `Failed to fetch source output data` ); }); break; @@ -282,10 +282,10 @@ export function OutputTransformModal(props: OutputTransformModalProps) { <> {outputOptions.length === 1 ? ( - Expected output + Transformed output ) : ( Expected output for} + prepend={Transformed output for} options={outputOptions} value={selectedOutputOption} onChange={(e) => { diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx index 14a9cde9..9ac13b15 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx @@ -3,12 +3,17 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React from 'react'; +import React, { useEffect } from 'react'; import { EuiFlexGroup, EuiFlexItem, EuiHorizontalRule } from '@elastic/eui'; import { ConfigureSearchRequest } from './configure_search_request'; import { EnrichSearchRequest } from './enrich_search_request'; import { EnrichSearchResponse } from './enrich_search_response'; -import { WorkflowConfig } from '../../../../../common'; +import { + OMIT_SYSTEM_INDEX_PATTERN, + WorkflowConfig, +} from '../../../../../common'; +import { catIndices, useAppDispatch } from '../../../../store'; +import { getDataSourceId } from '../../../../utils'; interface SearchInputsProps { uiConfig: WorkflowConfig; @@ -21,6 +26,15 @@ interface SearchInputsProps { * The base component containing all of the search-related inputs */ export function SearchInputs(props: SearchInputsProps) { + const dispatch = useAppDispatch(); + const dataSourceId = getDataSourceId(); + // re-fetch indices on initial load. When users are first creating, + // they may enter this page without getting the updated index info + // for a newly-created index, so we re-fetch that here. + useEffect(() => { + dispatch(catIndices({ pattern: OMIT_SYSTEM_INDEX_PATTERN, dataSourceId })); + }, []); + return ( diff --git a/public/pages/workflows/new_workflow/quick_configure_inputs.tsx b/public/pages/workflows/new_workflow/quick_configure_inputs.tsx index 1ce8fd68..e3b61326 100644 --- a/public/pages/workflows/new_workflow/quick_configure_inputs.tsx +++ b/public/pages/workflows/new_workflow/quick_configure_inputs.tsx @@ -17,6 +17,10 @@ import { } from '@elastic/eui'; import { COHERE_DIMENSIONS, + DEFAULT_IMAGE_FIELD, + DEFAULT_LABEL_FIELD, + DEFAULT_TEXT_FIELD, + DEFAULT_VECTOR_FIELD, MODEL_STATE, Model, OPENAI_DIMENSIONS, @@ -30,10 +34,6 @@ interface QuickConfigureInputsProps { setFields(fields: QuickConfigureFields): void; } -const DEFAULT_TEXT_FIELD = 'my_text'; -const DEFAULT_VECTOR_FIELD = 'my_embedding'; -const DEFAULT_IMAGE_FIELD = 'my_image'; - // Dynamic component to allow optional input configuration fields for different use cases. // Hooks back to the parent component with such field values export function QuickConfigureInputs(props: QuickConfigureInputsProps) { @@ -60,26 +60,38 @@ export function QuickConfigureInputs(props: QuickConfigureInputsProps) { // defaults for the field values for certain workflow types useEffect(() => { let defaultFieldValues = {} as QuickConfigureFields; - if ( - props.workflowType === WORKFLOW_TYPE.SEMANTIC_SEARCH || - props.workflowType === WORKFLOW_TYPE.MULTIMODAL_SEARCH || - props.workflowType === WORKFLOW_TYPE.HYBRID_SEARCH - ) { - defaultFieldValues = { - textField: DEFAULT_TEXT_FIELD, - vectorField: DEFAULT_VECTOR_FIELD, - }; - } - if (props.workflowType === WORKFLOW_TYPE.MULTIMODAL_SEARCH) { - defaultFieldValues = { - ...defaultFieldValues, - imageField: DEFAULT_IMAGE_FIELD, - }; + switch (props.workflowType) { + case WORKFLOW_TYPE.SEMANTIC_SEARCH: + case WORKFLOW_TYPE.HYBRID_SEARCH: { + defaultFieldValues = { + textField: DEFAULT_TEXT_FIELD, + vectorField: DEFAULT_VECTOR_FIELD, + }; + break; + } + case WORKFLOW_TYPE.MULTIMODAL_SEARCH: { + defaultFieldValues = { + textField: DEFAULT_TEXT_FIELD, + vectorField: DEFAULT_VECTOR_FIELD, + imageField: DEFAULT_IMAGE_FIELD, + }; + break; + } + case WORKFLOW_TYPE.SENTIMENT_ANALYSIS: { + defaultFieldValues = { + textField: DEFAULT_TEXT_FIELD, + labelField: DEFAULT_LABEL_FIELD, + }; + break; + } + case WORKFLOW_TYPE.CUSTOM: + default: + break; } if (deployedModels.length > 0) { defaultFieldValues = { ...defaultFieldValues, - embeddingModelId: deployedModels[0].id, + modelId: deployedModels[0].id, }; } setFieldValues(defaultFieldValues); @@ -93,7 +105,7 @@ export function QuickConfigureInputs(props: QuickConfigureInputsProps) { // Try to pre-fill the dimensions based on the chosen model useEffect(() => { const selectedModel = deployedModels.find( - (model) => model.id === fieldValues.embeddingModelId + (model) => model.id === fieldValues.modelId ); if (selectedModel?.connectorId !== undefined) { const connector = connectors[selectedModel.connectorId]; @@ -127,13 +139,14 @@ export function QuickConfigureInputs(props: QuickConfigureInputsProps) { } } } - }, [fieldValues.embeddingModelId, deployedModels, connectors]); + }, [fieldValues.modelId, deployedModels, connectors]); return ( <> {(props.workflowType === WORKFLOW_TYPE.SEMANTIC_SEARCH || props.workflowType === WORKFLOW_TYPE.MULTIMODAL_SEARCH || - props.workflowType === WORKFLOW_TYPE.HYBRID_SEARCH) && ( + props.workflowType === WORKFLOW_TYPE.HYBRID_SEARCH || + props.workflowType === WORKFLOW_TYPE.SENTIMENT_ANALYSIS) && ( <> ) )} - valueOfSelected={fieldValues?.embeddingModelId || ''} + valueOfSelected={fieldValues?.modelId || ''} onChange={(option: string) => { setFieldValues({ ...fieldValues, - embeddingModelId: option, + modelId: option, }); }} isInvalid={false} @@ -185,7 +206,11 @@ export function QuickConfigureInputs(props: QuickConfigureInputsProps) { )} - - { - setFieldValues({ - ...fieldValues, - vectorField: e.target.value, - }); - }} - /> - - - - { - setFieldValues({ - ...fieldValues, - embeddingLength: Number(e.target.value), - }); - }} - /> - + {(props.workflowType === WORKFLOW_TYPE.SEMANTIC_SEARCH || + props.workflowType === WORKFLOW_TYPE.MULTIMODAL_SEARCH || + props.workflowType === WORKFLOW_TYPE.HYBRID_SEARCH) && ( + <> + + { + setFieldValues({ + ...fieldValues, + vectorField: e.target.value, + }); + }} + /> + + + + { + setFieldValues({ + ...fieldValues, + embeddingLength: Number(e.target.value), + }); + }} + /> + + + )} + {props.workflowType === WORKFLOW_TYPE.SENTIMENT_ANALYSIS && ( + + { + setFieldValues({ + ...fieldValues, + labelField: e.target.value, + }); + }} + /> + + )} )} diff --git a/public/pages/workflows/new_workflow/quick_configure_modal.tsx b/public/pages/workflows/new_workflow/quick_configure_modal.tsx index e519eb46..62a3cf1d 100644 --- a/public/pages/workflows/new_workflow/quick_configure_modal.tsx +++ b/public/pages/workflows/new_workflow/quick_configure_modal.tsx @@ -5,6 +5,9 @@ import React, { useState, useEffect } from 'react'; import { useHistory } from 'react-router-dom'; +import { isEmpty } from 'lodash'; +import { useSelector } from 'react-redux'; +import { flattie } from 'flattie'; import { EuiSmallButton, EuiModal, @@ -17,8 +20,9 @@ import { EuiCompressedFormRow, } from '@elastic/eui'; import { - EMPTY_MAP_ENTRY, IMAGE_FIELD_PATTERN, + IndexMappings, + LABEL_FIELD_PATTERN, MODEL_ID_PATTERN, MapArrayFormValue, MapFormValue, @@ -32,6 +36,7 @@ import { Workflow, WorkflowConfig, customStringify, + isVectorSearchUseCase, } from '../../../../common'; import { APP_PATH } from '../../../utils'; import { processWorkflowName } from './utils'; @@ -43,8 +48,6 @@ import { parseModelOutputs, } from '../../../utils/utils'; import { QuickConfigureInputs } from './quick_configure_inputs'; -import { isEmpty } from 'lodash'; -import { useSelector } from 'react-redux'; interface QuickConfigureModalProps { workflow: Workflow; @@ -89,10 +92,8 @@ export function QuickConfigureModal(props: QuickConfigureModalProps) { // fetching model interface if available. used to prefill some // of the input/output maps useEffect(() => { - setModelInterface( - models[quickConfigureFields.embeddingModelId || '']?.interface - ); - }, [models, quickConfigureFields.embeddingModelId]); + setModelInterface(models[quickConfigureFields.modelId || '']?.interface); + }, [models, quickConfigureFields.modelId]); return ( props.onClose()} style={{ width: '30vw' }}> @@ -182,16 +183,16 @@ function injectQuickConfigureFields( ): Workflow { if (workflow.ui_metadata?.type) { switch (workflow.ui_metadata?.type) { - // Semantic search / hybrid search: set defaults in the ingest processor, the index mappings, - // and the preset query case WORKFLOW_TYPE.SEMANTIC_SEARCH: case WORKFLOW_TYPE.HYBRID_SEARCH: - case WORKFLOW_TYPE.MULTIMODAL_SEARCH: { + case WORKFLOW_TYPE.MULTIMODAL_SEARCH: + case WORKFLOW_TYPE.SENTIMENT_ANALYSIS: { if (!isEmpty(quickConfigureFields) && workflow.ui_metadata?.config) { workflow.ui_metadata.config = updateIngestProcessorConfig( workflow.ui_metadata.config, quickConfigureFields, - modelInterface + modelInterface, + isVectorSearchUseCase(workflow) ); workflow.ui_metadata.config = updateIndexConfig( workflow.ui_metadata.config, @@ -204,7 +205,8 @@ function injectQuickConfigureFields( workflow.ui_metadata.config = updateSearchRequestProcessorConfig( workflow.ui_metadata.config, quickConfigureFields, - modelInterface + modelInterface, + isVectorSearchUseCase(workflow) ); } break; @@ -222,11 +224,12 @@ function injectQuickConfigureFields( function updateIngestProcessorConfig( config: WorkflowConfig, fields: QuickConfigureFields, - modelInterface: ModelInterface | undefined + modelInterface: ModelInterface | undefined, + isVectorSearchUseCase: boolean ): WorkflowConfig { config.ingest.enrich.processors[0].fields.forEach((field) => { - if (field.id === 'model' && fields.embeddingModelId) { - field.value = { id: fields.embeddingModelId }; + if (field.id === 'model' && fields.modelId) { + field.value = { id: fields.modelId }; } if (field.id === 'input_map') { const inputMap = generateMapFromModelInputs(modelInterface); @@ -260,14 +263,17 @@ function updateIngestProcessorConfig( } if (field.id === 'output_map') { const outputMap = generateMapFromModelOutputs(modelInterface); - if (fields.vectorField) { + const defaultField = isVectorSearchUseCase + ? fields.vectorField + : fields.labelField; + if (defaultField) { if (outputMap.length > 0) { outputMap[0] = { ...outputMap[0], - key: fields.vectorField, + key: defaultField, }; } else { - outputMap.push({ key: fields.vectorField, value: '' }); + outputMap.push({ key: defaultField, value: '' }); } } field.value = [outputMap] as MapArrayFormValue; @@ -282,36 +288,49 @@ function updateIngestProcessorConfig( function updateSearchRequestProcessorConfig( config: WorkflowConfig, fields: QuickConfigureFields, - modelInterface: ModelInterface | undefined + modelInterface: ModelInterface | undefined, + isVectorSearchUseCase: boolean ): WorkflowConfig { + let defaultQueryValue = '' as string; + try { + defaultQueryValue = Object.keys( + flattie(JSON.parse(config.search?.request?.value as string)) + )[0]; + } catch {} config.search.enrichRequest.processors[0].fields.forEach((field) => { - if (field.id === 'model' && fields.embeddingModelId) { - field.value = { id: fields.embeddingModelId }; + if (field.id === 'model' && fields.modelId) { + field.value = { id: fields.modelId }; } if (field.id === 'input_map') { const inputMap = generateMapFromModelInputs(modelInterface); - // TODO: pre-populate more if the query becomes standard - field.value = - inputMap.length > 0 - ? [inputMap] - : ([[EMPTY_MAP_ENTRY]] as MapArrayFormValue); + if (inputMap.length > 0) { + inputMap[0] = { + ...inputMap[0], + value: defaultQueryValue, + }; + } else { + inputMap.push({ + key: '', + value: defaultQueryValue, + }); + } + field.value = [inputMap] as MapArrayFormValue; } if (field.id === 'output_map') { - // prepopulate 'vector' constant as the model output transformed field, - // so it is consistent and used in the downstream query_template, if configured. const outputMap = generateMapFromModelOutputs(modelInterface); + const defaultKey = isVectorSearchUseCase ? VECTOR : defaultQueryValue; if (outputMap.length > 0) { outputMap[0] = { ...outputMap[0], - key: VECTOR, + key: defaultKey, }; } else { outputMap.push({ - key: VECTOR, + key: defaultKey, value: '', }); } - field.value = [outputMap]; + field.value = [outputMap] as MapArrayFormValue; } }); config.search.enrichRequest.processors[0].optionalFields = config.search.enrichRequest.processors[0].optionalFields?.map( @@ -335,47 +354,45 @@ function updateIndexConfig( config: WorkflowConfig, fields: QuickConfigureFields ): WorkflowConfig { - if (fields.textField) { - const existingMappings = JSON.parse( - config.ingest.index.mappings.value as string - ); - config.ingest.index.mappings.value = customStringify({ - ...existingMappings, - properties: { - ...(existingMappings.properties || {}), - [fields.textField]: { - type: 'text', - }, - }, - }); - } - if (fields.imageField) { - const existingMappings = JSON.parse( - config.ingest.index.mappings.value as string - ); - config.ingest.index.mappings.value = customStringify({ - ...existingMappings, - properties: { - ...(existingMappings.properties || {}), - [fields.imageField]: { - type: 'binary', - }, - }, - }); - } - if (fields.vectorField) { + if ( + fields.textField || + fields.imageField || + fields.vectorField || + fields.labelField + ) { const existingMappings = JSON.parse( config.ingest.index.mappings.value as string ); + let properties = {} as { [key: string]: {} }; + try { + properties = (JSON.parse( + config.ingest.index.mappings.value as string + ) as IndexMappings).properties; + } catch {} + if (fields.textField) { + properties[fields.textField] = { + type: 'text', + }; + } + if (fields.imageField) { + properties[fields.imageField] = { + type: 'binary', + }; + } + if (fields.vectorField) { + properties[fields.vectorField] = { + type: 'knn_vector', + dimension: fields.embeddingLength || '', + }; + } + if (fields.labelField) { + properties[fields.labelField] = { + type: 'text', + }; + } config.ingest.index.mappings.value = customStringify({ ...existingMappings, - properties: { - ...(existingMappings.properties || {}), - [fields.vectorField]: { - type: 'knn_vector', - dimension: fields.embeddingLength || '', - }, - }, + properties: { ...properties }, }); } return config; @@ -387,10 +404,10 @@ function injectPlaceholderValues( fields: QuickConfigureFields ): string { let finalRequestString = requestString; - if (fields.embeddingModelId) { + if (fields.modelId) { finalRequestString = finalRequestString.replace( new RegExp(MODEL_ID_PATTERN, 'g'), - fields.embeddingModelId + fields.modelId ); } if (fields.textField) { @@ -411,7 +428,12 @@ function injectPlaceholderValues( fields.imageField ); } - + if (fields.labelField) { + finalRequestString = finalRequestString.replace( + new RegExp(LABEL_FIELD_PATTERN, 'g'), + fields.labelField + ); + } return finalRequestString; } diff --git a/public/pages/workflows/new_workflow/utils.ts b/public/pages/workflows/new_workflow/utils.ts index 59a5bb06..841290b8 100644 --- a/public/pages/workflows/new_workflow/utils.ts +++ b/public/pages/workflows/new_workflow/utils.ts @@ -17,7 +17,8 @@ import { WORKFLOW_TYPE, FETCH_ALL_QUERY, customStringify, - TERM_QUERY, + TERM_QUERY_TEXT, + TERM_QUERY_LABEL, MULTIMODAL_SEARCH_QUERY_BOOL, IProcessorConfig, VECTOR_TEMPLATE_PLACEHOLDER, @@ -45,6 +46,10 @@ export function enrichPresetWorkflowWithUiMetadata( uiMetadata = fetchHybridSearchMetadata(); break; } + case WORKFLOW_TYPE.SENTIMENT_ANALYSIS: { + uiMetadata = fetchSentimentAnalysisMetadata(); + break; + } default: { uiMetadata = fetchEmptyMetadata(); break; @@ -133,7 +138,7 @@ export function fetchSemanticSearchMetadata(): UIState { baseState.config.ingest.index.settings.value = customStringify({ [`index.knn`]: true, }); - baseState.config.search.request.value = customStringify(TERM_QUERY); + baseState.config.search.request.value = customStringify(TERM_QUERY_TEXT); baseState.config.search.enrichRequest.processors = [ injectQueryTemplateInProcessor( new MLSearchRequestProcessor().toObj(), @@ -171,7 +176,7 @@ export function fetchHybridSearchMetadata(): UIState { baseState.config.ingest.index.settings.value = customStringify({ [`index.knn`]: true, }); - baseState.config.search.request.value = customStringify(TERM_QUERY); + baseState.config.search.request.value = customStringify(TERM_QUERY_TEXT); baseState.config.search.enrichResponse.processors = [ injectDefaultWeightsInNormalizationProcessor( new NormalizationProcessor().toObj() @@ -186,6 +191,21 @@ export function fetchHybridSearchMetadata(): UIState { return baseState; } +export function fetchSentimentAnalysisMetadata(): UIState { + let baseState = fetchEmptyMetadata(); + baseState.type = WORKFLOW_TYPE.SENTIMENT_ANALYSIS; + baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()]; + baseState.config.ingest.index.name.value = generateId('knn_index', 6); + baseState.config.ingest.index.settings.value = customStringify({ + [`index.knn`]: true, + }); + baseState.config.search.request.value = customStringify(TERM_QUERY_LABEL); + baseState.config.search.enrichRequest.processors = [ + new MLSearchRequestProcessor().toObj(), + ]; + return baseState; +} + // Utility fn to process workflow names from their presentable/readable titles // on the UI, to a valid name format. // This leads to less friction if users decide to save the name later on. diff --git a/public/pages/workflows/workflow_list/workflow_list.tsx b/public/pages/workflows/workflow_list/workflow_list.tsx index e43999d9..8fdad6e0 100644 --- a/public/pages/workflows/workflow_list/workflow_list.tsx +++ b/public/pages/workflows/workflow_list/workflow_list.tsx @@ -44,33 +44,13 @@ const sorting = { }, }; -const filterOptions = [ +const filterOptions = Object.values(WORKFLOW_TYPE).map((type) => { // @ts-ignore - { - name: WORKFLOW_TYPE.SEMANTIC_SEARCH, + return { + name: type, checked: 'on', - } as EuiFilterSelectItem, - // @ts-ignore - { - name: WORKFLOW_TYPE.MULTIMODAL_SEARCH, - checked: 'on', - } as EuiFilterSelectItem, - // @ts-ignore - { - name: WORKFLOW_TYPE.HYBRID_SEARCH, - checked: 'on', - } as EuiFilterSelectItem, - // @ts-ignore - { - name: WORKFLOW_TYPE.CUSTOM, - checked: 'on', - } as EuiFilterSelectItem, - // @ts-ignore - { - name: WORKFLOW_TYPE.UNKNOWN, - checked: 'on', - } as EuiFilterSelectItem, -]; + } as EuiFilterSelectItem; +}); /** * The searchable list of created workflows. diff --git a/public/store/reducers/ml_reducer.ts b/public/store/reducers/ml_reducer.ts index 53791869..c74dbc6f 100644 --- a/public/store/reducers/ml_reducer.ts +++ b/public/store/reducers/ml_reducer.ts @@ -8,7 +8,7 @@ import { ConnectorDict, ModelDict } from '../../../common'; import { HttpFetchError } from '../../../../../src/core/public'; import { getRouteService } from '../../services'; -const initialState = { +export const INITIAL_ML_STATE = { loading: false, errorMessage: '', models: {} as ModelDict, @@ -64,7 +64,7 @@ export const searchConnectors = createAsyncThunk( const mlSlice = createSlice({ name: 'ml', - initialState, + initialState: INITIAL_ML_STATE, reducers: {}, extraReducers: (builder) => { builder diff --git a/public/store/reducers/opensearch_reducer.ts b/public/store/reducers/opensearch_reducer.ts index d40e9396..4fe76b64 100644 --- a/public/store/reducers/opensearch_reducer.ts +++ b/public/store/reducers/opensearch_reducer.ts @@ -8,11 +8,12 @@ import { getRouteService } from '../../services'; import { Index, IngestPipelineConfig, + OMIT_SYSTEM_INDEX_PATTERN, SimulateIngestPipelineDoc, } from '../../../common'; import { HttpFetchError } from '../../../../../src/core/public'; -const initialState = { +export const INITIAL_OPENSEARCH_STATE = { loading: false, errorMessage: '', indices: {} as { [key: string]: Index }, @@ -33,7 +34,7 @@ export const catIndices = createAsyncThunk( { rejectWithValue } ) => { // defaulting to fetch everything except system indices (starting with '.') - const patternString = pattern || '*,-.*'; + const patternString = pattern || OMIT_SYSTEM_INDEX_PATTERN; const response: any | HttpFetchError = await getRouteService().catIndices( patternString, dataSourceId @@ -182,7 +183,7 @@ export const simulatePipeline = createAsyncThunk( const opensearchSlice = createSlice({ name: OPENSEARCH_PREFIX, - initialState, + initialState: INITIAL_OPENSEARCH_STATE, reducers: {}, extraReducers: (builder) => { builder diff --git a/public/store/reducers/presets_reducer.ts b/public/store/reducers/presets_reducer.ts index 01321ac4..33baa16f 100644 --- a/public/store/reducers/presets_reducer.ts +++ b/public/store/reducers/presets_reducer.ts @@ -8,7 +8,7 @@ import { WorkflowTemplate } from '../../../common'; import { HttpFetchError } from '../../../../../src/core/public'; import { getRouteService } from '../../services'; -const initialState = { +export const INITIAL_PRESETS_STATE = { loading: false, errorMessage: '', presetWorkflows: [] as Partial[], @@ -35,7 +35,7 @@ export const getWorkflowPresets = createAsyncThunk( const presetsSlice = createSlice({ name: 'presets', - initialState, + initialState: INITIAL_PRESETS_STATE, reducers: {}, extraReducers: (builder) => { builder diff --git a/public/store/reducers/workflows_reducer.ts b/public/store/reducers/workflows_reducer.ts index 53a8e588..c34fd999 100644 --- a/public/store/reducers/workflows_reducer.ts +++ b/public/store/reducers/workflows_reducer.ts @@ -8,7 +8,7 @@ import { WorkflowDict, WorkflowTemplate } from '../../../common'; import { HttpFetchError } from '../../../../../src/core/public'; import { getRouteService } from '../../services'; -const initialState = { +export const INITIAL_WORKFLOWS_STATE = { loading: false, errorMessage: '', workflows: {} as WorkflowDict, @@ -223,7 +223,7 @@ export const deleteWorkflow = createAsyncThunk( const workflowsSlice = createSlice({ name: 'workflows', - initialState, + initialState: INITIAL_WORKFLOWS_STATE, reducers: {}, extraReducers: (builder) => { builder diff --git a/public/utils/utils.ts b/public/utils/utils.ts index 48450b66..9c3c80f8 100644 --- a/public/utils/utils.ts +++ b/public/utils/utils.ts @@ -9,7 +9,6 @@ import { escape, get } from 'lodash'; import { JSONPATH_ROOT_SELECTOR, MapFormValue, - ModelInput, ModelInputFormField, ModelInterface, ModelOutput, @@ -100,10 +99,12 @@ export function getObjFromJsonOrYamlString( fileContents?: string ): object | undefined { try { + // @ts-ignore const jsonObj = JSON.parse(fileContents); return jsonObj; } catch (e) {} try { + // @ts-ignore const yamlObj = yaml.load(fileContents) as object; return yamlObj; } catch (e) {} @@ -112,11 +113,11 @@ export function getObjFromJsonOrYamlString( // Based off of https://opensearch.org/docs/latest/automating-configurations/api/create-workflow/#request-fields // Only "name" field is required -export function isValidWorkflow(workflowObj: object | undefined): boolean { +export function isValidWorkflow(workflowObj: any): boolean { return workflowObj?.name !== undefined; } -export function isValidUiWorkflow(workflowObj: object | undefined): boolean { +export function isValidUiWorkflow(workflowObj: any): boolean { return ( isValidWorkflow(workflowObj) && workflowObj?.ui_metadata?.config !== undefined && @@ -191,12 +192,7 @@ export function generateTransform(input: {}, map: MapFormValue): {} { ...output, [mapEntry.key]: transformedResult || '', }; - } catch (e: any) { - getCore().notifications.toasts.addDanger( - 'Error generating expected output. Ensure your transforms are valid JSONPath or dot notation syntax.', - e - ); - } + } catch (e: any) {} }); return output; } diff --git a/server/resources/templates/sentiment_analysis.json b/server/resources/templates/sentiment_analysis.json new file mode 100644 index 00000000..ed2d4c92 --- /dev/null +++ b/server/resources/templates/sentiment_analysis.json @@ -0,0 +1,14 @@ +{ + "name": "Sentiment Analysis", + "description": "A basic workflow containing the ingest pipeline, search pipeline, and index configurations for performing sentiment analysis", + "version": { + "template": "1.0.0", + "compatibility": [ + "2.17.0", + "3.0.0" + ] + }, + "ui_metadata": { + "type": "Sentiment analysis" + } +} \ No newline at end of file diff --git a/test/utils.ts b/test/utils.ts index c0d053c4..68c9af5b 100644 --- a/test/utils.ts +++ b/test/utils.ts @@ -3,6 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { + INITIAL_ML_STATE, + INITIAL_OPENSEARCH_STATE, + INITIAL_PRESETS_STATE, + INITIAL_WORKFLOWS_STATE, +} from '../public/store'; import { WORKFLOW_TYPE } from '../common/constants'; import { UIState, Workflow } from '../common/interfaces'; import { @@ -19,13 +25,10 @@ export function mockStore( ) { return { getState: () => ({ - opensearch: { - errorMessage: '', - }, - ml: {}, + opensearch: INITIAL_OPENSEARCH_STATE, + ml: INITIAL_ML_STATE, workflows: { - loading: false, - errorMessage: '', + ...INITIAL_WORKFLOWS_STATE, workflows: { [workflowId]: generateWorkflow( workflowId, @@ -34,6 +37,7 @@ export function mockStore( ), }, }, + presets: INITIAL_PRESETS_STATE, }), dispatch: jest.fn(), subscribe: jest.fn(), diff --git a/yarn.lock b/yarn.lock index e6bd5793..6fca7a37 100644 --- a/yarn.lock +++ b/yarn.lock @@ -407,6 +407,11 @@ fast-levenshtein@~2.0.6: resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== +flattie@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/flattie/-/flattie-1.1.1.tgz#88182235723113667d36217fec55359275d6fe3d" + integrity sha512-9UbaD6XdAL97+k/n+N7JwX46K/M6Zc6KcFYskrYL8wbBV/Uyk0CTAMY0VT+qiK5PM7AIc9aTWYtq65U7T+aCNQ== + formik@2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/formik/-/formik-2.4.2.tgz#a1115457cfb012a5c782cea3ad4b40b2fe36fa18"