Skip to content

Commit

Permalink
Merge branch 'main' into tests_5
Browse files Browse the repository at this point in the history 
Signed-off-by: Sai Medhini Reddy Maryada <[email protected]>
  • Loading branch information
@saimedhi
saimedhi committed Sep 10, 2024
2 parents 169a8c4 + 7c5ad9a commit 5ad01e4
Show file tree
Hide file tree
Showing 23 changed files with 424 additions and 218 deletions.
20 changes: 18 additions & 2 deletions common/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ export enum WORKFLOW_TYPE {
SEMANTIC_SEARCH = 'Semantic search',
MULTIMODAL_SEARCH = 'Multimodal search',
HYBRID_SEARCH = 'Hybrid search',
SENTIMENT_ANALYSIS = 'Sentiment analysis',
CUSTOM = 'Custom',
UNKNOWN = 'Unknown',
}
Expand Down Expand Up @@ -181,9 +182,14 @@ export const SHARED_OPTIONAL_FIELDS = ['max_chunk_limit', 'description', 'tag'];
/**
* QUERY PRESETS
*/
export const DEFAULT_TEXT_FIELD = 'my_text';
export const DEFAULT_VECTOR_FIELD = 'my_embedding';
export const DEFAULT_IMAGE_FIELD = 'my_image';
export const DEFAULT_LABEL_FIELD = 'label';
export const VECTOR_FIELD_PATTERN = `{{vector_field}}`;
export const TEXT_FIELD_PATTERN = `{{text_field}}`;
export const IMAGE_FIELD_PATTERN = `{{image_field}}`;
export const LABEL_FIELD_PATTERN = `{{label_field}}`;
export const QUERY_TEXT_PATTERN = `{{query_text}}`;
export const QUERY_IMAGE_PATTERN = `{{query_image}}`;
export const MODEL_ID_PATTERN = `{{model_id}}`;
Expand All @@ -198,7 +204,7 @@ export const FETCH_ALL_QUERY = {
},
size: 1000,
};
export const TERM_QUERY = {
export const TERM_QUERY_TEXT = {
query: {
term: {
[TEXT_FIELD_PATTERN]: {
Expand All @@ -207,6 +213,15 @@ export const TERM_QUERY = {
},
},
};
export const TERM_QUERY_LABEL = {
query: {
term: {
[LABEL_FIELD_PATTERN]: {
value: QUERY_TEXT_PATTERN,
},
},
},
};
export const KNN_QUERY = {
_source: {
excludes: [VECTOR_FIELD_PATTERN],
Expand Down Expand Up @@ -353,7 +368,7 @@ export const QUERY_PRESETS = [
},
{
name: 'Term',
query: customStringify(TERM_QUERY),
query: customStringify(TERM_QUERY_TEXT),
},
{
name: 'Basic k-NN',
Expand Down Expand Up @@ -395,6 +410,7 @@ export const DEFAULT_NEW_WORKFLOW_STATE = WORKFLOW_STATE.NOT_STARTED;
export const DEFAULT_NEW_WORKFLOW_STATE_TYPE = ('NOT_STARTED' as any) as typeof WORKFLOW_STATE;
export const DATE_FORMAT_PATTERN = 'MM/DD/YY hh:mm A';
export const EMPTY_FIELD_STRING = '--';
export const OMIT_SYSTEM_INDEX_PATTERN = '*,-.*';
export const INDEX_NOT_FOUND_EXCEPTION = 'index_not_found_exception';
export const ERROR_GETTING_WORKFLOW_MSG = 'Failed to retrieve template';
export const NO_TEMPLATES_FOUND_MSG = 'There are no templates';
Expand Down
3 changes: 2 additions & 1 deletion common/interfaces.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -483,10 +483,11 @@ export type QueryPreset = {
};

export type QuickConfigureFields = {
embeddingModelId?: string;
modelId?: string;
vectorField?: string;
textField?: string;
imageField?: string;
labelField?: string;
embeddingLength?: number;
};

Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
},
"dependencies": {
"@types/jsonpath": "^0.2.4",
"flattie": "^1.1.1",
"formik": "2.4.2",
"jsonpath": "^1.1.1",
"reactflow": "^11.8.3",
Expand Down
3 changes: 2 additions & 1 deletion public/pages/workflow_detail/workflow_detail.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import {
FETCH_ALL_QUERY,
MAX_WORKFLOW_NAME_TO_DISPLAY,
NO_TEMPLATES_FOUND_MSG,
OMIT_SYSTEM_INDEX_PATTERN,
getCharacterLimitedString,
} from '../../../common';
import { MountPoint } from '../../../../../src/core/public';
Expand Down Expand Up @@ -106,7 +107,7 @@ export function WorkflowDetail(props: WorkflowDetailProps) {
useEffect(() => {
dispatch(getWorkflow({ workflowId, dataSourceId }));
dispatch(searchModels({ apiBody: FETCH_ALL_QUERY, dataSourceId }));
dispatch(catIndices({ pattern: '*,-.*', dataSourceId }));
dispatch(catIndices({ pattern: OMIT_SYSTEM_INDEX_PATTERN, dataSourceId }));
}, []);

return errorMessage.includes(ERROR_GETTING_WORKFLOW_MSG) ||
Expand Down
4 changes: 2 additions & 2 deletions public/pages/workflow_detail/workflow_inputs/input_fields/map_array_field.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ interface MapArrayFieldProps {
valuePlaceholder?: string;
onMapAdd?: (curArray: MapArrayFormValue) => void;
onMapDelete?: (idxToDelete: number) => void;
keyOptions?: any[];
valueOptions?: any[];
keyOptions?: { label: string }[];
valueOptions?: { label: string }[];
}

/**
Expand Down
4 changes: 2 additions & 2 deletions public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ interface MapFieldProps {
helpText?: string;
keyPlaceholder?: string;
valuePlaceholder?: string;
keyOptions?: any[];
valueOptions?: any[];
keyOptions?: { label: string }[];
valueOptions?: { label: string }[];
}

/**
Expand Down
16 changes: 9 additions & 7 deletions public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { WorkspaceFormValues } from '../../../../../common';
interface SelectWithCustomOptionsProps {
fieldPath: string;
placeholder: string;
options: any[];
options: { label: string }[];
}

/**
Expand Down Expand Up @@ -50,13 +50,15 @@ export function SelectWithCustomOptions(props: SelectWithCustomOptionsProps) {
return (
<EuiFlexGroup direction="row" alignItems="flexStart" gutterSize="s">
<EuiFlexItem grow={false}>
<EuiText size="s">{option.label}</EuiText>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiText size="xs" color="subdued" style={{ marginTop: '2px' }}>
{`(${option.type || 'unknown type'})`}
</EuiText>
<EuiText size="s">{option.label || ''}</EuiText>
</EuiFlexItem>
{option.type && (
<EuiFlexItem grow={false}>
<EuiText size="xs" color="subdued" style={{ marginTop: '2px' }}>
{`(${option.type})`}
</EuiText>
</EuiFlexItem>
)}
</EuiFlexGroup>
);
}
Expand Down
12 changes: 7 additions & 5 deletions public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ interface InputTransformModalProps {
inputMapField: IConfigField;
inputMapFieldPath: string;
modelInterface: ModelInterface | undefined;
valueOptions: { label: string }[];
onClose: () => void;
}

Expand Down Expand Up @@ -163,7 +164,7 @@ export function InputTransformModal(props: InputTransformModalProps) {
Fetch some sample input data and see how it is transformed.
</EuiText>
<EuiSpacer size="s" />
<EuiText>Expected input</EuiText>
<EuiText>Source input</EuiText>
<EuiSmallButton
style={{ width: '100px' }}
onClick={async () => {
Expand Down Expand Up @@ -266,7 +267,7 @@ export function InputTransformModal(props: InputTransformModalProps) {
})
.catch((error: any) => {
getCore().notifications.toasts.addDanger(
`Failed to fetch input data`
`Failed to fetch source input data`
);
});
break;
Expand Down Expand Up @@ -307,12 +308,13 @@ export function InputTransformModal(props: InputTransformModalProps) {
root object selector "${JSONPATH_ROOT_SELECTOR}"`}
helpLink={ML_INFERENCE_DOCS_LINK}
keyPlaceholder="Model input field"
keyOptions={parseModelInputs(props.modelInterface)}
valuePlaceholder={
props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
? 'Query field'
: 'Document field'
}
keyOptions={parseModelInputs(props.modelInterface)}
valueOptions={props.valueOptions}
// If the map we are adding is the first one, populate the selected option to index 0
onMapAdd={(curArray) => {
if (isEmpty(curArray)) {
Expand Down Expand Up @@ -356,10 +358,10 @@ export function InputTransformModal(props: InputTransformModalProps) {
)}
<EuiFlexItem grow={true}>
{outputOptions.length === 1 ? (
<EuiText>Expected output</EuiText>
<EuiText>Transformed input</EuiText>
) : (
<EuiCompressedSelect
prepend={<EuiText>Expected output for</EuiText>}
prepend={<EuiText>Transformed input for</EuiText>}
options={outputOptions}
value={selectedOutputOption}
onChange={(e) => {
Expand Down
101 changes: 90 additions & 11 deletions public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@

import React, { useState, useEffect } from 'react';
import { getIn, useFormikContext } from 'formik';
import { isEmpty } from 'lodash';
import { useSelector } from 'react-redux';
import { flattie } from 'flattie';
import {
EuiAccordion,
EuiSmallButtonEmpty,
Expand All @@ -25,14 +27,15 @@ import {
ML_INFERENCE_DOCS_LINK,
WorkflowFormValues,
ModelInterface,
IndexMappings,
} from '../../../../../common';
import { MapArrayField, ModelField } from '../input_fields';
import { isEmpty } from 'lodash';
import { InputTransformModal } from './input_transform_modal';
import { OutputTransformModal } from './output_transform_modal';
import { AppState } from '../../../../store';
import { AppState, getMappings, useAppDispatch } from '../../../../store';
import {
formikToPartialPipeline,
getDataSourceId,
parseModelInputs,
parseModelOutputs,
} from '../../../../utils';
Expand All @@ -52,7 +55,10 @@ interface MLProcessorInputsProps {
* output map configuration forms, respectively.
*/
export function MLProcessorInputs(props: MLProcessorInputsProps) {
const dispatch = useAppDispatch();
const dataSourceId = getDataSourceId();
const models = useSelector((state: AppState) => state.ml.models);
const indices = useSelector((state: AppState) => state.opensearch.indices);
const { values, setFieldValue, setFieldTouched } = useFormikContext<
WorkflowFormValues
>();
Expand Down Expand Up @@ -115,7 +121,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
// 1: update model interface states
// 2. clear out any persisted input_map/output_map form values, as those would now be invalid
function onModelChange(modelId: string) {
updateModelInterfaceStates(modelId);
setModelInterface(models[modelId]?.interface);
setFieldValue(inputMapFieldPath, []);
setFieldValue(outputMapFieldPath, []);
setFieldTouched(inputMapFieldPath, false);
Expand All @@ -127,16 +133,75 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
if (!isEmpty(models)) {
const modelId = getIn(values, `${modelFieldPath}.id`);
if (modelId) {
updateModelInterfaceStates(modelId);
setModelInterface(models[modelId]?.interface);
}
}
}, [models]);

// reusable function to update interface states based on the model ID
function updateModelInterfaceStates(modelId: string) {
const newSelectedModel = models[modelId];
setModelInterface(newSelectedModel?.interface);
}
// persisting doc/query/index mapping fields to collect a list
// of options to display in the dropdowns when configuring input / output maps
const [docFields, setDocFields] = useState<{ label: string }[]>([]);
const [queryFields, setQueryFields] = useState<{ label: string }[]>([]);
const [indexMappingFields, setIndexMappingFields] = useState<
{ label: string }[]
>([]);
useEffect(() => {
try {
const docObjKeys = Object.keys(
flattie((JSON.parse(values.ingest.docs) as {}[])[0])
);
if (docObjKeys.length > 0) {
setDocFields(
docObjKeys.map((key) => {
return {
label: key,
};
})
);
}
} catch {}
}, [values?.ingest?.docs]);
useEffect(() => {
try {
const queryObjKeys = Object.keys(
flattie(JSON.parse(values.search.request))
);
if (queryObjKeys.length > 0) {
setQueryFields(
queryObjKeys.map((key) => {
return {
label: key,
};
})
);
}
} catch {}
}, [values?.search?.request]);
useEffect(() => {
const indexName = values?.search?.index?.name as string | undefined;
if (indexName !== undefined && indices[indexName] !== undefined) {
dispatch(
getMappings({
index: indexName,
dataSourceId,
})
)
.unwrap()
.then((resp: IndexMappings) => {
const mappingsObjKeys = Object.keys(resp.properties);
if (mappingsObjKeys.length > 0) {
setIndexMappingFields(
mappingsObjKeys.map((key) => {
return {
label: key,
type: resp.properties[key]?.type,
};
})
);
}
});
}
}, [values?.search?.index?.name]);

return (
<>
Expand All @@ -148,6 +213,13 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
inputMapField={inputMapField}
inputMapFieldPath={inputMapFieldPath}
modelInterface={modelInterface}
valueOptions={
props.context === PROCESSOR_CONTEXT.INGEST
? docFields
: props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
? queryFields
: indexMappingFields
}
onClose={() => setIsInputTransformModalOpen(false)}
/>
)}
Expand Down Expand Up @@ -212,12 +284,19 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
root object selector "${JSONPATH_ROOT_SELECTOR}"`}
helpLink={ML_INFERENCE_DOCS_LINK}
keyPlaceholder="Model input field"
keyOptions={parseModelInputs(modelInterface)}
valuePlaceholder={
props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
? 'Query field'
: 'Document field'
}
keyOptions={parseModelInputs(modelInterface)}
valueOptions={
props.context === PROCESSOR_CONTEXT.INGEST
? docFields
: props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
? queryFields
: indexMappingFields
}
/>
<EuiSpacer size="l" />
<EuiFlexGroup direction="row">
Expand Down Expand Up @@ -262,7 +341,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
keyPlaceholder={
props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
? 'Query field'
: 'Document field'
: 'New document field'
}
valuePlaceholder="Model output field"
valueOptions={parseModelOutputs(modelInterface)}
Expand Down
Loading

0 comments on commit 5ad01e4

Please sign in to comment.