diff --git a/common/constants.ts b/common/constants.ts
index fb891729..6cdbbd63 100644
--- a/common/constants.ts
+++ b/common/constants.ts
@@ -3,7 +3,12 @@
* SPDX-License-Identifier: Apache-2.0
*/
-import { TemplateNode, WORKFLOW_STATE } from './interfaces';
+import {
+ MODEL_ALGORITHM,
+ PRETRAINED_MODEL_FORMAT,
+ PretrainedSentenceTransformer,
+ WORKFLOW_STATE,
+} from './interfaces';
export const PLUGIN_ID = 'flow-framework';
@@ -52,6 +57,42 @@ export const SEARCH_MODELS_NODE_API_PATH = `${BASE_MODEL_NODE_API_PATH}/search`;
*/
export const CREATE_INGEST_PIPELINE_STEP_TYPE = 'create_ingest_pipeline';
export const CREATE_INDEX_STEP_TYPE = 'create_index';
+export const REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE =
+ 'register_local_pretrained_model';
+
+/**
+ * ML PLUGIN PRETRAINED MODELS
+ * (based off of https://opensearch.org/docs/latest/ml-commons-plugin/pretrained-models/#sentence-transformers)
+ */
+export const ROBERTA_SENTENCE_TRANSFORMER = {
+ name: 'huggingface/sentence-transformers/all-distilroberta-v1',
+ shortenedName: 'all-distilroberta-v1',
+ description: 'A sentence transformer from Hugging Face',
+ format: PRETRAINED_MODEL_FORMAT.TORCH_SCRIPT,
+ algorithm: MODEL_ALGORITHM.TEXT_EMBEDDING,
+ version: '1.0.1',
+ vectorDimensions: 768,
+} as PretrainedSentenceTransformer;
+
+export const MPNET_SENTENCE_TRANSFORMER = {
+ name: 'huggingface/sentence-transformers/all-mpnet-base-v2',
+ shortenedName: 'all-mpnet-base-v2',
+ description: 'A sentence transformer from Hugging Face',
+ format: PRETRAINED_MODEL_FORMAT.TORCH_SCRIPT,
+ algorithm: MODEL_ALGORITHM.TEXT_EMBEDDING,
+ version: '1.0.1',
+ vectorDimensions: 768,
+} as PretrainedSentenceTransformer;
+
+export const BERT_SENTENCE_TRANSFORMER = {
+ name: 'huggingface/sentence-transformers/msmarco-distilbert-base-tas-b',
+ shortenedName: 'msmarco-distilbert-base-tas-b',
+ description: 'A sentence transformer from Hugging Face',
+ format: PRETRAINED_MODEL_FORMAT.TORCH_SCRIPT,
+ algorithm: MODEL_ALGORITHM.TEXT_EMBEDDING,
+ version: '1.0.2',
+ vectorDimensions: 768,
+} as PretrainedSentenceTransformer;
/**
* MISCELLANEOUS
diff --git a/common/interfaces.ts b/common/interfaces.ts
index d9c6bc43..858965cf 100644
--- a/common/interfaces.ts
+++ b/common/interfaces.ts
@@ -82,6 +82,16 @@ export type CreateIndexNode = TemplateNode & {
};
};
+export type RegisterPretrainedModelNode = TemplateNode & {
+ user_inputs: {
+ name: string;
+ description: string;
+ model_format: string;
+ version: string;
+ deploy: boolean;
+ };
+};
+
export type TemplateEdge = {
source: string;
dest: string;
@@ -130,9 +140,83 @@ export enum USE_CASE {
/**
********** ML PLUGIN TYPES/INTERFACES **********
*/
+
+// Based off of https://github.com/opensearch-project/ml-commons/blob/main/common/src/main/java/org/opensearch/ml/common/model/MLModelState.java
+export enum MODEL_STATE {
+ REGISTERED = 'Registered',
+ REGISTERING = 'Registering',
+ DEPLOYING = 'Deploying',
+ DEPLOYED = 'Deployed',
+ PARTIALLY_DEPLOYED = 'Partially deployed',
+ UNDEPLOYED = 'Undeployed',
+ DEPLOY_FAILED = 'Deploy failed',
+}
+
+// Based off of https://github.com/opensearch-project/ml-commons/blob/main/common/src/main/java/org/opensearch/ml/common/FunctionName.java
+export enum MODEL_ALGORITHM {
+ LINEAR_REGRESSION = 'Linear regression',
+ KMEANS = 'K-means',
+ AD_LIBSVM = 'AD LIBSVM',
+ SAMPLE_ALGO = 'Sample algorithm',
+ LOCAL_SAMPLE_CALCULATOR = 'Local sample calculator',
+ FIT_RCF = 'Fit RCF',
+ BATCH_RCF = 'Batch RCF',
+ ANOMALY_LOCALIZATION = 'Anomaly localization',
+ RCF_SUMMARIZE = 'RCF summarize',
+ LOGISTIC_REGRESSION = 'Logistic regression',
+ TEXT_EMBEDDING = 'Text embedding',
+ METRICS_CORRELATION = 'Metrics correlation',
+ REMOTE = 'Remote',
+ SPARSE_ENCODING = 'Sparse encoding',
+ SPARSE_TOKENIZE = 'Sparse tokenize',
+ TEXT_SIMILARITY = 'Text similarity',
+ QUESTION_ANSWERING = 'Question answering',
+ AGENT = 'Agent',
+}
+
+export enum MODEL_CATEGORY {
+ DEPLOYED = 'Deployed',
+ PRETRAINED = 'Pretrained',
+}
+
+export enum PRETRAINED_MODEL_FORMAT {
+ TORCH_SCRIPT = 'TORCH_SCRIPT',
+}
+
+export type PretrainedModel = {
+ name: string;
+ shortenedName: string;
+ description: string;
+ format: PRETRAINED_MODEL_FORMAT;
+ algorithm: MODEL_ALGORITHM;
+ version: string;
+};
+
+export type PretrainedSentenceTransformer = PretrainedModel & {
+ vectorDimensions: number;
+};
+
+export type ModelConfig = {
+ modelType?: string;
+ embeddingDimension?: number;
+};
+
export type Model = {
id: string;
- algorithm: string;
+ name: string;
+ algorithm: MODEL_ALGORITHM;
+ state: MODEL_STATE;
+ modelConfig?: ModelConfig;
+};
+
+export type ModelDict = {
+ [modelId: string]: Model;
+};
+
+export type ModelFormValue = {
+ id: string;
+ category?: MODEL_CATEGORY;
+ algorithm?: MODEL_ALGORITHM;
};
/**
@@ -171,7 +255,3 @@ export enum WORKFLOW_RESOURCE_TYPE {
export type WorkflowDict = {
[workflowId: string]: Workflow;
};
-
-export type ModelDict = {
- [modelId: string]: Model;
-};
diff --git a/public/app.tsx b/public/app.tsx
index 2e14044d..fa80d217 100644
--- a/public/app.tsx
+++ b/public/app.tsx
@@ -62,12 +62,21 @@ export const FlowFrameworkDashboardsApp = (props: Props) => {
)}
/>
- {/* Defaulting to Workflows page */}
+ {/*
+ Defaulting to Workflows page. The pathname will need to be updated
+ to handle the redirection and get the router props consistent.
+ */}
) => (
-
- )}
+ render={(routeProps: RouteComponentProps) => {
+ if (props.history.location.pathname !== APP_PATH.WORKFLOWS) {
+ props.history.replace({
+ ...history,
+ pathname: APP_PATH.WORKFLOWS,
+ });
+ }
+ return ;
+ }}
/>
diff --git a/public/component_types/interfaces.ts b/public/component_types/interfaces.ts
index 0056d700..055f44ff 100644
--- a/public/component_types/interfaces.ts
+++ b/public/component_types/interfaces.ts
@@ -10,7 +10,7 @@ import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../utils';
/**
* ************ Types *************************
*/
-export type FieldType = 'string' | 'json' | 'select';
+export type FieldType = 'string' | 'json' | 'select' | 'model';
export type SelectType = 'model';
export type FieldValue = string | {};
export type ComponentFormValues = FormikValues;
diff --git a/public/component_types/transformer/text_embedding_transformer.ts b/public/component_types/transformer/text_embedding_transformer.ts
index c856381e..affb996c 100644
--- a/public/component_types/transformer/text_embedding_transformer.ts
+++ b/public/component_types/transformer/text_embedding_transformer.ts
@@ -19,11 +19,10 @@ export class TextEmbeddingTransformer extends MLTransformer {
this.inputs = [];
this.createFields = [
{
- label: 'Model ID',
- id: 'modelId',
- type: 'select',
- selectType: 'model',
- helpText: 'The deployed text embedding model to use for embedding.',
+ label: 'Text Embedding Model',
+ id: 'model',
+ type: 'model',
+ helpText: 'A text embedding model for embedding text.',
helpLink:
'https://opensearch.org/docs/latest/ml-commons-plugin/integrating-ml-models/#choosing-a-model',
},
@@ -36,7 +35,6 @@ export class TextEmbeddingTransformer extends MLTransformer {
helpLink:
'https://opensearch.org/docs/latest/ingest-pipelines/processors/text-embedding/',
},
-
{
label: 'Vector Field',
id: 'vectorField',
diff --git a/public/pages/workflow_detail/component_details/component_details.tsx b/public/pages/workflow_detail/component_details/component_details.tsx
index 41ed85d0..1de35468 100644
--- a/public/pages/workflow_detail/component_details/component_details.tsx
+++ b/public/pages/workflow_detail/component_details/component_details.tsx
@@ -28,9 +28,12 @@ interface ComponentDetailsProps {
export function ComponentDetails(props: ComponentDetailsProps) {
return (
- {props.isDeprovisionable ? (
+ {/* TODO: determine if we need this view if we want the workspace to remain
+ readonly once provisioned */}
+ {/* {props.isDeprovisionable ? (
- ) : props.selectedComponent ? (
+ ) : */}
+ {props.selectedComponent ? (
{props.selectedComponent.data.label || ''}
+
+ {props.selectedComponent.data.description}
+
+
+
+
+ );
+ break;
+ }
case 'json': {
el = (
diff --git a/public/pages/workflow_detail/component_details/input_fields/index.ts b/public/pages/workflow_detail/component_details/input_fields/index.ts
index e2edf8bd..7d0561f5 100644
--- a/public/pages/workflow_detail/component_details/input_fields/index.ts
+++ b/public/pages/workflow_detail/component_details/input_fields/index.ts
@@ -6,3 +6,4 @@
export { TextField } from './text_field';
export { JsonField } from './json_field';
export { SelectField } from './select_field';
+export { ModelField } from './model_field';
diff --git a/public/pages/workflow_detail/component_details/input_fields/model_field.tsx b/public/pages/workflow_detail/component_details/input_fields/model_field.tsx
new file mode 100644
index 00000000..26ba8e37
--- /dev/null
+++ b/public/pages/workflow_detail/component_details/input_fields/model_field.tsx
@@ -0,0 +1,215 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import React, { useEffect, useState } from 'react';
+import { useSelector } from 'react-redux';
+import { Field, FieldProps, useFormikContext } from 'formik';
+import {
+ EuiFormRow,
+ EuiLink,
+ EuiRadioGroup,
+ EuiRadioGroupOption,
+ EuiSpacer,
+ EuiSuperSelect,
+ EuiSuperSelectOption,
+ EuiText,
+} from '@elastic/eui';
+import {
+ BERT_SENTENCE_TRANSFORMER,
+ IComponentField,
+ MODEL_STATE,
+ ROBERTA_SENTENCE_TRANSFORMER,
+ WorkspaceFormValues,
+ isFieldInvalid,
+ ModelFormValue,
+ MODEL_CATEGORY,
+ MPNET_SENTENCE_TRANSFORMER,
+} from '../../../../../common';
+import { AppState } from '../../../../store';
+
+interface ModelFieldProps {
+ field: IComponentField;
+ componentId: string;
+ onFormChange: () => void;
+}
+
+type ModelItem = ModelFormValue & {
+ name: string;
+};
+
+// TODO: there is no concrete UX for model selection and model provisioning. This component is TBD
+// and simply provides the ability to select existing models, or deploy some pretrained ones,
+// and persist all of this in form state.
+/**
+ * A specific field for selecting existing deployed models, or available pretrained models.
+ */
+export function ModelField(props: ModelFieldProps) {
+ // Initial store is fetched when loading base page. We don't
+ // re-fetch here as it could overload client-side if user clicks back and forth /
+ // keeps re-rendering this component (and subsequently re-fetching data) as they're building flows
+ const models = useSelector((state: AppState) => state.models.models);
+
+ const formField = `${props.componentId}.${props.field.id}`;
+ const { errors, touched } = useFormikContext();
+
+ // Deployed models state
+ const [deployedModels, setDeployedModels] = useState([]);
+ const [pretrainedModels, setPretrainedModels] = useState([]);
+ const [selectableModels, setSelectableModels] = useState([]);
+
+ // Radio options state
+ const radioOptions = [
+ {
+ id: MODEL_CATEGORY.DEPLOYED,
+ label: 'Existing deployed models',
+ },
+ {
+ id: MODEL_CATEGORY.PRETRAINED,
+ label: 'Pretrained models',
+ },
+ ] as EuiRadioGroupOption[];
+ const [selectedRadioId, setSelectedRadioId] = useState<
+ MODEL_CATEGORY | undefined
+ >(undefined);
+
+ // Initialize available deployed models
+ useEffect(() => {
+ if (models) {
+ const modelItems = [] as ModelItem[];
+ Object.keys(models).forEach((modelId) => {
+ if (models[modelId].state === MODEL_STATE.DEPLOYED) {
+ modelItems.push({
+ id: modelId,
+ name: models[modelId].name,
+ category: MODEL_CATEGORY.DEPLOYED,
+ algorithm: models[modelId].algorithm,
+ } as ModelItem);
+ }
+ });
+ setDeployedModels(modelItems);
+ }
+ }, [models]);
+
+ // Initialize available pretrained models
+ useEffect(() => {
+ const modelItems = [
+ {
+ id: ROBERTA_SENTENCE_TRANSFORMER.name,
+ name: ROBERTA_SENTENCE_TRANSFORMER.shortenedName,
+ category: MODEL_CATEGORY.PRETRAINED,
+ algorithm: ROBERTA_SENTENCE_TRANSFORMER.algorithm,
+ },
+ {
+ id: MPNET_SENTENCE_TRANSFORMER.name,
+ name: MPNET_SENTENCE_TRANSFORMER.shortenedName,
+ category: MODEL_CATEGORY.PRETRAINED,
+ algorithm: MPNET_SENTENCE_TRANSFORMER.algorithm,
+ },
+ {
+ id: BERT_SENTENCE_TRANSFORMER.name,
+ name: BERT_SENTENCE_TRANSFORMER.shortenedName,
+ category: MODEL_CATEGORY.PRETRAINED,
+ algorithm: BERT_SENTENCE_TRANSFORMER.algorithm,
+ },
+ ];
+ setPretrainedModels(modelItems);
+ }, []);
+
+ // Update the valid available models when the radio selection changes.
+ // e.g., only show deployed models when 'deployed' button is selected
+ useEffect(() => {
+ if (selectedRadioId !== undefined) {
+ if (selectedRadioId === MODEL_CATEGORY.DEPLOYED) {
+ setSelectableModels(deployedModels);
+ } else {
+ setSelectableModels(pretrainedModels);
+ }
+ }
+ }, [selectedRadioId, deployedModels, pretrainedModels]);
+
+ return (
+
+ {({ field, form }: FieldProps) => {
+ // a hook to update the model category and trigger reloading
+ // of valid models to select from
+ useEffect(() => {
+ setSelectedRadioId(field.value.category);
+ }, [field.value.category]);
+ return (
+
+
+ Learn more
+
+
+ ) : undefined
+ }
+ helpText={props.field.helpText || undefined}
+ >
+ <>
+ {
+ // if user selects a new category:
+ // 1. clear the saved ID
+ // 2. update the field category
+ form.setFieldValue(formField, {
+ id: '',
+ category: radioId,
+ } as ModelFormValue);
+ props.onFormChange();
+ }}
+ >
+
+
+ ({
+ value: option.id,
+ inputDisplay: (
+ <>
+ {option.name}
+
+ ),
+ dropdownDisplay: (
+ <>
+ {option.name}
+
+ {option.category}
+
+
+ {option.algorithm}
+
+
+ ),
+ disabled: false,
+ } as EuiSuperSelectOption)
+ )}
+ valueOfSelected={field.value.id || ''}
+ onChange={(option: string) => {
+ form.setFieldValue(formField, {
+ id: option,
+ category: selectedRadioId,
+ } as ModelFormValue);
+ props.onFormChange();
+ }}
+ isInvalid={isFieldInvalid(
+ props.componentId,
+ props.field.id,
+ errors,
+ touched
+ )}
+ />
+
+
+ );
+ }}
+
+ );
+}
diff --git a/public/pages/workflow_detail/component_details/input_fields/select_field.tsx b/public/pages/workflow_detail/component_details/input_fields/select_field.tsx
index 709fe18c..38017e04 100644
--- a/public/pages/workflow_detail/component_details/input_fields/select_field.tsx
+++ b/public/pages/workflow_detail/component_details/input_fields/select_field.tsx
@@ -19,7 +19,6 @@ import {
getInitialValue,
isFieldInvalid,
} from '../../../../../common';
-import { AppState } from '../../../../store';
interface SelectFieldProps {
field: IComponentField;
@@ -32,21 +31,15 @@ interface SelectFieldProps {
* options.
*/
export function SelectField(props: SelectFieldProps) {
- // Redux store state
- // Initial store is fetched when loading base page. We don't
- // re-fetch here as it could overload client-side if user clicks back and forth /
- // keeps re-rendering this component (and subsequently re-fetching data) as they're building flows
- const models = useSelector((state: AppState) => state.models.models);
-
// Options state
const [options, setOptions] = useState([]);
// Populate options depending on the select type
useEffect(() => {
- if (props.field.selectType === 'model' && models) {
- setOptions(Object.keys(models));
+ // TODO: figure out how we want to utilize select types to customize the options
+ if (props.field.selectType === 'model') {
}
- }, [models]);
+ }, []);
const formField = `${props.componentId}.${props.field.id}`;
const { errors, touched } = useFormikContext();
diff --git a/public/pages/workflow_detail/resources/resource_list.tsx b/public/pages/workflow_detail/resources/resource_list.tsx
index 0acb0f9f..72a10f1f 100644
--- a/public/pages/workflow_detail/resources/resource_list.tsx
+++ b/public/pages/workflow_detail/resources/resource_list.tsx
@@ -23,10 +23,16 @@ interface ResourceListProps {
export function ResourceList(props: ResourceListProps) {
const [allResources, setAllResources] = useState([]);
- // Hook to initialize all resources
+ // Hook to initialize all resources. Reduce to unique IDs, since
+ // the backend resources may include the same resource multiple times
+ // (e.g., register and deploy steps persist the same model ID resource)
useEffect(() => {
if (props.workflow?.resourcesCreated) {
- setAllResources(props.workflow.resourcesCreated);
+ const resourcesMap = {} as { [id: string]: WorkflowResource };
+ props.workflow.resourcesCreated.forEach((resource) => {
+ resourcesMap[resource.id] = resource;
+ });
+ setAllResources(Object.values(resourcesMap));
}
}, [props.workflow?.resourcesCreated]);
diff --git a/public/pages/workflow_detail/utils/workflow_to_template_utils.ts b/public/pages/workflow_detail/utils/workflow_to_template_utils.ts
index 3e9ce61f..399129d8 100644
--- a/public/pages/workflow_detail/utils/workflow_to_template_utils.ts
+++ b/public/pages/workflow_detail/utils/workflow_to_template_utils.ts
@@ -20,6 +20,15 @@ import {
CreateIndexNode,
TemplateFlow,
TemplateEdge,
+ ModelFormValue,
+ MODEL_CATEGORY,
+ RegisterPretrainedModelNode,
+ PretrainedSentenceTransformer,
+ ROBERTA_SENTENCE_TRANSFORMER,
+ MPNET_SENTENCE_TRANSFORMER,
+ BERT_SENTENCE_TRANSFORMER,
+ REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE,
+ generateId,
} from '../../../../common';
/**
@@ -68,13 +77,13 @@ function toProvisionTemplateFlow(
edges: ReactFlowEdge[]
): TemplateFlow {
const prevNodes = [] as ReactFlowComponent[];
- const templateNodes = [] as TemplateNode[];
+ const finalTemplateNodes = [] as TemplateNode[];
const templateEdges = [] as TemplateEdge[];
nodes.forEach((node) => {
- const templateNode = toTemplateNode(node, prevNodes, edges);
+ const templateNodes = toTemplateNodes(node, prevNodes, edges);
// it may be undefined if the node is not convertible for some reason
- if (templateNode) {
- templateNodes.push(templateNode);
+ if (templateNodes) {
+ finalTemplateNodes.push(...templateNodes);
prevNodes.push(node);
}
});
@@ -84,20 +93,20 @@ function toProvisionTemplateFlow(
});
return {
- nodes: templateNodes,
+ nodes: finalTemplateNodes,
edges: templateEdges,
};
}
-function toTemplateNode(
+function toTemplateNodes(
flowNode: ReactFlowComponent,
prevNodes: ReactFlowComponent[],
edges: ReactFlowEdge[]
-): TemplateNode | undefined {
+): TemplateNode[] | undefined {
if (flowNode.data.baseClasses?.includes(COMPONENT_CLASS.ML_TRANSFORMER)) {
- return toIngestPipelineNode(flowNode);
+ return transformerToTemplateNodes(flowNode);
} else if (flowNode.data.baseClasses?.includes(COMPONENT_CLASS.INDEXER)) {
- return toIndexerNode(flowNode, prevNodes, edges);
+ return [indexerToTemplateNode(flowNode, prevNodes, edges)];
}
}
@@ -110,9 +119,11 @@ function toTemplateEdge(flowEdge: ReactFlowEdge): TemplateEdge {
// General fn to process all ML transform nodes. Convert into a final
// ingest pipeline with a processor specific to the final class of the node.
-function toIngestPipelineNode(
+// Optionally prepend a register pretrained model step if the selected model
+// is a pretrained and undeployed one.
+function transformerToTemplateNodes(
flowNode: ReactFlowComponent
-): CreateIngestPipelineNode {
+): TemplateNode[] {
// TODO a few improvements to make here:
// 1. Consideration of multiple ingest processors and how to collect them all, and finally create
// a single ingest pipeline with all of them, in the same order as done on the UI
@@ -120,17 +131,54 @@ function toIngestPipelineNode(
switch (flowNode.data.type) {
case COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER:
default: {
- const { modelId, inputField, vectorField } = componentDataToFormik(
+ const { model, inputField, vectorField } = componentDataToFormik(
flowNode.data
- );
+ ) as {
+ model: ModelFormValue;
+ inputField: string;
+ vectorField: string;
+ };
+ const modelId = model.id;
+ const ingestPipelineName = generateId('ingest_pipeline');
- return {
+ let registerModelStep = undefined as
+ | RegisterPretrainedModelNode
+ | undefined;
+ if (model.category === MODEL_CATEGORY.PRETRAINED) {
+ const pretrainedModel = [
+ ROBERTA_SENTENCE_TRANSFORMER,
+ MPNET_SENTENCE_TRANSFORMER,
+ BERT_SENTENCE_TRANSFORMER,
+ ].find(
+ // the model ID in the form will be the unique name of the pretrained model
+ (model) => model.name === modelId
+ ) as PretrainedSentenceTransformer;
+ registerModelStep = {
+ id: REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE,
+ type: REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE,
+ user_inputs: {
+ name: pretrainedModel.name,
+ description: pretrainedModel.description,
+ model_format: pretrainedModel.format,
+ version: pretrainedModel.version,
+ deploy: true,
+ },
+ } as RegisterPretrainedModelNode;
+ }
+
+ // The model ID depends on if we are consuming it from a previous pretrained model step,
+ // or directly from the user
+ const finalModelId =
+ registerModelStep !== undefined
+ ? `\${{${REGISTER_LOCAL_PRETRAINED_MODEL_STEP_TYPE}.model_id}}`
+ : modelId;
+
+ const createIngestPipelineStep = {
id: flowNode.data.id,
type: CREATE_INGEST_PIPELINE_STEP_TYPE,
user_inputs: {
- // TODO: expose as customizable
- pipeline_id: 'test-pipeline',
- model_id: modelId,
+ pipeline_id: ingestPipelineName,
+ model_id: finalModelId,
input_field: inputField,
output_field: vectorField,
configurations: {
@@ -138,7 +186,7 @@ function toIngestPipelineNode(
processors: [
{
text_embedding: {
- model_id: modelId,
+ model_id: finalModelId,
field_map: {
[inputField]: vectorField,
},
@@ -147,13 +195,17 @@ function toIngestPipelineNode(
],
},
},
- };
+ } as CreateIngestPipelineNode;
+
+ return registerModelStep !== undefined
+ ? [registerModelStep, createIngestPipelineStep]
+ : [createIngestPipelineStep];
}
}
}
// General fn to convert an indexer node to a final CreateIndexNode template node.
-function toIndexerNode(
+function indexerToTemplateNode(
flowNode: ReactFlowComponent,
prevNodes: ReactFlowComponent[],
edges: ReactFlowEdge[]
@@ -191,6 +243,8 @@ function toIndexerNode(
properties: {
[vectorField]: {
type: 'knn_vector',
+ // TODO: remove hardcoding, fetch from the selected model
+ // (existing or from pretrained configuration)
dimension: 768,
method: {
engine: 'lucene',
diff --git a/public/pages/workflow_detail/workspace/resizable_workspace.tsx b/public/pages/workflow_detail/workspace/resizable_workspace.tsx
index 37a37599..90cba2fb 100644
--- a/public/pages/workflow_detail/workspace/resizable_workspace.tsx
+++ b/public/pages/workflow_detail/workspace/resizable_workspace.tsx
@@ -33,6 +33,7 @@ import {
processNodes,
reduceToTemplate,
ReactFlowEdge,
+ APP_PATH,
} from '../../../../common';
import { validateWorkspaceFlow, toTemplateFlows } from '../utils';
import {
@@ -116,6 +117,8 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) {
props.workflow !== undefined &&
!props.isNewWorkflow &&
props.workflow?.state !== WORKFLOW_STATE.NOT_STARTED;
+ // TODO: maybe remove this field. It depends on final UX if we want the
+ // workspace to be readonly once provisioned or not.
const readonly = props.workflow === undefined || isDeprovisionable;
// Loading state
@@ -168,7 +171,7 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) {
props.workflow && !props.workflow?.ui_metadata?.workspace_flow;
const missingCachedWorkflow = props.isNewWorkflow && !props.workflow;
if (missingUiFlow || missingCachedWorkflow) {
- history.replace('/workflows');
+ history.replace(APP_PATH.WORKFLOWS);
if (missingCachedWorkflow) {
getCore().notifications.toasts.addWarning('No workflow found');
} else {
@@ -320,6 +323,17 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) {
components.
)}
+ {isDeprovisionable && isDirty && (
+
+ Changes cannot be saved until the flow has first been
+ deprovisioned.
+
+ )}
{
const { workflow } = result;
- history.replace(`/workflows/${workflow.id}`);
+ history.replace(
+ `${APP_PATH.WORKFLOWS}/${workflow.id}`
+ );
history.go(0);
})
.catch((error: any) => {
@@ -455,7 +471,7 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) {
diff --git a/public/pages/workflows/new_workflow/use_case.tsx b/public/pages/workflows/new_workflow/use_case.tsx
index 111c4f7c..a6eceacf 100644
--- a/public/pages/workflows/new_workflow/use_case.tsx
+++ b/public/pages/workflows/new_workflow/use_case.tsx
@@ -13,7 +13,7 @@ import {
EuiHorizontalRule,
EuiButton,
} from '@elastic/eui';
-import { NEW_WORKFLOW_ID_URL, PLUGIN_ID } from '../../../../common';
+import { APP_PATH, NEW_WORKFLOW_ID_URL, PLUGIN_ID } from '../../../../common';
interface UseCaseProps {
title: string;
@@ -44,7 +44,7 @@ export function UseCase(props: UseCaseProps) {
disabled={false}
isLoading={false}
onClick={props.onClick}
- href={`${PLUGIN_ID}#/workflows/${NEW_WORKFLOW_ID_URL}`}
+ href={`${PLUGIN_ID}#${APP_PATH.WORKFLOWS}/${NEW_WORKFLOW_ID_URL}`}
>
Go
diff --git a/public/utils/constants.ts b/public/utils/constants.ts
index 0f3b75f6..74dd67b7 100644
--- a/public/utils/constants.ts
+++ b/public/utils/constants.ts
@@ -15,7 +15,7 @@ export enum APP_PATH {
}
export const BREADCRUMBS = Object.freeze({
- FLOW_FRAMEWORK: { text: 'Flow Framework', href: '#/' },
+ FLOW_FRAMEWORK: { text: 'Flow Framework' },
WORKFLOWS: { text: 'Workflows', href: `#${APP_PATH.WORKFLOWS}` },
});
diff --git a/public/utils/utils.ts b/public/utils/utils.ts
index 3f9a7b8b..76b6715e 100644
--- a/public/utils/utils.ts
+++ b/public/utils/utils.ts
@@ -18,6 +18,7 @@ import {
ReactFlowComponent,
Workflow,
WorkflowTemplate,
+ ModelFormValue,
} from '../../common';
// Append 16 random characters
@@ -82,6 +83,7 @@ export function reduceToTemplate(workflow: Workflow): WorkflowTemplate {
lastUpdated,
lastLaunched,
state,
+ resourcesCreated,
...workflowTemplate
} = workflow;
return workflowTemplate;
@@ -96,6 +98,13 @@ export function getInitialValue(fieldType: FieldType): FieldValue {
case 'select': {
return '';
}
+ case 'model': {
+ return {
+ id: '',
+ category: undefined,
+ algorithm: undefined,
+ } as ModelFormValue;
+ }
case 'json': {
return {};
}
@@ -162,6 +171,13 @@ function getFieldSchema(field: IComponentField): Schema {
baseSchema = yup.string().min(1, 'Too short').max(70, 'Too long');
break;
}
+ case 'model': {
+ baseSchema = yup.object().shape({
+ id: yup.string().min(1, 'Too short').max(70, 'Too long').required(),
+ category: yup.string().required(),
+ });
+ break;
+ }
case 'json': {
baseSchema = yup.object().json();
break;
diff --git a/server/routes/helpers.ts b/server/routes/helpers.ts
index b4d39991..afb404f7 100644
--- a/server/routes/helpers.ts
+++ b/server/routes/helpers.ts
@@ -6,6 +6,8 @@
import {
DEFAULT_NEW_WORKFLOW_STATE_TYPE,
INDEX_NOT_FOUND_EXCEPTION,
+ MODEL_ALGORITHM,
+ MODEL_STATE,
Model,
ModelDict,
WORKFLOW_RESOURCE_TYPE,
@@ -85,13 +87,25 @@ export function getWorkflowsFromResponses(
export function getModelsFromResponses(modelHits: any[]): ModelDict {
const modelDict = {} as ModelDict;
modelHits.forEach((modelHit: any) => {
- const modelId = modelHit._source?.model_id;
- // in case of schema changes from ML plugin, this may crash. That is ok, as the error
- // produced will help expose the root cause
- modelDict[modelId] = {
- id: modelId,
- algorithm: modelHit._source?.algorithm,
- } as Model;
+ // search model API returns hits for each deployed model chunk. ignore these hits
+ if (modelHit._source.chunk_number === undefined) {
+ const modelId = modelHit._id;
+ // in case of schema changes from ML plugin, this may crash. That is ok, as the error
+ // produced will help expose the root cause
+ modelDict[modelId] = {
+ id: modelId,
+ name: modelHit._source?.name,
+ // @ts-ignore
+ algorithm: MODEL_ALGORITHM[modelHit._source?.algorithm],
+ // @ts-ignore
+ state: MODEL_STATE[modelHit._source?.model_state],
+ modelConfig: {
+ modelType: modelHit._source?.model_config?.model_type,
+ embeddingDimension:
+ modelHit._source?.model_config?.embedding_dimension,
+ },
+ } as Model;
+ }
});
return modelDict;
}