-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add partial support of advanced input transformations (ingest) (#220)
Signed-off-by: Tyler Ohlsen <[email protected]> (cherry picked from commit 62d18e4)
- Loading branch information
1 parent
a273876
commit b42437d
Showing
23 changed files
with
791 additions
and
56 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
7 changes: 7 additions & 0 deletions
7
public/pages/workflow_detail/workflow_inputs/processor_inputs/index.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| export * from './ml_processor_inputs'; | ||
| export * from './processor_inputs'; |
198 changes: 198 additions & 0 deletions
198
public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,198 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| import React, { useState } from 'react'; | ||
| import { useFormikContext } from 'formik'; | ||
| import { | ||
| EuiButton, | ||
| EuiButtonEmpty, | ||
| EuiCodeBlock, | ||
| EuiCodeEditor, | ||
| EuiFlexGroup, | ||
| EuiFlexItem, | ||
| EuiModal, | ||
| EuiModalBody, | ||
| EuiModalFooter, | ||
| EuiModalHeader, | ||
| EuiModalHeaderTitle, | ||
| EuiSpacer, | ||
| EuiText, | ||
| } from '@elastic/eui'; | ||
| import { | ||
| IProcessorConfig, | ||
| IngestPipelineConfig, | ||
| PROCESSOR_CONTEXT, | ||
| SimulateIngestPipelineDoc, | ||
| SimulateIngestPipelineResponse, | ||
| WorkflowConfig, | ||
| WorkflowFormValues, | ||
| } from '../../../../../common'; | ||
| import { formikToIngestPipeline, generateId } from '../../../../utils'; | ||
| import { simulatePipeline, useAppDispatch } from '../../../../store'; | ||
| import { getCore } from '../../../../services'; | ||
|
|
||
| interface InputTransformModalProps { | ||
| uiConfig: WorkflowConfig; | ||
| config: IProcessorConfig; | ||
| context: PROCESSOR_CONTEXT; | ||
| onClose: () => void; | ||
| onConfirm: () => void; | ||
| } | ||
|
|
||
| /** | ||
| * A modal to configure advanced JSON-to-JSON transforms into a model's expected input | ||
| */ | ||
| export function InputTransformModal(props: InputTransformModalProps) { | ||
| const dispatch = useAppDispatch(); | ||
| const { values } = useFormikContext<WorkflowFormValues>(); | ||
|
|
||
| // source input / transformed output state | ||
| const [sourceInput, setSourceInput] = useState<string>('[]'); | ||
| const [transformedOutput, setTransformedOutput] = useState<string>('TODO'); | ||
|
|
||
| return ( | ||
| <EuiModal onClose={props.onClose} style={{ width: '70vw' }}> | ||
| <EuiModalHeader> | ||
| <EuiModalHeaderTitle> | ||
| <p>{`Configure input transform`}</p> | ||
| </EuiModalHeaderTitle> | ||
| </EuiModalHeader> | ||
| <EuiModalBody> | ||
| <EuiFlexGroup direction="column"> | ||
| <EuiFlexItem> | ||
| <> | ||
| <EuiText>Expected input</EuiText> | ||
| <EuiButton | ||
| style={{ width: '100px' }} | ||
| onClick={async () => { | ||
| switch (props.context) { | ||
| case PROCESSOR_CONTEXT.INGEST: { | ||
| const curIngestPipeline = formikToIngestPipeline( | ||
| values, | ||
| props.uiConfig, | ||
| props.config.id | ||
| ); | ||
| // if there are preceding processors, we need to generate the ingest pipeline | ||
| // up to this point and simulate, in order to get the latest transformed | ||
| // version of the docs | ||
| if (curIngestPipeline !== undefined) { | ||
| const curDocs = prepareDocsForSimulate( | ||
| values.ingest.docs, | ||
| values.ingest.index.name | ||
| ); | ||
| await dispatch( | ||
| simulatePipeline({ | ||
| pipeline: curIngestPipeline as IngestPipelineConfig, | ||
| docs: curDocs, | ||
| }) | ||
| ) | ||
| .unwrap() | ||
| .then((resp: SimulateIngestPipelineResponse) => { | ||
| setSourceInput(unwrapTransformedDocs(resp)); | ||
| }) | ||
| .catch((error: any) => { | ||
| getCore().notifications.toasts.addDanger( | ||
| `Failed to fetch input schema` | ||
| ); | ||
| }); | ||
| } else { | ||
| setSourceInput(values.ingest.docs); | ||
| } | ||
| break; | ||
| } | ||
| // TODO: complete for search request / search response contexts | ||
| } | ||
| }} | ||
| > | ||
| Fetch | ||
| </EuiButton> | ||
| <EuiSpacer size="s" /> | ||
| <EuiCodeBlock fontSize="m" isCopyable={false}> | ||
| {sourceInput} | ||
| </EuiCodeBlock> | ||
| </> | ||
| </EuiFlexItem> | ||
| <EuiFlexItem> | ||
| <> | ||
| <EuiText>Define transform with JSONPath</EuiText> | ||
| <EuiSpacer size="s" /> | ||
| <EuiCodeEditor | ||
| mode="json" | ||
| theme="textmate" | ||
| value={`TODO`} | ||
| readOnly={false} | ||
| setOptions={{ | ||
| fontSize: '12px', | ||
| autoScrollEditorIntoView: true, | ||
| }} | ||
| tabSize={2} | ||
| /> | ||
| </> | ||
| </EuiFlexItem> | ||
| <EuiFlexItem> | ||
| <> | ||
| <EuiText>Expected output</EuiText> | ||
| <EuiSpacer size="s" /> | ||
| <EuiCodeBlock fontSize="m" isCopyable={false}> | ||
| {transformedOutput} | ||
| </EuiCodeBlock> | ||
| </> | ||
| </EuiFlexItem> | ||
| </EuiFlexGroup> | ||
| </EuiModalBody> | ||
| <EuiModalFooter> | ||
| <EuiButtonEmpty onClick={props.onClose}>Cancel</EuiButtonEmpty> | ||
| <EuiButton onClick={props.onConfirm} fill={true} color="primary"> | ||
| Save | ||
| </EuiButton> | ||
| </EuiModalFooter> | ||
| </EuiModal> | ||
| ); | ||
| } | ||
|
|
||
| // docs are expected to be in a certain format to be passed to the simulate ingest pipeline API. | ||
| // for details, see https://opensearch.org/docs/latest/ingest-pipelines/simulate-ingest | ||
| function prepareDocsForSimulate( | ||
| docs: string, | ||
| indexName: string | ||
| ): SimulateIngestPipelineDoc[] { | ||
| const preparedDocs = [] as SimulateIngestPipelineDoc[]; | ||
| const docObjs = JSON.parse(docs) as {}[]; | ||
| docObjs.forEach((doc) => { | ||
| preparedDocs.push({ | ||
| _index: indexName, | ||
| _id: generateId(), | ||
| _source: doc, | ||
| }); | ||
| }); | ||
| return preparedDocs; | ||
| } | ||
|
|
||
| // docs are returned in a certain format from the simulate ingest pipeline API. We want | ||
| // to format them into a more readable string to display | ||
| function unwrapTransformedDocs( | ||
| simulatePipelineResponse: SimulateIngestPipelineResponse | ||
| ) { | ||
| let errorDuringSimulate = undefined as string | undefined; | ||
| const transformedDocsSources = simulatePipelineResponse.docs.map( | ||
| (transformedDoc) => { | ||
| if (transformedDoc.error !== undefined) { | ||
| errorDuringSimulate = transformedDoc.error.reason || ''; | ||
| } else { | ||
| return transformedDoc.doc._source; | ||
| } | ||
| } | ||
| ); | ||
|
|
||
| // there is an edge case where simulate may fail if there is some server-side or OpenSearch issue when | ||
| // running ingest (e.g., hitting rate limits on remote model) | ||
| // We pull out any returned error from a document and propagate it to the user. | ||
| if (errorDuringSimulate !== undefined) { | ||
| getCore().notifications.toasts.addDanger( | ||
| `Failed to simulate ingest on all documents: ${errorDuringSimulate}` | ||
| ); | ||
| } | ||
| return JSON.stringify(transformedDocsSources, undefined, 2); | ||
| } |
Oops, something went wrong.