Add existing indices as a source data option; improve config autofilling

public/pages/workflow_detail/resizable_workspace.tsx

@@ -4,7 +4,6 @@
  */
 
 import React, { useRef, useState, useEffect } from 'react';
-import { useSelector } from 'react-redux';
 import { Form, Formik } from 'formik';
 import * as yup from 'yup';
 import {
@@ -21,6 +20,7 @@ import {
   WorkflowConfig,
   WorkflowFormValues,
   WorkflowSchema,
+  customStringify,
 } from '../../../common';
 import {
   isValidUiWorkflow,
@@ -286,11 +286,7 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) {
       </EuiFlexItem>
       <EuiFlexItem grow={7}>
         <EuiCodeBlock language="json" fontSize="m" isCopyable={false}>
-          {JSON.stringify(
-            reduceToTemplate(props.workflow as Workflow),
-            undefined,
-            2
-          )}
+          {customStringify(reduceToTemplate(props.workflow as Workflow))}
         </EuiCodeBlock>
       </EuiFlexItem>
     </EuiFlexGroup>

public/pages/workflow_detail/workflow_detail.tsx

@@ -21,6 +21,7 @@ import { getCore } from '../../services';
 import { WorkflowDetailHeader } from './components';
 import {
   AppState,
+  catIndices,
   getWorkflow,
   searchModels,
   useAppDispatch,
@@ -101,9 +102,11 @@ export function WorkflowDetail(props: WorkflowDetailProps) {
   // On initial load:
   // - fetch workflow
   // - fetch available models as their IDs may be used when building flows
+  // - fetch all indices
   useEffect(() => {
     dispatch(getWorkflow({ workflowId, dataSourceId }));
     dispatch(searchModels({ apiBody: FETCH_ALL_QUERY, dataSourceId }));
+    dispatch(catIndices({ pattern: '*,-.*', dataSourceId }));
   }, []);
 
   return errorMessage.includes(ERROR_GETTING_WORKFLOW_MSG) ||

public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx

@@ -4,6 +4,7 @@
  */
 
 import React, { useEffect, useState } from 'react';
+import { useSelector } from 'react-redux';
 import { useFormikContext } from 'formik';
 import {
   EuiSmallButton,
@@ -18,19 +19,44 @@ import {
   EuiSpacer,
   EuiText,
   EuiTitle,
+  EuiFilterGroup,
+  EuiSmallFilterButton,
+  EuiSuperSelectOption,
+  EuiCompressedSuperSelect,
 } from '@elastic/eui';
 import { JsonField } from '../input_fields';
-import { WorkspaceFormValues } from '../../../../../common';
+import {
+  FETCH_ALL_QUERY,
+  SearchHit,
+  WorkspaceFormValues,
+  customStringify,
+} from '../../../../../common';
+import { AppState, searchIndex, useAppDispatch } from '../../../../store';
+import { getDataSourceId } from '../../../../utils';
 
 interface SourceDataProps {
   setIngestDocs: (docs: string) => void;
 }
 
+enum SOURCE_OPTIONS {
+  MANUAL = 'manual',
+  UPLOAD = 'upload',
+  EXISTING_INDEX = 'existing_index',
+}
+
 /**
  * Input component for configuring the source data for ingest.
  */
 export function SourceData(props: SourceDataProps) {
+  const dispatch = useAppDispatch();
+  const dataSourceId = getDataSourceId();
   const { values, setFieldValue } = useFormikContext<WorkspaceFormValues>();
+  const indices = useSelector((state: AppState) => state.opensearch.indices);
+
+  // selected option state
+  const [selectedOption, setSelectedOption] = useState<SOURCE_OPTIONS>(
+    SOURCE_OPTIONS.MANUAL
+  );
 
   // edit modal state
   const [isEditModalOpen, setIsEditModalOpen] = useState<boolean>(false);
@@ -43,8 +69,40 @@ export function SourceData(props: SourceDataProps) {
     }
   };
 
-  // Hook to listen when the docs form value changes.
-  // Try to set the ingestDocs if possible
+  // selected index state. when an index is selected, update the form value
+  const [selectedIndex, setSelectedIndex] = useState<string | undefined>(
+    undefined
+  );
+  useEffect(() => {
+    if (selectedIndex !== undefined) {
+      dispatch(
+        searchIndex({
+          apiBody: {
+            index: selectedIndex,
+            body: FETCH_ALL_QUERY,
+            searchPipeline: '_none',
+          },
+          dataSourceId,
+        })
+      )
+        .unwrap()
+        .then((resp) => {
+          const docObjs = resp.hits?.hits
+            ?.slice(0, 5)
+            ?.map((hit: SearchHit) => hit?._source);
+          setFieldValue('ingest.docs', customStringify(docObjs));
+        });
+    }
+  }, [selectedIndex]);
+
+  // hook to clear out the selected index when switching options
+  useEffect(() => {
+    if (selectedOption !== SOURCE_OPTIONS.EXISTING_INDEX) {
+      setSelectedIndex(undefined);
+    }
+  }, [selectedOption]);
+
+  // hook to listen when the docs form value changes.
   useEffect(() => {
     if (values?.ingest?.docs) {
       props.setIngestDocs(values.ingest.docs);
@@ -65,22 +123,74 @@ export function SourceData(props: SourceDataProps) {
           </EuiModalHeader>
           <EuiModalBody>
             <>
-              <EuiText color="subdued">
-                Upload a JSON file or enter manually.
-              </EuiText>{' '}
-              <EuiSpacer size="s" />
-              <EuiCompressedFilePicker
-                accept="application/json"
-                multiple={false}
-                initialPromptText="Upload file"
-                onChange={(files) => {
-                  if (files && files.length > 0) {
-                    fileReader.readAsText(files[0]);
+              <EuiFilterGroup>
+                <EuiSmallFilterButton
+                  id={SOURCE_OPTIONS.MANUAL}
+                  hasActiveFilters={selectedOption === SOURCE_OPTIONS.MANUAL}
+                  onClick={() => setSelectedOption(SOURCE_OPTIONS.MANUAL)}
+                >
+                  Manual
+                </EuiSmallFilterButton>
+                <EuiSmallFilterButton
+                  id={SOURCE_OPTIONS.UPLOAD}
+                  hasActiveFilters={selectedOption === SOURCE_OPTIONS.UPLOAD}
+                  onClick={() => setSelectedOption(SOURCE_OPTIONS.UPLOAD)}
+                >
+                  Upload
+                </EuiSmallFilterButton>
+                <EuiSmallFilterButton
+                  id={SOURCE_OPTIONS.EXISTING_INDEX}
+                  hasActiveFilters={
+                    selectedOption === SOURCE_OPTIONS.EXISTING_INDEX
                   }
-                }}
-                display="default"
-              />
-              <EuiSpacer size="s" />
+                  onClick={() =>
+                    setSelectedOption(SOURCE_OPTIONS.EXISTING_INDEX)
+                  }
+                >
+                  Existing index
+                </EuiSmallFilterButton>
+              </EuiFilterGroup>
+              <EuiSpacer size="m" />
+              {selectedOption === SOURCE_OPTIONS.UPLOAD && (
+                <>
+                  <EuiCompressedFilePicker
+                    accept="application/json"
+                    multiple={false}
+                    initialPromptText="Upload file"
+                    onChange={(files) => {
+                      if (files && files.length > 0) {
+                        fileReader.readAsText(files[0]);
+                      }
+                    }}
+                    display="default"
+                  />
+                  <EuiSpacer size="s" />
+                </>
+              )}
+              {selectedOption === SOURCE_OPTIONS.EXISTING_INDEX && (
+                <>
+                  <EuiCompressedSuperSelect
+                    options={Object.values(indices).map(
+                      (option) =>
+                        ({
+                          value: option.name,
+                          inputDisplay: <EuiText>{option.name}</EuiText>,
+                          disabled: false,
+                        } as EuiSuperSelectOption<string>)
+                    )}
+                    valueOfSelected={selectedIndex}
+                    onChange={(option) => {
+                      setSelectedIndex(option);
+                    }}
+                    isInvalid={false}
+                  />
+                  <EuiSpacer size="xs" />
+                  <EuiText color="subdued" size="s">
+                    Up to 5 sample documents will be automatically populated.
+                  </EuiText>
+                  <EuiSpacer size="s" />
+                </>
+              )}
               <JsonField
                 label="Documents"
                 fieldPath={'ingest.docs'}

public/pages/workflow_detail/workflow_inputs/input_fields/select_field.tsx

@@ -7,7 +7,7 @@ import React from 'react';
 import { Field, FieldProps, getIn, useFormikContext } from 'formik';
 import {
   EuiCompressedFormRow,
-  EuiSuperSelect,
+  EuiCompressedSuperSelect,
   EuiSuperSelectOption,
   EuiText,
 } from '@elastic/eui';
@@ -31,7 +31,7 @@ export function SelectField(props: SelectFieldProps) {
       {({ field, form }: FieldProps) => {
         return (
           <EuiCompressedFormRow label={camelCaseToTitleString(props.field.id)}>
-            <EuiSuperSelect
+            <EuiCompressedSuperSelect
               options={
                 props.field.selectOptions
                   ? props.field.selectOptions.map(

public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx

@@ -257,12 +257,10 @@ export function InputTransformModal(props: InputTransformModalProps) {
                         .unwrap()
                         .then(async (resp) => {
                           setSourceInput(
-                            JSON.stringify(
+                            customStringify(
                               resp.hits.hits.map(
                                 (hit: SearchHit) => hit._source
-                              ),
-                              undefined,
-                              2
+                              )
                             )
                           );
                         })

public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx

@@ -210,12 +210,10 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
                         .unwrap()
                         .then(async (resp) => {
                           setSourceInput(
-                            JSON.stringify(
+                            customStringify(
                               resp.hits.hits.map(
                                 (hit: SearchHit) => hit._source
-                              ),
-                              undefined,
-                              2
+                              )
                             )
                           );
                         })

public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx

@@ -12,20 +12,19 @@ import {
   EuiFlexGroup,
   EuiFlexItem,
   EuiCompressedFormRow,
-  EuiSuperSelect,
+  EuiCompressedSuperSelect,
   EuiSuperSelectOption,
   EuiText,
   EuiTitle,
   EuiSpacer,
 } from '@elastic/eui';
-import { SearchHit, WorkflowFormValues } from '../../../../../common';
-import { JsonField } from '../input_fields';
 import {
-  AppState,
-  catIndices,
-  searchIndex,
-  useAppDispatch,
-} from '../../../../store';
+  SearchHit,
+  WorkflowFormValues,
+  customStringify,
+} from '../../../../../common';
+import { JsonField } from '../input_fields';
+import { AppState, searchIndex, useAppDispatch } from '../../../../store';
 import { getDataSourceId } from '../../../../utils/utils';
 import { EditQueryModal } from './edit_query_modal';
 
@@ -74,14 +73,6 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) {
     }
   }, [values?.search?.request]);
 
-  // Initialization hook to fetch available indices (if applicable)
-  useEffect(() => {
-    if (!ingestEnabled) {
-      // Fetch all indices besides system indices
-      dispatch(catIndices({ pattern: '*,-.*', dataSourceId }));
-    }
-  }, []);
-
   return (
     <>
       {isEditModalOpen && (
@@ -104,7 +95,7 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) {
                 readOnly={true}
               />
             ) : (
-              <EuiSuperSelect
+              <EuiCompressedSuperSelect
                 options={Object.values(indices).map(
                   (option) =>
                     ({
@@ -162,10 +153,8 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) {
                   .unwrap()
                   .then(async (resp) => {
                     props.setQueryResponse(
-                      JSON.stringify(
-                        resp.hits.hits.map((hit: SearchHit) => hit._source),
-                        undefined,
-                        2
+                      customStringify(
+                        resp.hits.hits.map((hit: SearchHit) => hit._source)
                       )
                     );
                   })

public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx

@@ -534,10 +534,8 @@ export function WorkflowInputs(props: WorkflowInputsProps) {
             .unwrap()
             .then(async (resp) => {
               props.setQueryResponse(
-                JSON.stringify(
-                  resp.hits.hits.map((hit: SearchHit) => hit._source),
-                  undefined,
-                  2
+                customStringify(
+                  resp.hits.hits.map((hit: SearchHit) => hit._source)
                 )
               );
             })