diff --git a/changelogs/fragments/7289.yml b/changelogs/fragments/7289.yml new file mode 100644 index 000000000000..b181433dbeca --- /dev/null +++ b/changelogs/fragments/7289.yml @@ -0,0 +1,2 @@ +feat: +- Add DataSet dropdown with index patterns and indices ([#7289](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7289)) \ No newline at end of file diff --git a/changelogs/fragments/7368.yml b/changelogs/fragments/7368.yml new file mode 100644 index 000000000000..c8316dc939f0 --- /dev/null +++ b/changelogs/fragments/7368.yml @@ -0,0 +1,2 @@ +feat: +- [Discover] Adds a dataset selector for Discover ([#7368](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7368)) \ No newline at end of file diff --git a/package.json b/package.json index 43cdf376b155..b84348cdd6f7 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ "start": "scripts/use_node scripts/opensearch_dashboards --dev", "start:docker": "scripts/use_node scripts/opensearch_dashboards --dev --opensearch.hosts=$OPENSEARCH_HOSTS --opensearch.ignoreVersionMismatch=true --server.host=$SERVER_HOST", "start:security": "scripts/use_node scripts/opensearch_dashboards --dev --security", - "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true", + "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['home:useNewHomePage']=true", "debug": "scripts/use_node --nolazy --inspect scripts/opensearch_dashboards --dev", "debug-break": "scripts/use_node --nolazy --inspect-brk scripts/opensearch_dashboards --dev", "lint": "yarn run lint:es && yarn run lint:style", diff --git a/src/plugins/query_enhancements/public/data_source_connection/utils/index.ts b/src/plugins/data/common/data_sets/index.ts similarity index 70% rename from src/plugins/query_enhancements/public/data_source_connection/utils/index.ts rename to src/plugins/data/common/data_sets/index.ts index 9eccc9e6f35a..9f269633f307 100644 --- a/src/plugins/query_enhancements/public/data_source_connection/utils/index.ts +++ b/src/plugins/data/common/data_sets/index.ts @@ -3,4 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -export * from './create_extension'; +export * from './types'; diff --git a/src/plugins/data/common/data_sets/types.ts b/src/plugins/data/common/data_sets/types.ts new file mode 100644 index 000000000000..4f4236d20275 --- /dev/null +++ b/src/plugins/data/common/data_sets/types.ts @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** @public **/ +export enum SIMPLE_DATA_SOURCE_TYPES { + DEFAULT = 'data-source', + EXTERNAL = 'external-source', +} + +/** @public **/ +export enum SIMPLE_DATA_SET_TYPES { + INDEX_PATTERN = 'index-pattern', + TEMPORARY = 'temporary', + TEMPORARY_ASYNC = 'temporary-async', +} + +export interface SimpleObject { + id?: string; + title?: string; + dataSourceRef?: SimpleDataSource; +} + +export interface SimpleDataSource { + id: string; + name: string; + indices?: SimpleObject[]; + type: SIMPLE_DATA_SOURCE_TYPES; +} + +export interface SimpleDataSet extends SimpleObject { + fields?: any[]; + timeFieldName?: string; + timeFields?: any[]; + type?: SIMPLE_DATA_SET_TYPES; +} diff --git a/src/plugins/data/common/index.ts b/src/plugins/data/common/index.ts index d7b7e56e2280..0250a6ec2e01 100644 --- a/src/plugins/data/common/index.ts +++ b/src/plugins/data/common/index.ts @@ -31,6 +31,7 @@ export * from './constants'; export * from './opensearch_query'; export * from './data_frames'; +export * from './data_sets'; export * from './field_formats'; export * from './field_mapping'; export * from './index_patterns'; diff --git a/src/plugins/data/common/search/opensearch_search/types.ts b/src/plugins/data/common/search/opensearch_search/types.ts index f90a3f1de245..6d24e8c36dd3 100644 --- a/src/plugins/data/common/search/opensearch_search/types.ts +++ b/src/plugins/data/common/search/opensearch_search/types.ts @@ -48,6 +48,10 @@ export interface ISearchOptions { * Use this option to enable support for long numerals. */ withLongNumeralsSupport?: boolean; + /** + * Use this option to enable support for async. + */ + isAsync?: boolean; } export type ISearchRequestParams> = { diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index d9518e6a6cab..3236da2e572c 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -324,7 +324,12 @@ export class SearchSource { const dataFrame = createDataFrame({ name: searchRequest.index.title || searchRequest.index, fields: [], - ...(rawQueryString && { meta: { queryConfig: parseRawQueryString(rawQueryString) } }), + ...(rawQueryString && { + meta: { + queryConfig: parseRawQueryString(rawQueryString), + ...(searchRequest.dataSourceId && { dataSource: searchRequest.dataSourceId }), + }, + }), }); await this.setDataFrame(dataFrame); return this.getDataFrame(); @@ -353,6 +358,10 @@ export class SearchSource { if (getConfig(UI_SETTINGS.COURIER_BATCH_SEARCHES)) { response = await this.legacyFetch(searchRequest, options); } else if (this.isUnsupportedRequest(searchRequest)) { + const indexPattern = this.getField('index'); + searchRequest.dataSourceId = indexPattern?.dataSourceRef?.id; + + options = { ...options, isAsync: this.getField('type')?.includes('async') }; response = await this.fetchExternalSearch(searchRequest, options); } else { const indexPattern = this.getField('index'); diff --git a/src/plugins/data/common/types.ts b/src/plugins/data/common/types.ts index 6a1f6e5a99d3..1670fbf72d5d 100644 --- a/src/plugins/data/common/types.ts +++ b/src/plugins/data/common/types.ts @@ -35,6 +35,7 @@ export * from './query/types'; export * from './osd_field_types/types'; export * from './index_patterns/types'; export * from './data_frames/types'; +export * from './data_sets/types'; /** * If a service is being shared on both the client and the server, and diff --git a/src/plugins/data/public/antlr/shared/utils.ts b/src/plugins/data/public/antlr/shared/utils.ts index b2658b304e0f..94236aa81387 100644 --- a/src/plugins/data/public/antlr/shared/utils.ts +++ b/src/plugins/data/public/antlr/shared/utils.ts @@ -12,7 +12,7 @@ export interface IDataSourceRequestHandlerParams { } export const getRawSuggestionData$ = ( - connectionsService, + connectionsService: { getSelectedConnection$: any }, dataSourceReuqstHandler: ({ dataSourceId, title, @@ -21,11 +21,11 @@ export const getRawSuggestionData$ = ( ) => connectionsService.getSelectedConnection$().pipe( distinctUntilChanged(), - switchMap((connection) => { + switchMap((connection: any) => { if (connection === undefined) { return from(defaultReuqstHandler()); } - const dataSourceId = connection?.id; + const dataSourceId = connection?.dataSource?.id; const title = connection?.attributes?.title; return from(dataSourceReuqstHandler({ dataSourceId, title })); }) @@ -34,8 +34,8 @@ export const getRawSuggestionData$ = ( export const fetchData = ( tables: string[], queryFormatter: (table: string, dataSourceId?: string, title?: string) => any, - api, - connectionService + api: any, + connectionService: any ): Promise => { return new Promise((resolve, reject) => { getRawSuggestionData$( @@ -65,8 +65,8 @@ export const fetchData = ( ); } ).subscribe({ - next: (dataFrames) => resolve(dataFrames), - error: (err) => { + next: (dataFrames: any) => resolve(dataFrames), + error: (err: any) => { // TODO: pipe error to UI reject(err); }, @@ -74,7 +74,11 @@ export const fetchData = ( }); }; -export const fetchTableSchemas = (tables: string[], api, connectionService): Promise => { +export const fetchTableSchemas = ( + tables: string[], + api: any, + connectionService: any +): Promise => { return fetchData( tables, (table, dataSourceId, title) => ({ @@ -96,8 +100,8 @@ export const fetchTableSchemas = (tables: string[], api, connectionService): Pro export const fetchColumnValues = ( tables: string[], column: string, - api, - connectionService + api: any, + connectionService: any ): Promise => { return fetchData( tables, diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index f1ac419e9ec1..46390b819821 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -445,6 +445,8 @@ export { QueryEditorTopRow, // for BWC, keeping the old name IUiStart as DataPublicPluginStartUi, + DataSetNavigator, + DataSetOption, } from './ui'; /** diff --git a/src/plugins/data/public/ui/_index.scss b/src/plugins/data/public/ui/_index.scss index f7c738b8d09f..4aa425041f58 100644 --- a/src/plugins/data/public/ui/_index.scss +++ b/src/plugins/data/public/ui/_index.scss @@ -2,5 +2,6 @@ @import "./typeahead/index"; @import "./saved_query_management/index"; @import "./query_string_input/index"; +@import "./dataset_navigator/index"; @import "./query_editor/index"; @import "./shard_failure_modal/shard_failure_modal"; diff --git a/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss b/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss new file mode 100644 index 000000000000..73a8c8719500 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +.datasetNavigator { + min-width: 350px; + border-bottom: $euiBorderThin !important; +} + +.dataSetNavigatorFormWrapper { + padding: $euiSizeS; +} + +.dataSetNavigator__loading { + padding: $euiSizeS; +} diff --git a/src/plugins/data/public/ui/dataset_navigator/_index.scss b/src/plugins/data/public/ui/dataset_navigator/_index.scss new file mode 100644 index 000000000000..53acdffad43d --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/_index.scss @@ -0,0 +1 @@ +@import "./dataset_navigator"; diff --git a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx new file mode 100644 index 000000000000..6d19cd20d551 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { HttpStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { DataSetNavigator, DataSetNavigatorProps } from './'; + +// Updated function signature to include additional dependencies +export function createDataSetNavigator( + savedObjectsClient: SavedObjectsClientContract, + http: HttpStart +) { + // Return a function that takes props, omitting the dependencies from the props type + return (props: Omit) => ( + + ); +} diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx new file mode 100644 index 000000000000..fefe37b36a00 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -0,0 +1,660 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useCallback, useEffect, useState } from 'react'; +import { + EuiButton, + EuiButtonEmpty, + EuiContextMenu, + EuiContextMenuPanelItemDescriptor, + EuiForm, + EuiFormRow, + EuiLoadingSpinner, + EuiPanel, + EuiPopover, + EuiSelect, +} from '@elastic/eui'; +import { HttpStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import _ from 'lodash'; +import { i18n } from '@osd/i18n'; +import { + SIMPLE_DATA_SET_TYPES, + SIMPLE_DATA_SOURCE_TYPES, + SimpleDataSet, + SimpleDataSource, + SimpleObject, +} from '../../../common'; +import { + useLoadDatabasesToCache, + useLoadExternalDataSourcesToCache, + useLoadTablesToCache, +} from './lib/catalog_cache/cache_loader'; +import { CatalogCacheManager } from './lib/catalog_cache/cache_manager'; +import { CachedDataSourceStatus, DataSet, DirectQueryLoadingStatus } from './lib/types'; +import { + getIndexPatterns, + getNotifications, + getQueryService, + getSearchService, + getUiService, +} from '../../services'; +import { + fetchDataSources, + fetchIndexPatterns, + fetchIndices, + isCatalogCacheFetching, + fetchIfExternalDataSourcesEnabled, +} from './lib'; + +export interface DataSetNavigatorProps { + dataSet: SimpleDataSet | undefined; + savedObjectsClient?: SavedObjectsClientContract; + http?: HttpStart; + onSelectDataSet: (dataSet: SimpleDataSet) => void; +} + +export const DataSetNavigator = (props: DataSetNavigatorProps) => { + const { savedObjectsClient, http, onSelectDataSet } = props; + const searchService = getSearchService(); + const queryService = getQueryService(); + const uiService = getUiService(); + const indexPatternsService = getIndexPatterns(); + const notifications = getNotifications(); + + const [isOpen, setIsOpen] = useState(false); + const [isLoading, setIsLoading] = useState(false); + const [isMounted, setIsMounted] = useState(false); + const [isExternalDataSourcesEnabled, setIsExternalDataSourcesEnabled] = useState(false); + const [selectedTimeFieldName, setSelectedTimeFieldName] = useState(); + const [selectedObject, setSelectedObject] = useState(); + const [selectedDataSource, setSelectedDataSource] = useState(); + const [selectedDataSourceObjects, setSelectedDataSourceObjects] = useState([]); + const [selectedExternalDataSource, setSelectedExternalDataSource] = useState(); + const [dataSources, setDataSources] = useState([]); + const [externalDataSources, setExternalDataSources] = useState([]); + const [indexPatterns, setIndexPatterns] = useState([]); + + const [selectedDatabase, setSelectedDatabase] = useState(); + const [cachedDatabases, setCachedDatabases] = useState([]); + const [cachedTables, setCachedTables] = useState([]); + const [failed, setFailed] = useState(false); + + const { + loadStatus: dataSourcesLoadStatus, + loadExternalDataSources: startLoadingDataSources, + } = useLoadExternalDataSourcesToCache(http!, notifications); + const { + loadStatus: databasesLoadStatus, + startLoading: startLoadingDatabases, + } = useLoadDatabasesToCache(http!, notifications); + const { loadStatus: tablesLoadStatus, startLoading: startLoadingTables } = useLoadTablesToCache( + http!, + notifications + ); + + const closePopover = () => setIsOpen(false); + + const handleExternalDataSourcesRefresh = () => { + if (!isCatalogCacheFetching(dataSourcesLoadStatus) && dataSources.length > 0) { + startLoadingDataSources(dataSources.map((dataSource) => dataSource.id)); + } + }; + + useEffect(() => { + if (!isMounted) { + setIsLoading(true); + Promise.all([ + fetchIndexPatterns(savedObjectsClient!, ''), + fetchDataSources(savedObjectsClient!), + fetchIfExternalDataSourcesEnabled(http!), + ]) + .then(([defaultIndexPatterns, defaultDataSources, isExternalDSEnabled]) => { + setIsExternalDataSourcesEnabled(isExternalDSEnabled); + setIndexPatterns(defaultIndexPatterns); + setDataSources(defaultDataSources); + const selectedPattern = defaultIndexPatterns.find( + (pattern) => pattern.id === props.dataSet?.id + ); + if (selectedPattern) { + onSelectDataSet({ + id: selectedPattern.id ?? selectedPattern.title, + title: selectedPattern.title, + type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, + }); + } + }) + .finally(() => { + setIsMounted(true); + setIsLoading(false); + }); + } + }, [ + indexPatternsService, + savedObjectsClient, + isMounted, + http, + onSelectDataSet, + props.dataSet?.id, + ]); + + useEffect(() => { + const status = dataSourcesLoadStatus.toLowerCase(); + const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); + if (status === DirectQueryLoadingStatus.SUCCESS) { + setExternalDataSources( + externalDataSourcesCache.externalDataSources.map((ds) => ({ + id: ds.dataSourceRef, + name: ds.name, + type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, + })) + ); + } else if ( + status === DirectQueryLoadingStatus.CANCELED || + status === DirectQueryLoadingStatus.FAILED + ) { + setFailed(true); + } + }, [dataSourcesLoadStatus]); + + // Retrieve databases from cache upon success + useEffect(() => { + const status = databasesLoadStatus.toLowerCase(); + if (selectedExternalDataSource) { + const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( + selectedExternalDataSource?.name, + selectedExternalDataSource?.id + ); + if (status === DirectQueryLoadingStatus.SUCCESS) { + setCachedDatabases(dataSourceCache.databases); + } else if ( + status === DirectQueryLoadingStatus.CANCELED || + status === DirectQueryLoadingStatus.FAILED + ) { + setFailed(true); + } + } + }, [selectedExternalDataSource, databasesLoadStatus]); + + // Start loading databases for datasource + const handleSelectExternalDataSource = useCallback( + async (externalDataSource) => { + if (selectedExternalDataSource) { + const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( + selectedExternalDataSource.name, + selectedExternalDataSource.id + ); + if ( + (dataSourceCache.status === CachedDataSourceStatus.Empty || + dataSourceCache.status === CachedDataSourceStatus.Failed) && + !isCatalogCacheFetching(databasesLoadStatus) + ) { + startLoadingDatabases({ + dataSourceName: selectedExternalDataSource.name, + dataSourceMDSId: selectedExternalDataSource.id, + }); + } else if (dataSourceCache.status === CachedDataSourceStatus.Updated) { + setCachedDatabases(dataSourceCache.databases); + } + } + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [selectedExternalDataSource] + ); + + // Start loading tables for selected database + const handleSelectExternalDatabase = useCallback( + async (externalDatabase) => { + if (selectedExternalDataSource && externalDatabase) { + let databaseCache; + try { + databaseCache = CatalogCacheManager.getDatabase( + selectedExternalDataSource.name, + selectedDatabase, + selectedExternalDataSource.id + ); + } catch (error) { + return; + } + if ( + databaseCache.status === CachedDataSourceStatus.Empty || + (databaseCache.status === CachedDataSourceStatus.Failed && + !isCatalogCacheFetching(tablesLoadStatus)) + ) { + await startLoadingTables({ + dataSourceName: selectedExternalDataSource.name, + databaseName: externalDatabase, + dataSourceMDSId: selectedExternalDataSource.id, + }); + setSelectedDatabase(externalDatabase); + } else if (databaseCache.status === CachedDataSourceStatus.Updated) { + setCachedTables(databaseCache.tables); + } + } + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [selectedExternalDataSource, selectedDatabase] + ); + + // Retrieve tables from cache upon success + useEffect(() => { + if (selectedExternalDataSource && selectedDatabase) { + const tablesStatus = tablesLoadStatus.toLowerCase(); + let databaseCache; + try { + databaseCache = CatalogCacheManager.getDatabase( + selectedExternalDataSource.name, + selectedDatabase, + selectedExternalDataSource.id + ); + } catch (error) { + return; + } + if (tablesStatus === DirectQueryLoadingStatus.SUCCESS) { + setCachedTables(databaseCache.tables); + } else if ( + tablesStatus === DirectQueryLoadingStatus.CANCELED || + tablesStatus === DirectQueryLoadingStatus.FAILED + ) { + setFailed(true); + } + } + }, [selectedExternalDataSource, selectedDatabase, tablesLoadStatus]); + + const handleSelectedDataSource = useCallback( + async (source) => { + if (source) { + setIsLoading(true); + await fetchIndices(searchService, source.id).then((indices) => { + const objects = indices.map((indexName: string) => ({ + id: indexName, + title: indexName, + dataSourceRef: { + id: source.id, + name: source.name, + type: source.type, + }, + })); + setSelectedDataSourceObjects(objects); + setIsLoading(false); + }); + } + }, + [searchService] + ); + + const handleSelectedObject = useCallback( + async (object) => { + setIsLoading(true); + if (object) { + const fields = await indexPatternsService.getFieldsForWildcard({ + pattern: object.title, + dataSourceId: object.dataSourceRef?.id, + }); + + const timeFields = fields.filter((field: any) => field.type === 'date'); + const timeFieldName = timeFields?.length > 0 ? timeFields[0].name : undefined; + setSelectedTimeFieldName(timeFieldName); + setSelectedObject({ + id: object.id, + title: object.title, + fields, + timeFields, + timeFieldName: selectedTimeFieldName, + dataSourceRef: object.dataSourceRef, + type: SIMPLE_DATA_SET_TYPES.TEMPORARY, + }); + setIsLoading(false); + } + }, + [indexPatternsService, selectedTimeFieldName] + ); + + const handleSelectedDataSet = useCallback( + async (selectedDataSet: SimpleDataSet) => { + const getInitialQuery = (dataSet: SimpleDataSet) => { + const language = uiService.Settings.getUserQueryLanguage(); + const input = uiService.Settings.getQueryEnhancements(language)?.searchBar?.queryStringInput + ?.initialValue; + + if (!dataSet || !input) + return { + query: '', + language, + }; + + return { + query: input.replace('', dataSet.title!), + language, + }; + }; + + const onDataSetSelected = async (dataSet: SimpleDataSet) => { + if ( + dataSet.type === SIMPLE_DATA_SET_TYPES.TEMPORARY || + dataSet.type === SIMPLE_DATA_SET_TYPES.TEMPORARY_ASYNC + ) { + const fieldsMap = dataSet.fields?.reduce((acc: any, field: any) => { + acc[field.name] = field; + return acc; + }, {}); + const temporaryIndexPattern = await indexPatternsService.create( + { + id: dataSet.id, + title: dataSet.title, + // type: dataSet.type, + fields: fieldsMap, + dataSourceRef: { + id: dataSet.dataSourceRef?.id!, + name: dataSet.dataSourceRef?.name!, + type: dataSet.dataSourceRef?.type!, + }, + timeFieldName: dataSet.timeFieldName, + }, + true + ); + indexPatternsService.saveToCache(temporaryIndexPattern.title, temporaryIndexPattern); + } + + CatalogCacheManager.addRecentDataSet({ + id: dataSet.id, + name: dataSet.title ?? dataSet.id!, + dataSourceRef: dataSet.dataSourceRef?.id, + }); + searchService.df.clear(); + onSelectDataSet({ + id: dataSet.id, + title: dataSet.title, + dataSourceRef: dataSet.dataSourceRef, + timeFieldName: dataSet.timeFieldName, + type: dataSet.type, + }); + queryService.queryString.setQuery(getInitialQuery(dataSet)); + closePopover(); + }; + + if (selectedDataSet) { + await onDataSetSelected(selectedDataSet); + } + }, + [ + indexPatternsService, + onSelectDataSet, + queryService.queryString, + searchService.df, + uiService.Settings, + ] + ); + + const RefreshButton = ( + + ); + + const LoadingSpinner = ( + + + + ); + + const indexPatternsLabel = i18n.translate('data.query.dataSetNavigator.indexPatternsName', { + defaultMessage: 'Index patterns', + }); + const indicesLabel = i18n.translate('data.query.dataSetNavigator.indicesName', { + defaultMessage: 'Indexes', + }); + const S3DataSourcesLabel = i18n.translate('data.query.dataSetNavigator.S3DataSourcesLabel', { + defaultMessage: 'S3', + }); + + return ( + setIsOpen(!isOpen)} + > + {`${props.dataSet?.dataSourceRef?.name ? `${props.dataSet.dataSourceRef?.name}::` : ''}${ + props.dataSet?.title + }`} + + } + isOpen={isOpen} + closePopover={closePopover} + anchorPosition="downLeft" + panelPaddingSize="none" + > + 0 + ? [ + { + name: 'Recently Used', + panel: 8, + }, + ] + : []), + { + name: indexPatternsLabel, + panel: 1, + }, + { + name: indicesLabel, + panel: 2, + }, + ...(isExternalDataSourcesEnabled + ? [ + { + name: S3DataSourcesLabel, + panel: 4, + onClick: async () => { + const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); + if ( + (externalDataSourcesCache.status === CachedDataSourceStatus.Empty || + externalDataSourcesCache.status === CachedDataSourceStatus.Failed) && + !isCatalogCacheFetching(dataSourcesLoadStatus) && + dataSources.length > 0 + ) { + startLoadingDataSources(dataSources.map((dataSource) => dataSource.id)); + } else if ( + externalDataSourcesCache.status === CachedDataSourceStatus.Updated + ) { + await handleSelectExternalDataSource( + externalDataSourcesCache.externalDataSources.map((ds) => ({ + id: ds.dataSourceRef, + name: ds.name, + type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, + })) + ); + } + }, + }, + ] + : []), + ], + }, + { + id: 1, + title: indexPatternsLabel, + items: indexPatterns.flatMap((indexPattern, indexNum, arr) => [ + { + name: indexPattern.title, + onClick: async () => { + await handleSelectedDataSet({ + id: indexPattern.id ?? indexPattern.title, + title: indexPattern.title, + fields: indexPattern.fields, + timeFieldName: indexPattern.timeFieldName, + type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, + }); + }, + }, + ...(indexNum < arr.length - 1 ? [{ isSeparator: true }] : []), + ]) as EuiContextMenuPanelItemDescriptor[], + content:
{isLoading && LoadingSpinner}
, + }, + { + id: 2, + title: 'Clusters', + items: [ + ...dataSources.map((dataSource) => ({ + name: dataSource.name, + panel: 3, + onClick: async () => await handleSelectedDataSource(dataSource), + })), + ], + content:
{isLoading && LoadingSpinner}
, + }, + { + id: 3, + title: selectedDataSource?.name ?? indicesLabel, + items: selectedDataSourceObjects.map((object) => ({ + name: object.title, + panel: 7, + onClick: async () => + await handleSelectedObject({ ...object, type: SIMPLE_DATA_SET_TYPES.TEMPORARY }), + })), + content:
{isLoading && LoadingSpinner}
, + }, + { + id: 4, + title: ( +
+ {S3DataSourcesLabel} + {CatalogCacheManager.getExternalDataSourcesCache().status === + CachedDataSourceStatus.Updated && RefreshButton} +
+ ), + items: [ + ...externalDataSources.map((ds) => ({ + name: ds.name, + onClick: () => setSelectedExternalDataSource(ds), + panel: 5, + })), + ], + content:
{dataSourcesLoadStatus && LoadingSpinner}
, + }, + { + id: 5, + title: selectedExternalDataSource ? selectedExternalDataSource.name : 'Databases', + items: [ + ...cachedDatabases.map((db) => ({ + name: db.name, + onClick: async () => { + await handleSelectExternalDatabase(db.name); + }, + panel: 6, + })), + ], + content:
{isCatalogCacheFetching(databasesLoadStatus) && LoadingSpinner}
, + }, + { + id: 6, + title: selectedDatabase ? selectedDatabase : 'Tables', + items: [ + ...cachedTables.map((table) => ({ + name: table.name, + onClick: async () => { + await handleSelectedDataSet({ + id: table.name, + title: `${selectedExternalDataSource!.name}.${selectedDatabase}.${table.name}`, + dataSourceRef: { + id: selectedExternalDataSource!.id, + name: selectedExternalDataSource!.name, + type: selectedExternalDataSource!.type, + }, + type: SIMPLE_DATA_SET_TYPES.TEMPORARY_ASYNC, + }); + }, + })), + ], + content:
{isCatalogCacheFetching(tablesLoadStatus) && LoadingSpinner}
, + }, + { + id: 7, + title: selectedObject?.title, + content: + isLoading || !selectedObject ? ( +
{LoadingSpinner}
+ ) : ( + + + 0 + ? [ + ...selectedObject!.timeFields.map((field: any) => ({ + value: field.name, + text: field.name, + })), + ] + : []), + { value: 'no-time-filter', text: "I don't want to use a time filter" }, + ]} + onChange={(event) => { + setSelectedTimeFieldName( + event.target.value !== 'no-time-filter' ? event.target.value : undefined + ); + }} + aria-label="Select a date field" + /> + + { + await handleSelectedDataSet({ + ...selectedObject, + timeFieldName: selectedTimeFieldName, + } as SimpleDataSet); + }} + > + Select + + + ), + }, + { + id: 8, + title: 'Recently Used', + items: CatalogCacheManager.getRecentDataSets().map((ds) => ({ + name: ds.name, + onClick: async () => + await handleSelectedDataSet({ + id: ds.id, + title: ds.name, + dataSourceRef: { + id: ds.dataSourceRef!, + name: ds.dataSourceRef!, + type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, + }, + type: SIMPLE_DATA_SET_TYPES.TEMPORARY_ASYNC, + }), + })), + }, + ]} + /> + + ); +}; + +// eslint-disable-next-line import/no-default-export +export default DataSetNavigator; diff --git a/src/plugins/data/public/ui/dataset_navigator/index.tsx b/src/plugins/data/public/ui/dataset_navigator/index.tsx new file mode 100644 index 000000000000..238f2c704e44 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/index.tsx @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { DataSetNavigator, DataSetNavigatorProps } from './dataset_navigator'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts new file mode 100644 index 000000000000..0526cfd51212 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { HttpFetchOptionsWithPath, IHttpInterceptController } from 'opensearch-dashboards/public'; +import { SECURITY_DASHBOARDS_LOGOUT_URL } from '../constants'; +import { CatalogCacheManager } from './cache_manager'; + +export function catalogRequestIntercept(): any { + return ( + fetchOptions: Readonly, + _controller: IHttpInterceptController + ) => { + if (fetchOptions.path.includes(SECURITY_DASHBOARDS_LOGOUT_URL)) { + // Clears all user catalog cache details + CatalogCacheManager.clearDataSourceCache(); + CatalogCacheManager.clearAccelerationsCache(); + CatalogCacheManager.clearExternalDataSourcesCache(); + CatalogCacheManager.clearRecentDataSetsCache(); + } + }; +} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx new file mode 100644 index 000000000000..9a42bfc74aa8 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx @@ -0,0 +1,470 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useEffect, useRef, useState } from 'react'; +import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public'; +import { ASYNC_POLLING_INTERVAL, SPARK_HIVE_TABLE_REGEX, SPARK_PARTITION_INFO } from '../constants'; +import { + AsyncPollingResult, + CachedColumn, + CachedDataSourceStatus, + CachedTable, + LoadCacheType, + StartLoadingParams, + DirectQueryLoadingStatus, + DirectQueryRequest, +} from '../types'; +import { getAsyncSessionId, setAsyncSessionId } from '../utils/query_session_utils'; +import { + addBackticksIfNeeded, + combineSchemaAndDatarows, + get as getObjValue, + formatError, +} from '../utils/shared'; +import { usePolling } from '../utils/use_polling'; +import { SQLService } from '../requests/sql'; +import { CatalogCacheManager } from './cache_manager'; +import { fetchExternalDataSources } from '../utils'; + +export const updateDatabasesToCache = ( + dataSourceName: string, + pollingResult: AsyncPollingResult, + dataSourceMDSId?: string +) => { + const cachedDataSource = CatalogCacheManager.getOrCreateDataSource( + dataSourceName, + dataSourceMDSId + ); + + const currentTime = new Date().toUTCString(); + + if (!pollingResult) { + CatalogCacheManager.addOrUpdateDataSource( + { + ...cachedDataSource, + databases: [], + lastUpdated: currentTime, + status: CachedDataSourceStatus.Failed, + ...(dataSourceMDSId && { dataSourceMDSId }), + }, + dataSourceMDSId + ); + return; + } + + const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); + const newDatabases = combinedData.map((row: any) => ({ + name: row.namespace, + tables: [], + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + })); + + CatalogCacheManager.addOrUpdateDataSource( + { + ...cachedDataSource, + databases: newDatabases, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + ...(dataSourceMDSId && { dataSourceMDSId }), + }, + dataSourceMDSId + ); +}; + +export const updateTablesToCache = ( + dataSourceName: string, + databaseName: string, + pollingResult: AsyncPollingResult, + dataSourceMDSId?: string +) => { + try { + const cachedDatabase = CatalogCacheManager.getDatabase( + dataSourceName, + databaseName, + dataSourceMDSId + ); + const currentTime = new Date().toUTCString(); + + if (!pollingResult) { + CatalogCacheManager.updateDatabase( + dataSourceName, + { + ...cachedDatabase, + tables: [], + lastUpdated: currentTime, + status: CachedDataSourceStatus.Failed, + }, + dataSourceMDSId + ); + return; + } + + const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); + const newTables = combinedData + .filter((row: any) => !SPARK_HIVE_TABLE_REGEX.test(row.information)) + .map((row: any) => ({ + name: row.tableName, + })); + + CatalogCacheManager.updateDatabase( + dataSourceName, + { + ...cachedDatabase, + tables: newTables, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + }, + dataSourceMDSId + ); + } catch (error) { + // eslint-disable-next-line no-console + console.error(error); + } +}; + +export const updateAccelerationsToCache = ( + dataSourceName: string, + pollingResult: AsyncPollingResult, + dataSourceMDSId?: string +) => { + const currentTime = new Date().toUTCString(); + + if (!pollingResult) { + CatalogCacheManager.addOrUpdateAccelerationsByDataSource({ + name: dataSourceName, + accelerations: [], + lastUpdated: currentTime, + status: CachedDataSourceStatus.Failed, + ...(dataSourceMDSId && { dataSourceMDSId }), + }); + return; + } + + const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); + + const newAccelerations: any[] = combinedData.map((row: any) => ({ + flintIndexName: row.flint_index_name, + type: row.kind === 'mv' ? 'materialized' : row.kind, + database: row.database, + table: row.table, + indexName: row.index_name, + autoRefresh: row.auto_refresh, + status: row.status, + })); + + CatalogCacheManager.addOrUpdateAccelerationsByDataSource({ + name: dataSourceName, + accelerations: newAccelerations, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + ...(dataSourceMDSId && { dataSourceMDSId }), + }); +}; + +export const updateTableColumnsToCache = ( + dataSourceName: string, + databaseName: string, + tableName: string, + pollingResult: AsyncPollingResult, + dataSourceMDSId?: string +) => { + try { + if (!pollingResult) { + return; + } + const cachedDatabase = CatalogCacheManager.getDatabase( + dataSourceName, + databaseName, + dataSourceMDSId + ); + const currentTime = new Date().toUTCString(); + + const combinedData: Array<{ col_name: string; data_type: string }> = combineSchemaAndDatarows( + pollingResult.schema, + pollingResult.datarows + ); + + const tableColumns: CachedColumn[] = []; + for (const row of combinedData) { + if (row.col_name === SPARK_PARTITION_INFO) { + break; + } + tableColumns.push({ + fieldName: row.col_name, + dataType: row.data_type, + }); + } + + const newTables: CachedTable[] = cachedDatabase.tables.map((ts) => + ts.name === tableName ? { ...ts, columns: tableColumns } : { ...ts } + ); + + if (cachedDatabase.status === CachedDataSourceStatus.Updated) { + CatalogCacheManager.updateDatabase( + dataSourceName, + { + ...cachedDatabase, + tables: newTables, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + }, + dataSourceMDSId + ); + } + } catch (error) { + // eslint-disable-next-line no-console + console.error(error); + } +}; + +export const updateToCache = ( + pollResults: any, + loadCacheType: LoadCacheType, + dataSourceName: string, + databaseName?: string, + tableName?: string, + dataSourceMDSId?: string +) => { + switch (loadCacheType) { + case 'databases': + updateDatabasesToCache(dataSourceName, pollResults, dataSourceMDSId); + break; + case 'tables': + updateTablesToCache(dataSourceName, databaseName!, pollResults, dataSourceMDSId); + break; + case 'accelerations': + updateAccelerationsToCache(dataSourceName, pollResults, dataSourceMDSId); + break; + case 'tableColumns': + updateTableColumnsToCache( + dataSourceName, + databaseName!, + tableName!, + pollResults, + dataSourceMDSId + ); + default: + break; + } +}; + +export const createLoadQuery = ( + loadCacheType: LoadCacheType, + dataSourceName: string, + databaseName?: string, + tableName?: string +) => { + let query; + switch (loadCacheType) { + case 'databases': + query = `SHOW SCHEMAS IN ${addBackticksIfNeeded(dataSourceName)}`; + break; + case 'tables': + query = `SHOW TABLE EXTENDED IN ${addBackticksIfNeeded( + dataSourceName + )}.${addBackticksIfNeeded(databaseName!)} LIKE '*'`; + break; + case 'accelerations': + query = `SHOW FLINT INDEX in ${addBackticksIfNeeded(dataSourceName)}`; + break; + case 'tableColumns': + query = `DESC ${addBackticksIfNeeded(dataSourceName)}.${addBackticksIfNeeded( + databaseName! + )}.${addBackticksIfNeeded(tableName!)}`; + break; + default: + query = ''; + break; + } + return query; +}; + +export const useLoadToCache = ( + loadCacheType: LoadCacheType, + http: HttpStart, + notifications: NotificationsStart +) => { + const sqlService = new SQLService(http); + const [currentDataSourceName, setCurrentDataSourceName] = useState(''); + const [currentDatabaseName, setCurrentDatabaseName] = useState(''); + const [currentTableName, setCurrentTableName] = useState(''); + const [loadStatus, setLoadStatus] = useState( + DirectQueryLoadingStatus.INITIAL + ); + const dataSourceMDSClientId = useRef(''); + + const { + data: pollingResult, + loading: _pollingLoading, + error: pollingError, + startPolling, + stopPolling: stopLoading, + } = usePolling((params) => { + return sqlService.fetchWithJobId(params, dataSourceMDSClientId.current); + }, ASYNC_POLLING_INTERVAL); + + const onLoadingFailed = () => { + setLoadStatus(DirectQueryLoadingStatus.FAILED); + updateToCache( + null, + loadCacheType, + currentDataSourceName, + currentDatabaseName, + currentTableName, + dataSourceMDSClientId.current + ); + }; + + const startLoading = async ({ + dataSourceName, + dataSourceMDSId, + databaseName, + tableName, + }: StartLoadingParams) => { + setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); + setCurrentDataSourceName(dataSourceName); + setCurrentDatabaseName(databaseName); + setCurrentTableName(tableName); + dataSourceMDSClientId.current = dataSourceMDSId || ''; + + let requestPayload: DirectQueryRequest = { + lang: 'sql', + query: createLoadQuery(loadCacheType, dataSourceName, databaseName, tableName), + datasource: dataSourceName, + }; + + const sessionId = getAsyncSessionId(dataSourceName); + if (sessionId) { + requestPayload = { ...requestPayload, sessionId }; + } + await sqlService + .fetch(requestPayload, dataSourceMDSId) + .then((result) => { + setAsyncSessionId(dataSourceName, getObjValue(result, 'sessionId', null)); + if (result.queryId) { + startPolling({ + queryId: result.queryId, + }); + } else { + // eslint-disable-next-line no-console + console.error('No query id found in response'); + onLoadingFailed(); + } + }) + .catch((e) => { + onLoadingFailed(); + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + e.body?.message + ); + notifications.toasts.addError(formattedError, { + title: 'Query Failed', + }); + // eslint-disable-next-line no-console + console.error(e); + }); + }; + + useEffect(() => { + // cancel direct query + if (!pollingResult) return; + const { status: anyCaseStatus, datarows, error } = pollingResult; + const status = anyCaseStatus?.toLowerCase(); + + if (status === DirectQueryLoadingStatus.SUCCESS || datarows) { + setLoadStatus(status); + stopLoading(); + updateToCache( + pollingResult, + loadCacheType, + currentDataSourceName, + currentDatabaseName, + currentTableName, + dataSourceMDSClientId.current + ); + } else if (status === DirectQueryLoadingStatus.FAILED) { + onLoadingFailed(); + stopLoading(); + + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + error + ); + notifications.toasts.addError(formattedError, { + title: 'Query Failed', + }); + } else { + setLoadStatus(status); + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [pollingResult, pollingError]); + + return { loadStatus, startLoading, stopLoading }; +}; + +export const useLoadDatabasesToCache = (http: HttpStart, notifications: NotificationsStart) => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache( + 'databases', + http, + notifications + ); + return { loadStatus, startLoading, stopLoading }; +}; + +export const useLoadTablesToCache = (http: HttpStart, notifications: NotificationsStart) => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache('tables', http, notifications); + return { loadStatus, startLoading, stopLoading }; +}; + +export const useLoadTableColumnsToCache = (http: HttpStart, notifications: NotificationsStart) => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache( + 'tableColumns', + http, + notifications + ); + return { loadStatus, startLoading, stopLoading }; +}; + +export const useLoadAccelerationsToCache = (http: HttpStart, notifications: NotificationsStart) => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache( + 'accelerations', + http, + notifications + ); + return { loadStatus, startLoading, stopLoading }; +}; + +export const useLoadExternalDataSourcesToCache = ( + http: HttpStart, + notifications: NotificationsStart +) => { + const [loadStatus, setLoadStatus] = useState( + DirectQueryLoadingStatus.INITIAL + ); + + const loadExternalDataSources = async (connectedClusters: string[]) => { + setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); + CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Empty); + + try { + const externalDataSources = await fetchExternalDataSources(http, connectedClusters); + CatalogCacheManager.updateExternalDataSources(externalDataSources); + setLoadStatus(DirectQueryLoadingStatus.SUCCESS); + CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Updated); + } catch (error) { + setLoadStatus(DirectQueryLoadingStatus.FAILED); + CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Failed); + notifications.toasts.addError(error, { + title: 'Failed to load external datasources', + }); + } + }; + + return { loadStatus, loadExternalDataSources }; +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts new file mode 100644 index 000000000000..a55130e419fd --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts @@ -0,0 +1,416 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE, + CATALOG_CACHE_VERSION, + RECENT_DATASET_OPTIONS_CACHE, +} from '../constants'; +import { ASYNC_QUERY_ACCELERATIONS_CACHE, ASYNC_QUERY_DATASOURCE_CACHE } from '../utils/shared'; +import { + AccelerationsCacheData, + CachedAccelerationByDataSource, + CachedDataSource, + CachedDataSourceStatus, + CachedDatabase, + CachedTable, + DataSetOption, + DataSourceCacheData, + ExternalDataSource, + ExternalDataSourcesCacheData, + RecentDataSetOptionsCacheData, +} from '../types'; + +/** + * Manages caching for catalog data including data sources and accelerations. + */ +export class CatalogCacheManager { + /** + * Key for the data source cache in local storage. + */ + private static readonly datasourceCacheKey = ASYNC_QUERY_DATASOURCE_CACHE; + + /** + * Key for the accelerations cache in local storage. + */ + private static readonly accelerationsCacheKey = ASYNC_QUERY_ACCELERATIONS_CACHE; + + /** + * Key for external datasources cache in local storage + */ + private static readonly externalDataSourcesCacheKey = ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE; + + /** + * Key for recently selected datasets in local storage + */ + private static readonly recentDataSetCacheKey = RECENT_DATASET_OPTIONS_CACHE; + + // TODO: make this an advanced setting + private static readonly maxRecentDataSet = 4; + + /** + * Saves data source cache to local storage. + * @param {DataSourceCacheData} cacheData - The data source cache data to save. + */ + static saveDataSourceCache(cacheData: DataSourceCacheData): void { + sessionStorage.setItem(this.datasourceCacheKey, JSON.stringify(cacheData)); + } + + /** + * Retrieves data source cache from local storage. + * @returns {DataSourceCacheData} The retrieved data source cache. + */ + static getDataSourceCache(): DataSourceCacheData { + const catalogData = sessionStorage.getItem(this.datasourceCacheKey); + + if (catalogData) { + return JSON.parse(catalogData); + } else { + const defaultCacheObject = { version: CATALOG_CACHE_VERSION, dataSources: [] }; + this.saveDataSourceCache(defaultCacheObject); + return defaultCacheObject; + } + } + + /** + * Saves accelerations cache to local storage. + * @param {AccelerationsCacheData} cacheData - The accelerations cache data to save. + */ + static saveAccelerationsCache(cacheData: AccelerationsCacheData): void { + sessionStorage.setItem(this.accelerationsCacheKey, JSON.stringify(cacheData)); + } + + /** + * Retrieves accelerations cache from local storage. + * @returns {AccelerationsCacheData} The retrieved accelerations cache. + */ + static getAccelerationsCache(): AccelerationsCacheData { + const accelerationCacheData = sessionStorage.getItem(this.accelerationsCacheKey); + + if (accelerationCacheData) { + return JSON.parse(accelerationCacheData); + } else { + const defaultCacheObject = { + version: CATALOG_CACHE_VERSION, + dataSources: [], + }; + this.saveAccelerationsCache(defaultCacheObject); + return defaultCacheObject; + } + } + + /** + * Adds or updates a data source in the accelerations cache. + * @param {CachedAccelerationByDataSource} dataSource - The data source to add or update. + */ + static addOrUpdateAccelerationsByDataSource( + dataSource: CachedAccelerationByDataSource, + dataSourceMDSId?: string + ): void { + let index = -1; + const accCacheData = this.getAccelerationsCache(); + if (dataSourceMDSId) { + index = accCacheData.dataSources.findIndex( + (ds: CachedAccelerationByDataSource) => + ds.name === dataSource.name && ds.dataSourceMDSId === dataSourceMDSId + ); + } else { + index = accCacheData.dataSources.findIndex( + (ds: CachedAccelerationByDataSource) => ds.name === dataSource.name + ); + } + if (index !== -1) { + accCacheData.dataSources[index] = dataSource; + } else { + accCacheData.dataSources.push(dataSource); + } + this.saveAccelerationsCache(accCacheData); + } + + /** + * Retrieves accelerations cache from local storage by the datasource name. + * @param {string} dataSourceName - The name of the data source. + * @returns {CachedAccelerationByDataSource} The retrieved accelerations by datasource in cache. + * @throws {Error} If the data source is not found. + */ + static getOrCreateAccelerationsByDataSource( + dataSourceName: string, + dataSourceMDSId?: string + ): CachedAccelerationByDataSource { + const accCacheData = this.getAccelerationsCache(); + let cachedDataSource; + if (dataSourceMDSId) { + cachedDataSource = accCacheData.dataSources.find( + (ds) => ds.name === dataSourceName && ds.dataSourceMDSId === dataSourceMDSId + ); + } else { + cachedDataSource = accCacheData.dataSources.find((ds) => ds.name === dataSourceName); + } + if (cachedDataSource) return cachedDataSource; + else { + let defaultDataSourceObject: CachedAccelerationByDataSource = { + name: dataSourceName, + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + accelerations: [], + }; + + if (dataSourceMDSId !== '' && dataSourceMDSId !== undefined) { + defaultDataSourceObject = { ...defaultDataSourceObject, dataSourceMDSId }; + } + this.addOrUpdateAccelerationsByDataSource(defaultDataSourceObject, dataSourceMDSId); + return defaultDataSourceObject; + } + } + + /** + * Adds or updates a data source in the cache. + * @param {CachedDataSource} dataSource - The data source to add or update. + */ + static addOrUpdateDataSource(dataSource: CachedDataSource, dataSourceMDSId?: string): void { + const cacheData = this.getDataSourceCache(); + let index; + if (dataSourceMDSId) { + index = cacheData.dataSources.findIndex( + (ds: CachedDataSource) => + ds.name === dataSource.name && ds.dataSourceMDSId === dataSourceMDSId + ); + } + index = cacheData.dataSources.findIndex((ds: CachedDataSource) => ds.name === dataSource.name); + if (index !== -1) { + cacheData.dataSources[index] = dataSource; + } else { + cacheData.dataSources.push(dataSource); + } + this.saveDataSourceCache(cacheData); + } + + /** + * Retrieves or creates a data source with the specified name. + * @param {string} dataSourceName - The name of the data source. + * @returns {CachedDataSource} The retrieved or created data source. + */ + static getOrCreateDataSource(dataSourceName: string, dataSourceMDSId?: string): CachedDataSource { + let cachedDataSource; + if (dataSourceMDSId) { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName + ); + } else { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.name === dataSourceName + ); + } + if (cachedDataSource) { + return cachedDataSource; + } else { + let defaultDataSourceObject: CachedDataSource = { + name: dataSourceName, + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + databases: [], + }; + if (dataSourceMDSId !== '' && dataSourceMDSId !== undefined) { + defaultDataSourceObject = { ...defaultDataSourceObject, dataSourceMDSId }; + } + this.addOrUpdateDataSource(defaultDataSourceObject, dataSourceMDSId); + return defaultDataSourceObject; + } + } + + /** + * Retrieves a database from the cache. + * @param {string} dataSourceName - The name of the data source containing the database. + * @param {string} databaseName - The name of the database. + * @returns {CachedDatabase} The retrieved database. + * @throws {Error} If the data source or database is not found. + */ + static getDatabase( + dataSourceName: string, + databaseName: string, + dataSourceMDSId?: string + ): CachedDatabase { + let cachedDataSource; + if (dataSourceMDSId) { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName + ); + } else { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.name === dataSourceName + ); + } + if (!cachedDataSource) { + throw new Error('DataSource not found exception: ' + dataSourceName); + } + + const cachedDatabase = cachedDataSource.databases.find((db) => db.name === databaseName); + if (!cachedDatabase) { + throw new Error('Database not found exception: ' + databaseName); + } + + return cachedDatabase; + } + + /** + * Retrieves a table from the cache. + * @param {string} dataSourceName - The name of the data source containing the database. + * @param {string} databaseName - The name of the database. + * @param {string} tableName - The name of the database. + * @returns {Cachedtable} The retrieved database. + * @throws {Error} If the data source, database or table is not found. + */ + static getTable( + dataSourceName: string, + databaseName: string, + tableName: string, + dataSourceMDSId?: string + ): CachedTable { + const cachedDatabase = this.getDatabase(dataSourceName, databaseName, dataSourceMDSId); + + const cachedTable = cachedDatabase.tables!.find((table) => table.name === tableName); + if (!cachedTable) { + throw new Error('Table not found exception: ' + tableName); + } + return cachedTable; + } + + /** + * Updates a database in the cache. + * @param {string} dataSourceName - The name of the data source containing the database. + * @param {CachedDatabase} database - The database to be updated. + * @throws {Error} If the data source or database is not found. + */ + static updateDatabase( + dataSourceName: string, + database: CachedDatabase, + dataSourceMDSId?: string + ): void { + let cachedDataSource; + if (dataSourceMDSId) { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName + ); + } else { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.name === dataSourceName + ); + } + + if (!cachedDataSource) { + throw new Error('DataSource not found exception: ' + dataSourceName); + } + + const index = cachedDataSource.databases.findIndex((db) => db.name === database.name); + if (index !== -1) { + cachedDataSource.databases[index] = database; + this.addOrUpdateDataSource(cachedDataSource, dataSourceMDSId); + } else { + throw new Error('Database not found exception: ' + database.name); + } + } + + /** + * Clears the data source cache from local storage. + */ + static clearDataSourceCache(): void { + sessionStorage.removeItem(this.datasourceCacheKey); + this.clearExternalDataSourcesCache(); + } + + /** + * Clears the accelerations cache from local storage. + */ + static clearAccelerationsCache(): void { + sessionStorage.removeItem(this.accelerationsCacheKey); + } + + static saveExternalDataSourcesCache(cacheData: ExternalDataSourcesCacheData): void { + sessionStorage.setItem(this.externalDataSourcesCacheKey, JSON.stringify(cacheData)); + } + + static getExternalDataSourcesCache(): ExternalDataSourcesCacheData { + const externalDataSourcesData = sessionStorage.getItem(this.externalDataSourcesCacheKey); + + if (externalDataSourcesData) { + return JSON.parse(externalDataSourcesData); + } else { + const defaultCacheObject: ExternalDataSourcesCacheData = { + version: CATALOG_CACHE_VERSION, + externalDataSources: [], + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + }; + this.saveExternalDataSourcesCache(defaultCacheObject); + return defaultCacheObject; + } + } + + static updateExternalDataSources(externalDataSources: ExternalDataSource[]): void { + const currentTime = new Date().toUTCString(); + const cacheData = this.getExternalDataSourcesCache(); + cacheData.externalDataSources = externalDataSources; + cacheData.lastUpdated = currentTime; + cacheData.status = CachedDataSourceStatus.Updated; + this.saveExternalDataSourcesCache(cacheData); + } + + static getExternalDataSources(): ExternalDataSourcesCacheData { + return this.getExternalDataSourcesCache(); + } + + static clearExternalDataSourcesCache(): void { + sessionStorage.removeItem(this.externalDataSourcesCacheKey); + } + + static setExternalDataSourcesLoadingStatus(status: CachedDataSourceStatus): void { + const cacheData = this.getExternalDataSourcesCache(); + cacheData.status = status; + this.saveExternalDataSourcesCache(cacheData); + } + + static saveRecentDataSetsCache(cacheData: RecentDataSetOptionsCacheData): void { + sessionStorage.setItem(this.recentDataSetCacheKey, JSON.stringify(cacheData)); + } + + static getRecentDataSetsCache(): RecentDataSetOptionsCacheData { + const recentDataSetOptionsData = sessionStorage.getItem(this.recentDataSetCacheKey); + + if (recentDataSetOptionsData) { + return JSON.parse(recentDataSetOptionsData); + } else { + const defaultCacheObject: RecentDataSetOptionsCacheData = { + version: CATALOG_CACHE_VERSION, + recentDataSets: [], + }; + this.saveRecentDataSetsCache(defaultCacheObject); + return defaultCacheObject; + } + } + + static addRecentDataSet(dataSetOption: DataSetOption): void { + const cacheData = this.getRecentDataSetsCache(); + + cacheData.recentDataSets = cacheData.recentDataSets.filter( + (option) => option.id !== dataSetOption.id + ); + + cacheData.recentDataSets.push(dataSetOption); + + if (cacheData.recentDataSets.length > this.maxRecentDataSet) { + cacheData.recentDataSets.shift(); + } + + this.saveRecentDataSetsCache(cacheData); + } + + static getRecentDataSets(): DataSetOption[] { + return this.getRecentDataSetsCache().recentDataSets; + } + + static clearRecentDataSetsCache(): void { + sessionStorage.removeItem(this.recentDataSetCacheKey); + } +} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx new file mode 100644 index 000000000000..5449277b2bd8 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx @@ -0,0 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './cache_intercept'; +export * from './cache_loader'; +export * from './cache_manager'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts b/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts new file mode 100644 index 000000000000..e22da95ff4c6 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts @@ -0,0 +1,101 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; +export const ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE = 'async_query_external_datasources_cache'; +export const RECENT_DATASET_OPTIONS_CACHE = 'recent_dataset_options_cache'; + +export const DATA_SOURCE_NAME_URL_PARAM_KEY = 'datasourceName'; +export const DATA_SOURCE_TYPE_URL_PARAM_KEY = 'datasourceType'; +export const OLLY_QUESTION_URL_PARAM_KEY = 'olly_q'; +export const INDEX_URL_PARAM_KEY = 'indexPattern'; +export const DEFAULT_DATA_SOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; +export const DEFAULT_DATA_SOURCE_NAME = 'Default cluster'; +export const DEFAULT_DATA_SOURCE_OBSERVABILITY_DISPLAY_NAME = 'OpenSearch'; +export const DEFAULT_DATA_SOURCE_TYPE_NAME = 'Default Group'; +export const enum QUERY_LANGUAGE { + PPL = 'PPL', + SQL = 'SQL', + DQL = 'DQL', +} +export enum DATA_SOURCE_TYPES { + DEFAULT_CLUSTER_TYPE = DEFAULT_DATA_SOURCE_TYPE, + SPARK = 'spark', + S3Glue = 's3glue', +} +export const ASYNC_POLLING_INTERVAL = 2000; + +export const CATALOG_CACHE_VERSION = '1.0'; +export const ACCELERATION_DEFUALT_SKIPPING_INDEX_NAME = 'skipping'; +export const ACCELERATION_TIME_INTERVAL = [ + { text: 'millisecond(s)', value: 'millisecond' }, + { text: 'second(s)', value: 'second' }, + { text: 'minutes(s)', value: 'minute' }, + { text: 'hour(s)', value: 'hour' }, + { text: 'day(s)', value: 'day' }, + { text: 'week(s)', value: 'week' }, +]; +export const ACCELERATION_REFRESH_TIME_INTERVAL = [ + { text: 'minutes(s)', value: 'minute' }, + { text: 'hour(s)', value: 'hour' }, + { text: 'day(s)', value: 'day' }, + { text: 'week(s)', value: 'week' }, +]; + +export const ACCELERATION_ADD_FIELDS_TEXT = '(add fields here)'; +export const ACCELERATION_INDEX_NAME_REGEX = /^[a-z0-9_]+$/; +export const ACCELERATION_S3_URL_REGEX = /^(s3|s3a):\/\/[a-zA-Z0-9.\-]+/; +export const SPARK_HIVE_TABLE_REGEX = /Provider:\s*hive/; +export const SANITIZE_QUERY_REGEX = /\s+/g; +export const SPARK_TIMESTAMP_DATATYPE = 'timestamp'; +export const SPARK_STRING_DATATYPE = 'string'; + +export const ACCELERATION_INDEX_TYPES = [ + { label: 'Skipping Index', value: 'skipping' }, + { label: 'Covering Index', value: 'covering' }, + { label: 'Materialized View', value: 'materialized' }, +]; + +export const ACC_INDEX_TYPE_DOCUMENTATION_URL = + 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md'; +export const ACC_CHECKPOINT_DOCUMENTATION_URL = + 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md#create-index-options'; + +export const ACCELERATION_INDEX_NAME_INFO = `All OpenSearch acceleration indices have a naming format of pattern: \`prefix__suffix\`. They share a common prefix structure, which is \`flint____\`. Additionally, they may have a suffix that varies based on the index type. +##### Skipping Index +- For 'Skipping' indices, a fixed index name 'skipping' is used, and this name cannot be modified by the user. The suffix added to this type is \`_index\`. + - An example of a 'Skipping' index name would be: \`flint_mydatasource_mydb_mytable_skipping_index\`. +##### Covering Index +- 'Covering' indices allow users to specify their index name. The suffix added to this type is \`_index\`. + - For instance, a 'Covering' index name could be: \`flint_mydatasource_mydb_mytable_myindexname_index\`. +##### Materialized View Index +- 'Materialized View' indices also enable users to define their index name, but they do not have a suffix. + - An example of a 'Materialized View' index name might look like: \`flint_mydatasource_mydb_mytable_myindexname\`. +##### Note: +- All user given index names must be in lowercase letters, numbers and underscore. Spaces, commas, and characters -, :, ", *, +, /, \, |, ?, #, >, or < are not allowed. + `; + +export const SKIPPING_INDEX_ACCELERATION_METHODS = [ + { value: 'PARTITION', text: 'Partition' }, + { value: 'VALUE_SET', text: 'Value Set' }, + { value: 'MIN_MAX', text: 'Min Max' }, + { value: 'BLOOM_FILTER', text: 'Bloom Filter' }, +]; + +export const ACCELERATION_AGGREGRATION_FUNCTIONS = [ + { label: 'window.start' }, + { label: 'count' }, + { label: 'sum' }, + { label: 'avg' }, + { label: 'max' }, + { label: 'min' }, +]; + +export const SPARK_PARTITION_INFO = `# Partition Information`; +export const OBS_DEFAULT_CLUSTER = 'observability-default'; // prefix key for generating data source id for default cluster in data selector +export const OBS_S3_DATA_SOURCE = 'observability-s3'; // prefix key for generating data source id for s3 data sources in data selector +export const S3_DATA_SOURCE_GROUP_DISPLAY_NAME = 'Amazon S3'; // display group name for Amazon-managed-s3 data sources in data selector +export const S3_DATA_SOURCE_GROUP_SPARK_DISPLAY_NAME = 'Spark'; // display group name for OpenSearch-spark-s3 data sources in data selector +export const SECURITY_DASHBOARDS_LOGOUT_URL = '/logout'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/hooks/direct_query_hook.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/hooks/direct_query_hook.tsx new file mode 100644 index 000000000000..a2b05f47e9ee --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/hooks/direct_query_hook.tsx @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useEffect, useState } from 'react'; +import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public'; +import { ASYNC_POLLING_INTERVAL } from '../constants'; +import { DirectQueryLoadingStatus, DirectQueryRequest } from '../types'; +import { getAsyncSessionId, setAsyncSessionId } from '../utils/query_session_utils'; +import { get as getObjValue, formatError } from '../utils/shared'; +import { usePolling } from '../utils/use_polling'; +import { SQLService } from '../requests/sql'; + +export const useDirectQuery = ( + http: HttpStart, + notifications: NotificationsStart, + dataSourceMDSId?: string +) => { + const sqlService = new SQLService(http); + const [loadStatus, setLoadStatus] = useState( + DirectQueryLoadingStatus.SCHEDULED + ); + + const { + data: pollingResult, + loading: _pollingLoading, + error: pollingError, + startPolling, + stopPolling: stopLoading, + } = usePolling((params) => { + return sqlService.fetchWithJobId(params, dataSourceMDSId || ''); + }, ASYNC_POLLING_INTERVAL); + + const startLoading = (requestPayload: DirectQueryRequest) => { + setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); + + const sessionId = getAsyncSessionId(requestPayload.datasource); + if (sessionId) { + requestPayload = { ...requestPayload, sessionId }; + } + + sqlService + .fetch(requestPayload, dataSourceMDSId) + .then((result) => { + setAsyncSessionId(requestPayload.datasource, getObjValue(result, 'sessionId', null)); + if (result.queryId) { + startPolling({ + queryId: result.queryId, + }); + } else { + // eslint-disable-next-line no-console + console.error('No query id found in response'); + setLoadStatus(DirectQueryLoadingStatus.FAILED); + } + }) + .catch((e) => { + setLoadStatus(DirectQueryLoadingStatus.FAILED); + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + e.body?.message + ); + notifications.toasts.addError(formattedError, { + title: 'Query Failed', + }); + // eslint-disable-next-line no-console + console.error(e); + }); + }; + + useEffect(() => { + // cancel direct query + if (!pollingResult) return; + const { status: anyCaseStatus, datarows, error } = pollingResult; + const status = anyCaseStatus?.toLowerCase(); + + if (status === DirectQueryLoadingStatus.SUCCESS || datarows) { + setLoadStatus(status); + stopLoading(); + } else if (status === DirectQueryLoadingStatus.FAILED) { + setLoadStatus(status); + stopLoading(); + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + error + ); + notifications.toasts.addError(formattedError, { + title: 'Query Failed', + }); + } else { + setLoadStatus(status); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [pollingResult, pollingError, stopLoading]); + + return { loadStatus, startLoading, stopLoading, pollingResult }; +}; diff --git a/src/plugins/query_enhancements/public/data_source_connection/components/index.ts b/src/plugins/data/public/ui/dataset_navigator/lib/hooks/index.tsx similarity index 61% rename from src/plugins/query_enhancements/public/data_source_connection/components/index.ts rename to src/plugins/data/public/ui/dataset_navigator/lib/hooks/index.tsx index 1ee969a1d079..88974a7c9420 100644 --- a/src/plugins/query_enhancements/public/data_source_connection/components/index.ts +++ b/src/plugins/data/public/ui/dataset_navigator/lib/hooks/index.tsx @@ -3,4 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -export { ConnectionsBar } from './connections_bar'; +export * from './direct_query_hook'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx new file mode 100644 index 000000000000..771fbd6eef3a --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx @@ -0,0 +1,9 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './catalog_cache'; +export * from './hooks'; +export * from './requests'; +export * from './utils'; diff --git a/src/plugins/query_enhancements/public/data_source_connection/services/index.ts b/src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx similarity index 58% rename from src/plugins/query_enhancements/public/data_source_connection/services/index.ts rename to src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx index 08eeda5a7aa1..3918a896bd0b 100644 --- a/src/plugins/query_enhancements/public/data_source_connection/services/index.ts +++ b/src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx @@ -3,4 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -export { ConnectionsService } from './connections_service'; +export * from './sql'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts b/src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts new file mode 100644 index 000000000000..f2c9c30c79b9 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { HttpStart } from 'opensearch-dashboards/public'; +import { DirectQueryRequest } from '../types'; + +export class SQLService { + private http: HttpStart; + + constructor(http: HttpStart) { + this.http = http; + } + + fetch = async ( + params: DirectQueryRequest, + dataSourceMDSId?: string, + errorHandler?: (error: any) => void + ) => { + const query = { + dataSourceMDSId, + }; + return this.http + .post('/api/observability/query/jobs', { + body: JSON.stringify(params), + query, + }) + .catch((error) => { + // eslint-disable-next-line no-console + console.error('fetch error: ', error.body); + if (errorHandler) errorHandler(error); + throw error; + }); + }; + + fetchWithJobId = async ( + params: { queryId: string }, + dataSourceMDSId?: string, + errorHandler?: (error: any) => void + ) => { + return this.http + .get(`/api/observability/query/jobs/${params.queryId}/${dataSourceMDSId ?? ''}`) + .catch((error) => { + // eslint-disable-next-line no-console + console.error('fetch error: ', error.body); + if (errorHandler) errorHandler(error); + throw error; + }); + }; + + deleteWithJobId = async (params: { queryId: string }, errorHandler?: (error: any) => void) => { + return this.http.delete(`/api/observability/query/jobs/${params.queryId}`).catch((error) => { + // eslint-disable-next-line no-console + console.error('delete error: ', error.body); + if (errorHandler) errorHandler(error); + throw error; + }); + }; +} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx new file mode 100644 index 000000000000..1718323ca531 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx @@ -0,0 +1,334 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { EuiComboBoxOptionOption } from '@elastic/eui'; + +export enum DirectQueryLoadingStatus { + SUCCESS = 'success', + FAILED = 'failed', + RUNNING = 'running', + SCHEDULED = 'scheduled', + CANCELED = 'canceled', + WAITING = 'waiting', + INITIAL = 'initial', +} + +export interface DirectQueryRequest { + query: string; + lang: string; + datasource: string; + sessionId?: string; +} + +export type AccelerationStatus = 'ACTIVE' | 'INACTIVE'; + +export interface PermissionsConfigurationProps { + roles: Role[]; + selectedRoles: Role[]; + setSelectedRoles: React.Dispatch>; + layout: 'horizontal' | 'vertical'; + hasSecurityAccess: boolean; +} + +export interface TableColumn { + name: string; + dataType: string; +} + +export interface Acceleration { + name: string; + status: AccelerationStatus; + type: string; + database: string; + table: string; + destination: string; + dateCreated: number; + dateUpdated: number; + index: string; + sql: string; +} + +export interface AssociatedObject { + tableName: string; + datasource: string; + id: string; + name: string; + database: string; + type: AssociatedObjectIndexType; + accelerations: CachedAcceleration[] | AssociatedObject; + columns?: CachedColumn[]; +} + +export type Role = EuiComboBoxOptionOption; + +export type DatasourceType = 'S3GLUE' | 'PROMETHEUS'; + +export interface S3GlueProperties { + 'glue.indexstore.opensearch.uri': string; + 'glue.indexstore.opensearch.region': string; +} + +export interface PrometheusProperties { + 'prometheus.uri': string; +} + +export type DatasourceStatus = 'ACTIVE' | 'DISABLED'; + +export interface DatasourceDetails { + allowedRoles: string[]; + name: string; + connector: DatasourceType; + description: string; + properties: S3GlueProperties | PrometheusProperties; + status: DatasourceStatus; +} + +interface AsyncApiDataResponse { + status: string; + schema?: Array<{ name: string; type: string }>; + datarows?: any; + total?: number; + size?: number; + error?: string; +} + +export interface AsyncApiResponse { + data: { + ok: boolean; + resp: AsyncApiDataResponse; + }; +} + +export type PollingCallback = (statusObj: AsyncApiResponse) => void; + +export type AssociatedObjectIndexType = AccelerationIndexType | 'table'; + +export type AccelerationIndexType = 'skipping' | 'covering' | 'materialized'; + +export type LoadCacheType = 'databases' | 'tables' | 'accelerations' | 'tableColumns'; + +export enum CachedDataSourceStatus { + Updated = 'Updated', + Failed = 'Failed', + Empty = 'Empty', +} + +export interface CachedColumn { + fieldName: string; + dataType: string; +} + +export interface CachedTable { + name: string; + columns?: CachedColumn[]; +} + +export interface CachedDatabase { + name: string; + tables: CachedTable[]; + lastUpdated: string; // date string in UTC format + status: CachedDataSourceStatus; +} + +export interface CachedDataSource { + name: string; + lastUpdated: string; // date string in UTC format + status: CachedDataSourceStatus; + databases: CachedDatabase[]; + dataSourceMDSId?: string; +} + +export interface DataSourceCacheData { + version: string; + dataSources: CachedDataSource[]; +} + +export interface CachedAcceleration { + flintIndexName: string; + type: AccelerationIndexType; + database: string; + table: string; + indexName: string; + autoRefresh: boolean; + status: string; +} + +export interface CachedAccelerationByDataSource { + name: string; + accelerations: CachedAcceleration[]; + lastUpdated: string; // date string in UTC format + status: CachedDataSourceStatus; + dataSourceMDSId?: string; +} + +export interface AccelerationsCacheData { + version: string; + dataSources: CachedAccelerationByDataSource[]; +} + +export interface PollingSuccessResult { + schema: Array<{ name: string; type: string }>; + datarows: Array>; +} + +export type AsyncPollingResult = PollingSuccessResult | null; + +export type AggregationFunctionType = 'count' | 'sum' | 'avg' | 'max' | 'min' | 'window.start'; + +export interface MaterializedViewColumn { + id: string; + functionName: AggregationFunctionType; + functionParam?: string; + fieldAlias?: string; +} + +export type SkippingIndexAccMethodType = 'PARTITION' | 'VALUE_SET' | 'MIN_MAX' | 'BLOOM_FILTER'; + +export interface SkippingIndexRowType { + id: string; + fieldName: string; + dataType: string; + accelerationMethod: SkippingIndexAccMethodType; +} + +export interface DataTableFieldsType { + id: string; + fieldName: string; + dataType: string; +} + +export interface RefreshIntervalType { + refreshWindow: number; + refreshInterval: string; +} + +export interface WatermarkDelayType { + delayWindow: number; + delayInterval: string; +} + +export interface GroupByTumbleType { + timeField: string; + tumbleWindow: number; + tumbleInterval: string; +} + +export interface MaterializedViewQueryType { + columnsValues: MaterializedViewColumn[]; + groupByTumbleValue: GroupByTumbleType; +} + +export interface FormErrorsType { + dataSourceError: string[]; + databaseError: string[]; + dataTableError: string[]; + skippingIndexError: string[]; + coveringIndexError: string[]; + materializedViewError: string[]; + indexNameError: string[]; + primaryShardsError: string[]; + replicaShardsError: string[]; + refreshIntervalError: string[]; + checkpointLocationError: string[]; + watermarkDelayError: string[]; +} + +export type AccelerationRefreshType = 'autoInterval' | 'manual' | 'manualIncrement'; + +export interface CreateAccelerationForm { + dataSource: string; + database: string; + dataTable: string; + dataTableFields: DataTableFieldsType[]; + accelerationIndexType: AccelerationIndexType; + skippingIndexQueryData: SkippingIndexRowType[]; + coveringIndexQueryData: string[]; + materializedViewQueryData: MaterializedViewQueryType; + accelerationIndexName: string; + primaryShardsCount: number; + replicaShardsCount: number; + refreshType: AccelerationRefreshType; + checkpointLocation: string | undefined; + watermarkDelay: WatermarkDelayType; + refreshIntervalOptions: RefreshIntervalType; + formErrors: FormErrorsType; +} + +export interface LoadCachehookOutput { + loadStatus: DirectQueryLoadingStatus; + startLoading: (params: StartLoadingParams) => void; + stopLoading: () => void; +} + +export interface StartLoadingParams { + dataSourceName: string; + dataSourceMDSId?: string; + databaseName?: string; + tableName?: string; +} + +export interface RenderAccelerationFlyoutParams { + dataSourceName: string; + dataSourceMDSId?: string; + databaseName?: string; + tableName?: string; + handleRefresh?: () => void; +} + +export interface RenderAssociatedObjectsDetailsFlyoutParams { + tableDetail: AssociatedObject; + dataSourceName: string; + handleRefresh?: () => void; + dataSourceMDSId?: string; +} + +export interface RenderAccelerationDetailsFlyoutParams { + acceleration: CachedAcceleration; + dataSourceName: string; + handleRefresh?: () => void; + dataSourceMDSId?: string; +} + +export interface DataSetOption { + id?: string; + name: string; + dataSourceRef?: string; +} + +export interface RecentDataSetOptionsCacheData { + version: string; + recentDataSets: DataSetOption[]; +} + +export interface ExternalDataSource { + name: string; + status: string; + dataSourceRef: string; +} + +export interface ExternalDataSourcesCacheData { + version: string; + externalDataSources: ExternalDataSource[]; + lastUpdated: string; + status: CachedDataSourceStatus; +} + +interface DataSourceMeta { + // ref: string; // MDS ID + // dsName?: string; // flint datasource + id: string; + name: string; + type?: string; +} + +export interface DataSet { + id: string | undefined; // index pattern ID, index name, or flintdatasource.database.table + datasource?: DataSourceMeta; + meta?: { + timestampField: string; + mapping?: any; + }; + type?: 'dataSet' | 'temporary'; +} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts new file mode 100644 index 000000000000..697852fdd772 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export enum DirectQueryLoadingStatus { + SUCCESS = 'success', + FAILED = 'failed', + RUNNING = 'running', + SCHEDULED = 'scheduled', + CANCELED = 'canceled', + WAITING = 'waiting', + INITIAL = 'initial', +} + +const catalogCacheFetchingStatus = [ + DirectQueryLoadingStatus.RUNNING, + DirectQueryLoadingStatus.WAITING, + DirectQueryLoadingStatus.SCHEDULED, +]; + +export const isCatalogCacheFetching = (...statuses: DirectQueryLoadingStatus[]) => { + return statuses.some((status: DirectQueryLoadingStatus) => + catalogCacheFetchingStatus.includes(status) + ); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts new file mode 100644 index 000000000000..7a10d7badb58 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { SimpleDataSource } from '../../../../../common'; + +export const fetchDataSources = async (client: SavedObjectsClientContract) => { + const resp = await client.find({ + type: 'data-source', + perPage: 10000, + }); + return resp.savedObjects.map((savedObject) => ({ + id: savedObject.id, + name: savedObject.attributes.title, + type: 'data-source', + })) as SimpleDataSource[]; +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts new file mode 100644 index 000000000000..a9272155e602 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { HttpStart } from 'opensearch-dashboards/public'; + +export const fetchIfExternalDataSourcesEnabled = async (http: HttpStart) => { + try { + await http.get('/api/dataconnections'); + return true; + } catch (e) { + return false; + } +}; + +export const fetchExternalDataSources = async (http: HttpStart, connectedClusters: string[]) => { + const results = await Promise.all( + connectedClusters.map(async (cluster) => { + const dataSources = await http.get(`/api/dataconnections/dataSourceMDSId=${cluster}`); + return dataSources + .filter((dataSource) => dataSource.connector === 'S3GLUE') + .map((dataSource) => ({ + name: dataSource.name, + status: dataSource.status, + dataSourceRef: cluster, + })); + }) + ); + + const flattenedResults = results.flat(); + return flattenedResults; +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts new file mode 100644 index 000000000000..85d491df1518 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { IIndexPattern } from '../.././../..'; + +export const fetchIndexPatterns = async (client: SavedObjectsClientContract, search: string) => { + const resp = await client.find({ + type: 'index-pattern', + fields: ['title'], + search: `${search}*`, + searchFields: ['title'], + perPage: 100, + }); + return resp.savedObjects.map((savedObject) => ({ + id: savedObject.id, + title: savedObject.attributes.title, + dataSourceId: savedObject.references[0]?.id, + })); +}; + +// export async function fetchIndexPatterns( +// savedObjectsClient: SavedObjectsClientContract, +// indexPatternStrings: string[], +// uiSettings: IUiSettingsClient +// ) { +// if (!indexPatternStrings || isEmpty(indexPatternStrings)) { +// return []; +// } + +// const searchString = indexPatternStrings.map((string) => `"${string}"`).join(' | '); +// const indexPatternsFromSavedObjects = await savedObjectsClient.find({ +// type: 'index-pattern', +// fields: ['title', 'fields'], +// search: searchString, +// searchFields: ['title'], +// }); + +// const exactMatches = indexPatternsFromSavedObjects.savedObjects.filter((savedObject) => { +// return indexPatternStrings.includes(savedObject.attributes.title); +// }); + +// const defaultIndex = uiSettings.get('defaultIndex'); + +// const allMatches = +// exactMatches.length === indexPatternStrings.length +// ? exactMatches +// : [ +// ...exactMatches, +// await savedObjectsClient.get('index-pattern', defaultIndex), +// ]; + +// return allMatches.map(indexPatterns.getFromSavedObject); +// } diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts new file mode 100644 index 000000000000..ef10c72bc08c --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { map } from 'rxjs/operators'; +import { ISearchStart } from '../../../../search'; + +export const fetchIndices = async (search: ISearchStart, dataSourceId?: string) => { + const buildSearchRequest = () => { + const request = { + params: { + ignoreUnavailable: true, + expand_wildcards: 'all', + index: '*', + body: { + size: 0, // no hits + aggs: { + indices: { + terms: { + field: '_index', + size: 100, + }, + }, + }, + }, + }, + dataSourceId, + }; + + return request; + }; + + const searchResponseToArray = (response: any) => { + const { rawResponse } = response; + return rawResponse.aggregations + ? rawResponse.aggregations.indices.buckets.map((bucket: { key: any }) => bucket.key) + : []; + }; + + return search + .getDefaultSearchInterceptor() + .search(buildSearchRequest()) + .pipe(map(searchResponseToArray)) + .toPromise(); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts new file mode 100644 index 000000000000..7dbe7ec2d4f4 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './fetch_catalog_cache_status'; +export * from './fetch_data_sources'; +export * from './fetch_external_data_sources'; +export * from './fetch_index_patterns'; +export * from './fetch_indices'; +export * from './query_session_utils'; +export * from './shared'; +export * from './use_polling'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts new file mode 100644 index 000000000000..beabcb48c197 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { ASYNC_QUERY_SESSION_ID } from '../constants'; + +export const setAsyncSessionId = (dataSource: string, value: string | null) => { + if (value !== null) { + sessionStorage.setItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`, value); + } +}; + +export const getAsyncSessionId = (dataSource: string) => { + return sessionStorage.getItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts new file mode 100644 index 000000000000..3e4afc94e80b --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts @@ -0,0 +1,332 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * TODO making this method type-safe is nontrivial: if you just define + * `Nested = { [k: string]: Nested | T }` then you can't accumulate because `T` is not `Nested` + * There might be a way to define a recursive type that accumulates cleanly but it's probably not + * worth the effort. + */ + +export function get(obj: Record, path: string, defaultValue?: T): T { + return path.split('.').reduce((acc: any, part: string) => acc && acc[part], obj) || defaultValue; +} + +export function addBackticksIfNeeded(input: string): string { + if (input === undefined) { + return ''; + } + // Check if the string already has backticks + if (input.startsWith('`') && input.endsWith('`')) { + return input; // Return the string as it is + } else { + // Add backticks to the string + return '`' + input + '`'; + } +} + +export function combineSchemaAndDatarows( + schema: Array<{ name: string; type: string }>, + datarows: Array> +): object[] { + const combinedData: object[] = []; + + datarows.forEach((row) => { + const rowData: { [key: string]: string | number | boolean } = {}; + schema.forEach((field, index) => { + rowData[field.name] = row[index]; + }); + combinedData.push(rowData); + }); + + return combinedData; +} + +export const formatError = (name: string, message: string, details: string) => { + return { + name, + message, + body: { + attributes: { + error: { + caused_by: { + type: '', + reason: details, + }, + }, + }, + }, + }; +}; + +// TODO: relocate to a more appropriate location +// Client route +export const PPL_BASE = '/api/ppl'; +export const PPL_SEARCH = '/search'; +export const DSL_BASE = '/api/dsl'; +export const DSL_SEARCH = '/search'; +export const DSL_CAT = '/cat.indices'; +export const DSL_MAPPING = '/indices.getFieldMapping'; +export const DSL_SETTINGS = '/indices.getFieldSettings'; +export const OBSERVABILITY_BASE = '/api/observability'; +export const INTEGRATIONS_BASE = '/api/integrations'; +export const JOBS_BASE = '/query/jobs'; +export const DATACONNECTIONS_BASE = '/api/dataconnections'; +export const EDIT = '/edit'; +export const DATACONNECTIONS_UPDATE_STATUS = '/status'; +export const SECURITY_ROLES = '/api/v1/configuration/roles'; +export const EVENT_ANALYTICS = '/event_analytics'; +export const SAVED_OBJECTS = '/saved_objects'; +export const SAVED_QUERY = '/query'; +export const SAVED_VISUALIZATION = '/vis'; +export const CONSOLE_PROXY = '/api/console/proxy'; +export const SECURITY_PLUGIN_ACCOUNT_API = '/api/v1/configuration/account'; + +// Server route +export const PPL_ENDPOINT = '/_plugins/_ppl'; +export const SQL_ENDPOINT = '/_plugins/_sql'; +export const DSL_ENDPOINT = '/_plugins/_dsl'; +export const DATACONNECTIONS_ENDPOINT = '/_plugins/_query/_datasources'; +export const JOBS_ENDPOINT_BASE = '/_plugins/_async_query'; +export const JOB_RESULT_ENDPOINT = '/result'; + +export const observabilityID = 'observability-logs'; +export const observabilityTitle = 'Observability'; +export const observabilityPluginOrder = 1500; + +export const observabilityApplicationsID = 'observability-applications'; +export const observabilityApplicationsTitle = 'Applications'; +export const observabilityApplicationsPluginOrder = 5090; + +export const observabilityLogsID = 'observability-logs'; +export const observabilityLogsTitle = 'Logs'; +export const observabilityLogsPluginOrder = 5091; + +export const observabilityMetricsID = 'observability-metrics'; +export const observabilityMetricsTitle = 'Metrics'; +export const observabilityMetricsPluginOrder = 5092; + +export const observabilityTracesID = 'observability-traces'; +export const observabilityTracesTitle = 'Traces'; +export const observabilityTracesPluginOrder = 5093; + +export const observabilityNotebookID = 'observability-notebooks'; +export const observabilityNotebookTitle = 'Notebooks'; +export const observabilityNotebookPluginOrder = 5094; + +export const observabilityPanelsID = 'observability-dashboards'; +export const observabilityPanelsTitle = 'Dashboards'; +export const observabilityPanelsPluginOrder = 5095; + +export const observabilityIntegrationsID = 'integrations'; +export const observabilityIntegrationsTitle = 'Integrations'; +export const observabilityIntegrationsPluginOrder = 9020; + +export const observabilityDataConnectionsID = 'datasources'; +export const observabilityDataConnectionsTitle = 'Data sources'; +export const observabilityDataConnectionsPluginOrder = 9030; + +export const queryWorkbenchPluginID = 'opensearch-query-workbench'; +export const queryWorkbenchPluginCheck = 'plugin:queryWorkbenchDashboards'; + +// Shared Constants +export const SQL_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest/search-plugins/sql/index/'; +export const PPL_DOCUMENTATION_URL = + 'https://opensearch.org/docs/latest/search-plugins/sql/ppl/index'; +export const PPL_PATTERNS_DOCUMENTATION_URL = + 'https://github.com/opensearch-project/sql/blob/2.x/docs/user/ppl/cmd/patterns.rst#description'; +export const UI_DATE_FORMAT = 'MM/DD/YYYY hh:mm A'; +export const PPL_DATE_FORMAT = 'YYYY-MM-DD HH:mm:ss.SSSSSS'; +export const OTEL_DATE_FORMAT = 'YYYY-MM-DDTHH:mm:ss'; +export const SPAN_REGEX = /span/; + +export const PROMQL_METRIC_SUBTYPE = 'promqlmetric'; +export const OTEL_METRIC_SUBTYPE = 'openTelemetryMetric'; +export const PPL_METRIC_SUBTYPE = 'metric'; + +export const PPL_SPAN_REGEX = /by\s*span/i; +export const PPL_STATS_REGEX = /\|\s*stats/i; +export const PPL_INDEX_INSERT_POINT_REGEX = /(search source|source|index)\s*=\s*([^|\s]+)(.*)/i; +export const PPL_INDEX_REGEX = /(search source|source|index)\s*=\s*([^|\s]+)/i; +export const PPL_WHERE_CLAUSE_REGEX = /\s*where\s+/i; +export const PPL_NEWLINE_REGEX = /[\n\r]+/g; +export const PPL_DESCRIBE_INDEX_REGEX = /(describe)\s+([^|\s]+)/i; + +// Observability plugin URI +const BASE_OBSERVABILITY_URI = '/_plugins/_observability'; +const BASE_DATACONNECTIONS_URI = '/_plugins/_query/_datasources'; +export const OPENSEARCH_PANELS_API = { + OBJECT: `${BASE_OBSERVABILITY_URI}/object`, +}; +export const OPENSEARCH_DATACONNECTIONS_API = { + DATACONNECTION: `${BASE_DATACONNECTIONS_URI}`, +}; + +// Saved Objects +export const SAVED_OBJECT = '/object'; + +// Color Constants +export const PLOTLY_COLOR = [ + '#3CA1C7', + '#54B399', + '#DB748A', + '#F2BE4B', + '#68CCC2', + '#2A7866', + '#843769', + '#374FB8', + '#BD6F26', + '#4C636F', +]; + +export const LONG_CHART_COLOR = PLOTLY_COLOR[1]; + +export const pageStyles: CSS.Properties = { + float: 'left', + width: '100%', + maxWidth: '1130px', +}; + +export enum VIS_CHART_TYPES { + Bar = 'bar', + HorizontalBar = 'horizontal_bar', + Line = 'line', + Pie = 'pie', + HeatMap = 'heatmap', + Text = 'text', + Histogram = 'histogram', +} + +export const NUMERICAL_FIELDS = ['short', 'integer', 'long', 'float', 'double']; + +export const ENABLED_VIS_TYPES = [ + VIS_CHART_TYPES.Bar, + VIS_CHART_TYPES.HorizontalBar, + VIS_CHART_TYPES.Line, + VIS_CHART_TYPES.Pie, + VIS_CHART_TYPES.HeatMap, + VIS_CHART_TYPES.Text, +]; + +// Live tail constants +export const LIVE_OPTIONS = [ + { + label: '5s', + startTime: 'now-5s', + delayTime: 5000, + }, + { + label: '10s', + startTime: 'now-10s', + delayTime: 10000, + }, + { + label: '30s', + startTime: 'now-30s', + delayTime: 30000, + }, + { + label: '1m', + startTime: 'now-1m', + delayTime: 60000, + }, + { + label: '5m', + startTime: 'now-5m', + delayTime: 60000 * 5, + }, + { + label: '15m', + startTime: 'now-15m', + delayTime: 60000 * 15, + }, + { + label: '30m', + startTime: 'now-30m', + delayTime: 60000 * 30, + }, + { + label: '1h', + startTime: 'now-1h', + delayTime: 60000 * 60, + }, + { + label: '2h', + startTime: 'now-2h', + delayTime: 60000 * 120, + }, +]; + +export const LIVE_END_TIME = 'now'; + +export interface DefaultChartStylesProps { + DefaultModeLine: string; + Interpolation: string; + LineWidth: number; + FillOpacity: number; + MarkerSize: number; + ShowLegend: string; + LegendPosition: string; + LabelAngle: number; + DefaultSortSectors: string; + DefaultModeScatter: string; +} + +export const DEFAULT_CHART_STYLES: DefaultChartStylesProps = { + DefaultModeLine: 'lines+markers', + Interpolation: 'spline', + LineWidth: 0, + FillOpacity: 100, + MarkerSize: 25, + ShowLegend: 'show', + LegendPosition: 'v', + LabelAngle: 0, + DefaultSortSectors: 'largest_to_smallest', + DefaultModeScatter: 'markers', +}; + +export const FILLOPACITY_DIV_FACTOR = 200; +export const SLIDER_MIN_VALUE = 0; +export const SLIDER_MAX_VALUE = 100; +export const SLIDER_STEP = 1; +export const THRESHOLD_LINE_WIDTH = 3; +export const THRESHOLD_LINE_OPACITY = 0.7; +export const MAX_BUCKET_LENGTH = 16; + +export enum BarOrientation { + horizontal = 'h', + vertical = 'v', +} + +export const PLOT_MARGIN = { + l: 30, + r: 5, + b: 30, + t: 50, + pad: 4, +}; + +export const WAITING_TIME_ON_USER_ACTIONS = 300; + +export const VISUALIZATION_ERROR = { + NO_DATA: 'No data found.', + INVALID_DATA: 'Invalid visualization data', + NO_SERIES: 'Add a field to start', + NO_METRIC: 'Invalid Metric MetaData', +}; + +export const S3_DATA_SOURCE_TYPE = 's3glue'; + +export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; +export const ASYNC_QUERY_DATASOURCE_CACHE = 'async-query-catalog-cache'; +export const ASYNC_QUERY_ACCELERATIONS_CACHE = 'async-query-acclerations-cache'; + +export const DIRECT_DUMMY_QUERY = 'select 1'; + +export const DEFAULT_START_TIME = 'now-15m'; +export const QUERY_ASSIST_START_TIME = 'now-40y'; +export const QUERY_ASSIST_END_TIME = 'now'; + +export const TIMESTAMP_DATETIME_TYPES = ['date', 'date_nanos']; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts new file mode 100644 index 000000000000..74fedd6cf110 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts @@ -0,0 +1,137 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useEffect, useRef, useState } from 'react'; + +type FetchFunction = (params?: P) => Promise; + +export interface PollingConfigurations { + tabId: string; +} + +export class UsePolling { + public data: T | null = null; + public error: Error | null = null; + public loading: boolean = true; + private shouldPoll: boolean = false; + private intervalRef?: NodeJS.Timeout; + + constructor( + private fetchFunction: FetchFunction, + private interval: number = 5000, + private onPollingSuccess?: (data: T, configurations: PollingConfigurations) => boolean, + private onPollingError?: (error: Error) => boolean, + private configurations?: PollingConfigurations + ) {} + + async fetchData(params?: P) { + this.loading = true; + try { + const result = await this.fetchFunction(params); + this.data = result; + this.loading = false; + + if (this.onPollingSuccess && this.onPollingSuccess(result, this.configurations!)) { + this.stopPolling(); + } + } catch (err) { + this.error = err as Error; + this.loading = false; + + if (this.onPollingError && this.onPollingError(this.error)) { + this.stopPolling(); + } + } + } + + startPolling(params?: P) { + this.shouldPoll = true; + if (!this.intervalRef) { + this.intervalRef = setInterval(() => { + if (this.shouldPoll) { + this.fetchData(params); + } + }, this.interval); + } + } + + stopPolling() { + this.shouldPoll = false; + if (this.intervalRef) { + clearInterval(this.intervalRef); + this.intervalRef = undefined; + } + } +} + +interface UsePollingReturn { + data: T | null; + loading: boolean; + error: Error | null; + startPolling: (params?: any) => void; + stopPolling: () => void; +} + +export function usePolling( + fetchFunction: FetchFunction, + interval: number = 5000, + onPollingSuccess?: (data: T, configurations: PollingConfigurations) => boolean, + onPollingError?: (error: Error) => boolean, + configurations?: PollingConfigurations +): UsePollingReturn { + const [data, setData] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(true); + const intervalRef = useRef(undefined); + const unmounted = useRef(false); + + const shouldPoll = useRef(false); + + const startPolling = (params?: P) => { + shouldPoll.current = true; + const intervalId = setInterval(() => { + if (shouldPoll.current) { + fetchData(params); + } + }, interval); + intervalRef.current = intervalId; + if (unmounted.current) { + clearInterval(intervalId); + } + }; + + const stopPolling = () => { + shouldPoll.current = false; + clearInterval(intervalRef.current); + }; + + const fetchData = async (params?: P) => { + try { + const result = await fetchFunction(params); + setData(result); + // Check the success condition and stop polling if it's met + if (onPollingSuccess && onPollingSuccess(result, configurations)) { + stopPolling(); + } + } catch (err: unknown) { + setError(err as Error); + + // Check the error condition and stop polling if it's met + if (onPollingError && onPollingError(err as Error)) { + stopPolling(); + } + } finally { + setLoading(false); + } + }; + + useEffect(() => { + return () => { + unmounted.current = true; + }; + }, []); + + return { data, loading, error, startPolling, stopPolling }; +} diff --git a/src/plugins/data/public/ui/index.ts b/src/plugins/data/public/ui/index.ts index 5483b540d5bf..f0824978966f 100644 --- a/src/plugins/data/public/ui/index.ts +++ b/src/plugins/data/public/ui/index.ts @@ -49,3 +49,4 @@ export { } from './query_editor'; export { SearchBar, SearchBarProps, StatefulSearchBarProps } from './search_bar'; export { SuggestionsComponent } from './typeahead'; +export { DataSetNavigator, DataSetOption } from './dataset_navigator'; diff --git a/src/plugins/data/public/ui/query_editor/_query_editor.scss b/src/plugins/data/public/ui/query_editor/_query_editor.scss index 8fc81308b533..ac411b38ab88 100644 --- a/src/plugins/data/public/ui/query_editor/_query_editor.scss +++ b/src/plugins/data/public/ui/query_editor/_query_editor.scss @@ -86,6 +86,12 @@ } } +.osdQueryEditor__dataSetNavigatorWrapper { + :first-child { + border-bottom: $euiBorderThin !important; + } +} + @include euiBreakpoint("xs", "s") { .osdQueryEditor--withDatePicker { > :first-child { diff --git a/src/plugins/data/public/ui/query_editor/query_editor.tsx b/src/plugins/data/public/ui/query_editor/query_editor.tsx index 44d000de1e8f..db4984b637d4 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor.tsx @@ -3,14 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { - EuiFlexGroup, - EuiFlexItem, - EuiForm, - EuiFormRow, - htmlIdGenerator, - PopoverAnchorPosition, -} from '@elastic/eui'; +import { EuiFlexGroup, EuiFlexItem, htmlIdGenerator, PopoverAnchorPosition } from '@elastic/eui'; import classNames from 'classnames'; import { isEqual } from 'lodash'; import React, { Component, createRef, RefObject } from 'react'; @@ -44,10 +37,7 @@ export interface QueryEditorProps { indexPatterns: Array; dataSource?: DataSource; query: Query; - container?: HTMLDivElement; - dataSourceContainerRef?: React.RefCallback; - containerRef?: React.RefCallback; - languageSelectorContainerRef?: React.RefCallback; + dataSetContainerRef?: React.RefCallback; settings: Settings; disableAutoFocus?: boolean; screenTitle?: string; @@ -60,7 +50,7 @@ export interface QueryEditorProps { onChange?: (query: Query, dateRange?: TimeRange) => void; onChangeQueryEditorFocus?: (isFocused: boolean) => void; onSubmit?: (query: Query, dateRange?: TimeRange) => void; - getQueryStringInitialValue?: (language: string) => string; + getQueryStringInitialValue?: (language: string, dataSetName?: string) => string; dataTestSubj?: string; size?: SuggestionsListSize; className?: string; @@ -77,8 +67,6 @@ interface Props extends QueryEditorProps { } interface State { - isDataSourcesVisible: boolean; - isDataSetsVisible: boolean; isSuggestionsVisible: boolean; index: number | null; suggestions: QuerySuggestion[]; @@ -105,8 +93,6 @@ const KEY_CODES = { // eslint-disable-next-line import/no-default-export export default class QueryEditorUI extends Component { public state: State = { - isDataSourcesVisible: false, - isDataSetsVisible: true, isSuggestionsVisible: false, index: null, suggestions: [], @@ -121,7 +107,6 @@ export default class QueryEditorUI extends Component { private persistedLog: PersistedLog | undefined; private abortController?: AbortController; private services = this.props.opensearchDashboards.services; - private componentIsUnmounting = false; private headerRef: RefObject = createRef(); private bannerRef: RefObject = createRef(); private extensionMap = this.props.settings?.getQueryEditorExtensionMap(); @@ -250,10 +235,6 @@ export default class QueryEditorUI extends Component { : undefined; this.onChange(newQuery, dateRange); this.onSubmit(newQuery, dateRange); - this.setState({ - isDataSourcesVisible: enhancement?.searchBar?.showDataSourcesSelector ?? true, - isDataSetsVisible: enhancement?.searchBar?.showDataSetsSelector ?? true, - }); }; private initPersistedLog = () => { @@ -263,20 +244,6 @@ export default class QueryEditorUI extends Component { : getQueryLog(uiSettings, storage, appName, this.props.query.language); }; - private initDataSourcesVisibility = () => { - if (this.componentIsUnmounting) return; - - return this.props.settings.getQueryEnhancements(this.props.query.language)?.searchBar - ?.showDataSourcesSelector; - }; - - private initDataSetsVisibility = () => { - if (this.componentIsUnmounting) return; - - return this.props.settings.getQueryEnhancements(this.props.query.language)?.searchBar - ?.showDataSetsSelector; - }; - public onMouseEnterSuggestion = (index: number) => { this.setState({ index }); }; @@ -291,10 +258,6 @@ export default class QueryEditorUI extends Component { this.initPersistedLog(); // this.fetchIndexPatterns().then(this.updateSuggestions); - this.setState({ - isDataSourcesVisible: this.initDataSourcesVisibility() || true, - isDataSetsVisible: this.initDataSetsVisibility() || true, - }); } public componentDidUpdate(prevProps: Props) { @@ -308,7 +271,6 @@ export default class QueryEditorUI extends Component { public componentWillUnmount() { if (this.abortController) this.abortController.abort(); - this.componentIsUnmounting = true; } handleOnFocus = () => { @@ -431,6 +393,15 @@ export default class QueryEditorUI extends Component { const useQueryEditor = this.props.query.language !== 'kuery' && this.props.query.language !== 'lucene'; + const languageSelector = ( + + ); + return (
@@ -443,17 +414,9 @@ export default class QueryEditorUI extends Component { isCollapsed={!this.state.isCollapsed} /> - {this.state.isDataSourcesVisible && ( - -
- - )} - - {this.state.isDataSetsVisible && ( - -
- - )} + +
+ {(this.state.isCollapsed || !useQueryEditor) && ( @@ -496,14 +459,7 @@ export default class QueryEditorUI extends Component { )} {!useQueryEditor && ( -
- -
+
{languageSelector}
 )} @@ -557,15 +513,7 @@ export default class QueryEditorUI extends Component { } > - - - + {languageSelector} {this.state.lineCount} {this.state.lineCount === 1 ? 'line' : 'lines'} diff --git a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx index a482d7416418..971d13cfc050 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx @@ -39,8 +39,7 @@ const QueryEditor = withOpenSearchDashboards(QueryEditorUI); // @internal export interface QueryEditorTopRowProps { query?: Query; - dataSourceContainerRef?: React.RefCallback; - containerRef?: React.RefCallback; + dataSetContainerRef?: React.RefCallback; settings?: Settings; onSubmit: (payload: { dateRange: TimeRange; query?: Query }) => void; onChange: (payload: { dateRange: TimeRange; query?: Query }) => void; @@ -208,11 +207,10 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { ) return ''; - const defaultDataSource = indexPatterns[0]; - const dataSource = - typeof defaultDataSource === 'string' ? defaultDataSource : defaultDataSource.title; + const defaultDataSet = indexPatterns[0]; + const dataSet = typeof defaultDataSet === 'string' ? defaultDataSet : defaultDataSet.title; - return input.replace('', dataSource); + return input.replace('', dataSet); } function renderQueryEditor() { @@ -225,8 +223,7 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { dataSource={props.dataSource} prepend={props.prepend} query={parsedQuery} - dataSourceContainerRef={props.dataSourceContainerRef} - containerRef={props.containerRef} + dataSetContainerRef={props.dataSetContainerRef} settings={props.settings!} screenTitle={props.screenTitle} onChange={onQueryChange} diff --git a/src/plugins/data/public/ui/search_bar/create_search_bar.tsx b/src/plugins/data/public/ui/search_bar/create_search_bar.tsx index 31f3401dc76f..d722aeda510a 100644 --- a/src/plugins/data/public/ui/search_bar/create_search_bar.tsx +++ b/src/plugins/data/public/ui/search_bar/create_search_bar.tsx @@ -48,8 +48,7 @@ interface StatefulSearchBarDeps { data: Omit; storage: IStorageWrapper; settings: Settings; - setDataSourceContainerRef: (ref: HTMLDivElement | null) => void; - setContainerRef: (ref: HTMLDivElement | null) => void; + setDataSetContainerRef: (ref: HTMLDivElement | null) => void; } export type StatefulSearchBarProps = SearchBarOwnProps & { @@ -139,8 +138,7 @@ export function createSearchBar({ storage, data, settings, - setDataSourceContainerRef, - setContainerRef, + setDataSetContainerRef, }: StatefulSearchBarDeps) { // App name should come from the core application service. // Until it's available, we'll ask the user to provide it for the pre-wired component. @@ -176,15 +174,9 @@ export function createSearchBar({ notifications: core.notifications, }); - const dataSourceContainerRef = useCallback((node) => { + const dataSetContainerRef = useCallback((node) => { if (node) { - setDataSourceContainerRef(node); - } - }, []); - - const containerRef = useCallback((node) => { - if (node) { - setContainerRef(node); + setDataSetContainerRef(node); } }, []); @@ -228,8 +220,7 @@ export function createSearchBar({ filters={filters} query={query} settings={settings} - dataSourceContainerRef={dataSourceContainerRef} - containerRef={containerRef} + dataSetContainerRef={dataSetContainerRef} onFiltersUpdated={defaultFiltersUpdated(data.query)} onRefreshChange={defaultOnRefreshChange(data.query)} savedQuery={savedQuery} diff --git a/src/plugins/data/public/ui/search_bar/search_bar.tsx b/src/plugins/data/public/ui/search_bar/search_bar.tsx index b2ff6766e81c..4dddba69ff91 100644 --- a/src/plugins/data/public/ui/search_bar/search_bar.tsx +++ b/src/plugins/data/public/ui/search_bar/search_bar.tsx @@ -81,8 +81,7 @@ export interface SearchBarOwnProps { // Query bar - should be in SearchBarInjectedDeps query?: Query; settings?: Settings; - dataSourceContainerRef?: React.RefCallback; - containerRef?: React.RefCallback; + dataSetContainerRef?: React.RefCallback; // Show when user has privileges to save showSaveQuery?: boolean; savedQuery?: SavedQuery; @@ -493,8 +492,7 @@ class SearchBarUI extends Component { queryEditor = ( ; + DataSetNavigator: React.ComponentType; SearchBar: React.ComponentType; SuggestionsComponent: React.ComponentType; + /** + * @experimental - Subject to change + */ Settings: Settings; - dataSourceContainer$: Observable; - container$: Observable; + dataSetContainer$: Observable; } diff --git a/src/plugins/data/public/ui/ui_service.ts b/src/plugins/data/public/ui/ui_service.ts index 1e0e6be8b78c..b882f3eb7258 100644 --- a/src/plugins/data/public/ui/ui_service.ts +++ b/src/plugins/data/public/ui/ui_service.ts @@ -14,6 +14,7 @@ import { createSearchBar } from './search_bar/create_search_bar'; import { createSettings } from './settings'; import { SuggestionsComponent } from './typeahead'; import { IUiSetup, IUiStart, QueryEnhancement, UiEnhancements } from './types'; +import { createDataSetNavigator } from './dataset_navigator/create_dataset_navigator'; /** @internal */ // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -29,8 +30,7 @@ export class UiService implements Plugin { enhancementsConfig: ConfigSchema['enhancements']; private queryEnhancements: Map = new Map(); private queryEditorExtensionMap: Record = {}; - private dataSourceContainer$ = new BehaviorSubject(null); - private container$ = new BehaviorSubject(null); + private dataSetContainer$ = new BehaviorSubject(null); constructor(initializerContext: PluginInitializerContext) { const { enhancements } = initializerContext.config.get(); @@ -62,12 +62,8 @@ export class UiService implements Plugin { queryEditorExtensionMap: this.queryEditorExtensionMap, }); - const setDataSourceContainerRef = (ref: HTMLDivElement | null) => { - this.dataSourceContainer$.next(ref); - }; - - const setContainerRef = (ref: HTMLDivElement | null) => { - this.container$.next(ref); + const setDataSetContainerRef = (ref: HTMLDivElement | null) => { + this.dataSetContainer$.next(ref); }; const SearchBar = createSearchBar({ @@ -75,17 +71,16 @@ export class UiService implements Plugin { data: dataServices, storage, settings: Settings, - setDataSourceContainerRef, - setContainerRef, + setDataSetContainerRef, }); return { IndexPatternSelect: createIndexPatternSelect(core.savedObjects.client), + DataSetNavigator: createDataSetNavigator(core.savedObjects.client, core.http), SearchBar, SuggestionsComponent, Settings, - dataSourceContainer$: this.dataSourceContainer$, - container$: this.container$, + dataSetContainer$: this.dataSetContainer$, }; } diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index eea1860dc950..a6bd995c1f58 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -6,15 +6,23 @@ import React, { FC, useCallback, useEffect, useRef, useState } from 'react'; import { EuiPageSideBar, EuiPortal, EuiSplitPanel } from '@elastic/eui'; import { i18n } from '@osd/i18n'; +import { batch } from 'react-redux'; import { DataSource, DataSourceGroup, DataSourceSelectable } from '../../../../data/public'; import { DataSourceOption } from '../../../../data/public/'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { DataExplorerServices } from '../../types'; -import { setIndexPattern, useTypedDispatch, useTypedSelector } from '../../utils/state_management'; +import { + setDataSet, + setIndexPattern, + useTypedDispatch, + useTypedSelector, +} from '../../utils/state_management'; import './index.scss'; export const Sidebar: FC = ({ children }) => { - const { indexPattern: indexPatternId } = useTypedSelector((state) => state.metadata); + const { indexPattern: indexPatternId, dataSet: dataSet } = useTypedSelector( + (state) => state.metadata + ); const dispatch = useTypedDispatch(); const [selectedSources, setSelectedSources] = useState([]); const [dataSourceOptionList, setDataSourceOptionList] = useState([]); @@ -30,6 +38,8 @@ export const Sidebar: FC = ({ children }) => { }, } = useOpenSearchDashboards(); + const { DataSetNavigator } = ui; + useEffect(() => { const subscriptions = ui.Settings.getEnabledQueryEnhancementsUpdated$().subscribe( (enabledQueryEnhancements) => { @@ -48,17 +58,17 @@ export const Sidebar: FC = ({ children }) => { useEffect(() => { if (!isEnhancementsEnabled) return; - const subscriptions = ui.container$.subscribe((container) => { - if (container === null) return; + const subscriptions = ui.dataSetContainer$.subscribe((dataSetContainer) => { + if (dataSetContainer === null) return; if (containerRef.current) { - setContainerRef(container); + setContainerRef(dataSetContainer); } }); return () => { subscriptions.unsubscribe(); }; - }, [ui.container$, containerRef, setContainerRef, isEnhancementsEnabled]); + }, [ui.dataSetContainer$, containerRef, setContainerRef, isEnhancementsEnabled]); useEffect(() => { let isMounted = true; @@ -130,23 +140,21 @@ export const Sidebar: FC = ({ children }) => { [toasts] ); + const handleDataSetSelection = useCallback( + (selectedDataSet: any) => { + batch(() => { + const { id, ...ds } = selectedDataSet; + dispatch(setIndexPattern(id)); + dispatch(setDataSet(ds)); + }); + }, + [dispatch] + ); + const memorizedReload = useCallback(() => { dataSources.dataSourceService.reload(); }, [dataSources.dataSourceService]); - const dataSourceSelector = ( - - ); - return ( { containerRef.current = node; }} > - {dataSourceSelector} + )} {!isEnhancementsEnabled && ( @@ -171,7 +185,16 @@ export const Sidebar: FC = ({ children }) => { color="transparent" className="deSidebar_dataSource" > - {dataSourceSelector} + )} diff --git a/src/plugins/data_explorer/public/index.ts b/src/plugins/data_explorer/public/index.ts index f8adda434ced..6b0561261c16 100644 --- a/src/plugins/data_explorer/public/index.ts +++ b/src/plugins/data_explorer/public/index.ts @@ -18,4 +18,5 @@ export { useTypedSelector, useTypedDispatch, setIndexPattern, + setDataSet, } from './utils/state_management'; diff --git a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts index e9fe84713120..fa41a29259e3 100644 --- a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts +++ b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts @@ -5,11 +5,13 @@ import { createSlice, PayloadAction } from '@reduxjs/toolkit'; import { DataExplorerServices } from '../../types'; +import { SimpleDataSet } from '../../../../data/common'; export interface MetadataState { indexPattern?: string; originatingApp?: string; view?: string; + dataSet?: Omit; } const initialState: MetadataState = {}; @@ -40,6 +42,9 @@ export const slice = createSlice({ setIndexPattern: (state, action: PayloadAction) => { state.indexPattern = action.payload; }, + setDataSet: (state, action: PayloadAction>) => { + state.dataSet = action.payload; + }, setOriginatingApp: (state, action: PayloadAction) => { state.originatingApp = action.payload; }, @@ -53,4 +58,4 @@ export const slice = createSlice({ }); export const { reducer } = slice; -export const { setIndexPattern, setOriginatingApp, setView, setState } = slice.actions; +export const { setIndexPattern, setDataSet, setOriginatingApp, setView, setState } = slice.actions; diff --git a/src/plugins/data_explorer/public/utils/state_management/store.ts b/src/plugins/data_explorer/public/utils/state_management/store.ts index daf0b3d7e369..9d320de4b54b 100644 --- a/src/plugins/data_explorer/public/utils/state_management/store.ts +++ b/src/plugins/data_explorer/public/utils/state_management/store.ts @@ -116,4 +116,4 @@ export type RenderState = Omit; // Remaining state after export type Store = ReturnType; export type AppDispatch = Store['dispatch']; -export { MetadataState, setIndexPattern, setOriginatingApp } from './metadata_slice'; +export { MetadataState, setIndexPattern, setDataSet, setOriginatingApp } from './metadata_slice'; diff --git a/src/plugins/discover/public/application/helpers/get_data_set.ts b/src/plugins/discover/public/application/helpers/get_data_set.ts index b0431ac31c1e..762274f5bf00 100644 --- a/src/plugins/discover/public/application/helpers/get_data_set.ts +++ b/src/plugins/discover/public/application/helpers/get_data_set.ts @@ -3,23 +3,26 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { IndexPattern, IndexPatternsContract } from '../../../../data/public'; +import { DataSetOption, IndexPattern, IndexPatternsContract } from '../../../../data/public'; import { SearchData } from '../view_components/utils/use_search'; function getDataSet( - indexPattern: IndexPattern | undefined, + dataSet: IndexPattern | DataSetOption | undefined, state: SearchData, indexPatternsService: IndexPatternsContract ) { - if (!indexPattern) { + if (!dataSet) { return; } - return ( - (state.title && - state.title !== indexPattern?.title && - indexPatternsService.getByTitle(state.title!, true)) || - indexPattern - ); + if (dataSet instanceof IndexPattern) { + return ( + (state.title && + state.title !== dataSet?.title && + indexPatternsService.getByTitle(state.title!, true)) || + dataSet + ); + } + return dataSet; } export { getDataSet }; diff --git a/src/plugins/discover/public/application/utils/state_management/index.ts b/src/plugins/discover/public/application/utils/state_management/index.ts index 989b2662f0d4..e6df7e4774b8 100644 --- a/src/plugins/discover/public/application/utils/state_management/index.ts +++ b/src/plugins/discover/public/application/utils/state_management/index.ts @@ -7,6 +7,7 @@ import { TypedUseSelectorHook } from 'react-redux'; import { RootState, setIndexPattern as updateIndexPattern, + setDataSet as updateDataSet, useTypedDispatch, useTypedSelector, } from '../../../../../data_explorer/public'; @@ -20,4 +21,4 @@ export interface DiscoverRootState extends RootState { export const useSelector: TypedUseSelectorHook = useTypedSelector; export const useDispatch = useTypedDispatch; -export { updateIndexPattern }; +export { updateIndexPattern, updateDataSet }; diff --git a/src/plugins/discover/public/application/view_components/utils/update_search_source.ts b/src/plugins/discover/public/application/view_components/utils/update_search_source.ts index a8480fdad18a..aa35dd37c229 100644 --- a/src/plugins/discover/public/application/view_components/utils/update_search_source.ts +++ b/src/plugins/discover/public/application/view_components/utils/update_search_source.ts @@ -30,18 +30,18 @@ export const updateSearchSource = async ({ histogramConfigs, }: Props) => { const { uiSettings, data } = services; - let dataSet = indexPattern; - const dataFrame = searchSource?.getDataFrame(); - if ( - searchSource && - dataFrame && - dataFrame.name && - dataFrame.name !== '' && - dataSet.title !== dataFrame.name - ) { - dataSet = data.indexPatterns.getByTitle(dataFrame.name, true) ?? dataSet; - searchSource.setField('index', dataSet); - } + const dataSet = indexPattern; + // const dataFrame = searchSource?.getDataFrame(); + // if ( + // searchSource && + // dataFrame && + // dataFrame.name && + // dataFrame.name !== '' && + // dataSet.title !== dataFrame.name + // ) { + // dataSet = data.indexPatterns.getByTitle(dataFrame.name, true) ?? dataSet; + // searchSource.setField('index', dataSet); + // } const sortForSearchSource = getSortForSearchSource( sort, diff --git a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts index e8a81234278e..28b239064996 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts @@ -6,6 +6,7 @@ import { useEffect, useState } from 'react'; import { i18n } from '@osd/i18n'; import { IndexPattern } from '../../../../../data/public'; +import { SimpleDataSet } from '../../../../../data/common'; import { useSelector, updateIndexPattern } from '../../utils/state_management'; import { DiscoverViewServices } from '../../../build_services'; import { getIndexPatternId } from '../../helpers/get_index_pattern_id'; @@ -26,10 +27,35 @@ import { getIndexPatternId } from '../../helpers/get_index_pattern_id'; */ export const useIndexPattern = (services: DiscoverViewServices) => { const indexPatternIdFromState = useSelector((state) => state.metadata.indexPattern); + const dataSetFromState = useSelector((state) => state.metadata.dataSet); const [indexPattern, setIndexPattern] = useState(undefined); const { data, toastNotifications, uiSettings: config, store } = services; useEffect(() => { + const checkDataSet = async (idFromState: string, dataSet?: Omit) => { + if (dataSet) { + const temporaryIndexPattern = await data.indexPatterns.create( + { + id: idFromState, + title: dataSet.title, + ...(dataSet.dataSourceRef + ? { + dataSourceRef: { + id: dataSet.dataSourceRef.id ?? dataSet.dataSourceRef.name, + name: dataSet.dataSourceRef.name, + type: dataSet.type!, + }, + } + : {}), + timeFieldName: dataSet.timeFieldName, + }, + true + ); + data.indexPatterns.saveToCache(temporaryIndexPattern.title, temporaryIndexPattern); + } + fetchIndexPatternDetails(idFromState); + }; + let isMounted = true; const fetchIndexPatternDetails = (id: string) => { @@ -65,13 +91,20 @@ export const useIndexPattern = (services: DiscoverViewServices) => { fetchIndexPatternDetails(newId); }); } else { - fetchIndexPatternDetails(indexPatternIdFromState); + checkDataSet(indexPatternIdFromState, dataSetFromState); } return () => { isMounted = false; }; - }, [indexPatternIdFromState, data.indexPatterns, toastNotifications, config, store]); + }, [ + indexPatternIdFromState, + data.indexPatterns, + toastNotifications, + config, + store, + dataSetFromState, + ]); return indexPattern; }; diff --git a/src/plugins/query_enhancements/opensearch_dashboards.json b/src/plugins/query_enhancements/opensearch_dashboards.json index b09494aab0ca..69d8fd3bd667 100644 --- a/src/plugins/query_enhancements/opensearch_dashboards.json +++ b/src/plugins/query_enhancements/opensearch_dashboards.json @@ -3,7 +3,7 @@ "version": "opensearchDashboards", "server": true, "ui": true, - "requiredPlugins": ["data", "opensearchDashboardsReact", "opensearchDashboardsUtils", "dataSourceManagement", "savedObjects", "uiActions"], + "requiredPlugins": ["data", "opensearchDashboardsReact", "opensearchDashboardsUtils", "savedObjects", "uiActions"], "optionalPlugins": ["dataSource"] } diff --git a/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx b/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx deleted file mode 100644 index 3fd592e50b31..000000000000 --- a/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React, { useEffect, useRef, useState } from 'react'; -import { EuiPortal } from '@elastic/eui'; -import { distinctUntilChanged } from 'rxjs/operators'; -import { ToastsSetup } from 'opensearch-dashboards/public'; -import { DataPublicPluginStart, QueryEditorExtensionDependencies } from '../../../../data/public'; -import { DataSourceSelector } from '../../../../data_source_management/public'; -import { ConnectionsService } from '../services'; - -interface ConnectionsProps { - dependencies: QueryEditorExtensionDependencies; - toasts: ToastsSetup; - connectionsService: ConnectionsService; -} - -export const ConnectionsBar: React.FC = ({ connectionsService, toasts }) => { - const [isDataSourceEnabled, setIsDataSourceEnabled] = useState(false); - const [uiService, setUiService] = useState(undefined); - const containerRef = useRef(null); - - useEffect(() => { - const uiServiceSubscription = connectionsService.getUiService().subscribe(setUiService); - const dataSourceEnabledSubscription = connectionsService - .getIsDataSourceEnabled$() - .subscribe(setIsDataSourceEnabled); - - return () => { - uiServiceSubscription.unsubscribe(); - dataSourceEnabledSubscription.unsubscribe(); - }; - }, [connectionsService]); - - useEffect(() => { - if (!uiService || !isDataSourceEnabled || !containerRef.current) return; - const subscriptions = uiService.dataSourceContainer$.subscribe((container) => { - if (container && containerRef.current) { - container.append(containerRef.current); - } - }); - - return () => subscriptions.unsubscribe(); - }, [uiService, isDataSourceEnabled]); - - useEffect(() => { - const selectedConnectionSubscription = connectionsService - .getSelectedConnection$() - .pipe(distinctUntilChanged()) - .subscribe((connection) => { - if (connection) { - // Assuming setSelectedConnection$ is meant to update some state or perform an action outside this component - connectionsService.setSelectedConnection$(connection); - } - }); - - return () => selectedConnectionSubscription.unsubscribe(); - }, [connectionsService]); - - const handleSelectedConnection = (id: string | undefined) => { - if (!id) { - connectionsService.setSelectedConnection$(undefined); - return; - } - connectionsService.getConnectionById(id).subscribe((connection) => { - connectionsService.setSelectedConnection$(connection); - }); - }; - - return ( - { - containerRef.current = node; - }} - > -
- - handleSelectedConnection(dataSource[0]?.id || undefined) - } - /> -
- - ); -}; diff --git a/src/plugins/query_enhancements/public/data_source_connection/index.ts b/src/plugins/query_enhancements/public/data_source_connection/index.ts deleted file mode 100644 index e334163d91d4..000000000000 --- a/src/plugins/query_enhancements/public/data_source_connection/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export { createDataSourceConnectionExtension } from './utils'; -export * from './services'; diff --git a/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx b/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx deleted file mode 100644 index e5822c4b378e..000000000000 --- a/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React from 'react'; -import { ToastsSetup } from 'opensearch-dashboards/public'; -import { QueryEditorExtensionConfig } from '../../../../data/public'; -import { ConfigSchema } from '../../../common/config'; -import { ConnectionsBar } from '../components'; -import { ConnectionsService } from '../services'; - -export const createDataSourceConnectionExtension = ( - connectionsService: ConnectionsService, - toasts: ToastsSetup, - config: ConfigSchema -): QueryEditorExtensionConfig => { - return { - id: 'data-source-connection', - order: 2000, - isEnabled$: (dependencies) => { - return connectionsService.getIsDataSourceEnabled$(); - }, - getComponent: (dependencies) => { - return ( - - ); - }, - }; -}; diff --git a/src/plugins/query_enhancements/public/plugin.tsx b/src/plugins/query_enhancements/public/plugin.tsx index d65676b70e78..b74c00ced7e0 100644 --- a/src/plugins/query_enhancements/public/plugin.tsx +++ b/src/plugins/query_enhancements/public/plugin.tsx @@ -7,10 +7,9 @@ import moment from 'moment'; import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '../../../core/public'; import { IStorageWrapper, Storage } from '../../opensearch_dashboards_utils/public'; import { ConfigSchema } from '../common/config'; -import { ConnectionsService, createDataSourceConnectionExtension } from './data_source_connection'; +import { ConnectionsService, setData, setStorage } from './services'; import { createQueryAssistExtension } from './query_assist'; -import { PPLSearchInterceptor, SQLAsyncSearchInterceptor, SQLSearchInterceptor } from './search'; -import { setData, setStorage } from './services'; +import { PPLSearchInterceptor, SQLSearchInterceptor } from './search'; import { QueryEnhancementsPluginSetup, QueryEnhancementsPluginSetupDependencies, @@ -44,38 +43,21 @@ export class QueryEnhancementsPlugin http: core.http, }); - const pplSearchInterceptor = new PPLSearchInterceptor( - { - toasts: core.notifications.toasts, - http: core.http, - uiSettings: core.uiSettings, - startServices: core.getStartServices(), - usageCollector: data.search.usageCollector, - }, - this.connectionsService - ); - - const sqlSearchInterceptor = new SQLSearchInterceptor( - { - toasts: core.notifications.toasts, - http: core.http, - uiSettings: core.uiSettings, - startServices: core.getStartServices(), - usageCollector: data.search.usageCollector, - }, - this.connectionsService - ); + const pplSearchInterceptor = new PPLSearchInterceptor({ + toasts: core.notifications.toasts, + http: core.http, + uiSettings: core.uiSettings, + startServices: core.getStartServices(), + usageCollector: data.search.usageCollector, + }); - const sqlAsyncSearchInterceptor = new SQLAsyncSearchInterceptor( - { - toasts: core.notifications.toasts, - http: core.http, - uiSettings: core.uiSettings, - startServices: core.getStartServices(), - usageCollector: data.search.usageCollector, - }, - this.connectionsService - ); + const sqlSearchInterceptor = new SQLSearchInterceptor({ + toasts: core.notifications.toasts, + http: core.http, + uiSettings: core.uiSettings, + startServices: core.getStartServices(), + usageCollector: data.search.usageCollector, + }); data.__enhance({ ui: { @@ -89,7 +71,7 @@ export class QueryEnhancementsPlugin initialTo: moment().add(2, 'days').toISOString(), }, showFilterBar: false, - showDataSetsSelector: false, + showDataSetsSelector: true, showDataSourcesSelector: true, }, fields: { @@ -110,7 +92,7 @@ export class QueryEnhancementsPlugin searchBar: { showDatePicker: false, showFilterBar: false, - showDataSetsSelector: false, + showDataSetsSelector: true, showDataSourcesSelector: true, queryStringInput: { initialValue: 'SELECT * FROM ' }, }, @@ -125,29 +107,6 @@ export class QueryEnhancementsPlugin }, }); - data.__enhance({ - ui: { - query: { - language: 'SQLAsync', - search: sqlAsyncSearchInterceptor, - searchBar: { - showDatePicker: false, - showFilterBar: false, - showDataSetsSelector: false, - showDataSourcesSelector: true, - queryStringInput: { initialValue: 'SHOW DATABASES IN ::mys3::' }, - }, - fields: { - filterable: false, - visualizable: false, - }, - showDocLinks: false, - supportedAppNames: ['discover'], - connectionService: this.connectionsService, - }, - }, - }); - data.__enhance({ ui: { queryEditorExtension: createQueryAssistExtension( @@ -158,16 +117,6 @@ export class QueryEnhancementsPlugin }, }); - data.__enhance({ - ui: { - queryEditorExtension: createDataSourceConnectionExtension( - this.connectionsService, - core.notifications.toasts, - this.config - ), - }, - }); - return {}; } diff --git a/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx b/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx index e87e74ce2998..c28c5cb8b0be 100644 --- a/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx +++ b/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx @@ -12,7 +12,7 @@ import { } from '../../../../data/public'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { QueryAssistParameters } from '../../../common/query_assist'; -import { ConnectionsService } from '../../data_source_connection'; +import { ConnectionsService } from '../../services'; import { getStorage } from '../../services'; import { useGenerateQuery } from '../hooks'; import { getPersistedLog, ProhibitedQueryError } from '../utils'; @@ -45,7 +45,7 @@ export const QueryAssistBar: React.FC = (props) => { const subscription = props.connectionsService .getSelectedConnection$() .subscribe((connection) => { - dataSourceIdRef.current = connection?.dataSource.id; + dataSourceIdRef.current = connection?.dataSource?.id; }); return () => subscription.unsubscribe(); }, [props.connectionsService]); diff --git a/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx b/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx index e088457a0717..23611e39501e 100644 --- a/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx +++ b/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx @@ -13,7 +13,7 @@ import { } from '../../../../data/public'; import { API } from '../../../common'; import { ConfigSchema } from '../../../common/config'; -import { ConnectionsService } from '../../data_source_connection'; +import { ConnectionsService } from '../../services'; import { QueryAssistBar, QueryAssistBanner } from '../components'; /** @@ -28,7 +28,7 @@ const getAvailableLanguages$ = ( connectionsService.getSelectedConnection$().pipe( distinctUntilChanged(), switchMap(async (connection) => { - const dataSourceId = connection?.dataSource.id; + const dataSourceId = connection?.dataSource?.id; const cached = availableLanguagesByDataSource.get(dataSourceId); if (cached !== undefined) return cached; const languages = await http diff --git a/src/plugins/query_enhancements/public/search/index.ts b/src/plugins/query_enhancements/public/search/index.ts index 9835c1345f02..624e7cf6e7b5 100644 --- a/src/plugins/query_enhancements/public/search/index.ts +++ b/src/plugins/query_enhancements/public/search/index.ts @@ -5,4 +5,3 @@ export { PPLSearchInterceptor } from './ppl_search_interceptor'; export { SQLSearchInterceptor } from './sql_search_interceptor'; -export { SQLAsyncSearchInterceptor } from './sql_async_search_interceptor'; diff --git a/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts b/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts index bca9961fea3b..cc218fb6990f 100644 --- a/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts @@ -5,7 +5,6 @@ import { trimEnd } from 'lodash'; import { Observable, throwError } from 'rxjs'; -import { i18n } from '@osd/i18n'; import { concatMap } from 'rxjs/operators'; import { DataFrameAggConfig, @@ -34,16 +33,12 @@ import { fetchDataFrame, } from '../../common'; import { QueryEnhancementsPluginStartDependencies } from '../types'; -import { ConnectionsService } from '../data_source_connection'; export class PPLSearchInterceptor extends SearchInterceptor { protected queryService!: DataPublicPluginStart['query']; protected aggsService!: DataPublicPluginStart['search']['aggs']; - constructor( - deps: SearchInterceptorDeps, - private readonly connectionsService: ConnectionsService - ) { + constructor(deps: SearchInterceptorDeps) { super(deps); deps.startServices.then(([coreStart, depsStart]) => { @@ -165,9 +160,6 @@ export class PPLSearchInterceptor extends SearchInterceptor { ...dataFrame.meta, queryConfig: { ...dataFrame.meta.queryConfig, - ...(this.connectionsService.getSelectedConnection() && { - dataSourceId: this.connectionsService.getSelectedConnection()?.id, - }), }, }; const aggConfig = getAggConfig( diff --git a/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts deleted file mode 100644 index 9232ef146cdb..000000000000 --- a/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { trimEnd } from 'lodash'; -import { BehaviorSubject, Observable, throwError } from 'rxjs'; -import { i18n } from '@osd/i18n'; -import { concatMap, map } from 'rxjs/operators'; -import { - DATA_FRAME_TYPES, - DataPublicPluginStart, - IOpenSearchDashboardsSearchRequest, - IOpenSearchDashboardsSearchResponse, - ISearchOptions, - SearchInterceptor, - SearchInterceptorDeps, -} from '../../../data/public'; -import { getRawDataFrame, getRawQueryString, IDataFrameResponse } from '../../../data/common'; -import { - API, - DataFramePolling, - FetchDataFrameContext, - SEARCH_STRATEGY, - fetchDataFrame, - fetchDataFramePolling, -} from '../../common'; -import { QueryEnhancementsPluginStartDependencies } from '../types'; -import { ConnectionsService } from '../data_source_connection'; - -export class SQLAsyncSearchInterceptor extends SearchInterceptor { - protected queryService!: DataPublicPluginStart['query']; - protected aggsService!: DataPublicPluginStart['search']['aggs']; - protected indexPatterns!: DataPublicPluginStart['indexPatterns']; - protected dataFrame$ = new BehaviorSubject(undefined); - - constructor( - deps: SearchInterceptorDeps, - private readonly connectionsService: ConnectionsService - ) { - super(deps); - - deps.startServices.then(([coreStart, depsStart]) => { - this.queryService = (depsStart as QueryEnhancementsPluginStartDependencies).data.query; - this.aggsService = (depsStart as QueryEnhancementsPluginStartDependencies).data.search.aggs; - }); - } - - protected runSearch( - request: IOpenSearchDashboardsSearchRequest, - signal?: AbortSignal, - strategy?: string - ): Observable { - const { id, ...searchRequest } = request; - const path = trimEnd(API.SQL_ASYNC_SEARCH); - const dfContext: FetchDataFrameContext = { - http: this.deps.http, - path, - signal, - }; - - const dataFrame = getRawDataFrame(searchRequest); - if (!dataFrame) { - return throwError(this.handleSearchError('DataFrame is not defined', request, signal!)); - } - - const queryString = - dataFrame.meta?.queryConfig?.formattedQs() ?? getRawQueryString(searchRequest) ?? ''; - - dataFrame.meta = { - ...dataFrame.meta, - queryConfig: { - ...dataFrame.meta.queryConfig, - ...(this.connectionsService.getSelectedConnection() && - this.connectionsService.getSelectedConnection()?.dataSource && { - dataSourceId: this.connectionsService.getSelectedConnection()?.dataSource.id, - }), - }, - }; - - const onPollingSuccess = (pollingResult: any) => { - if (pollingResult && pollingResult.body.meta.status === 'SUCCESS') { - return false; - } - if (pollingResult && pollingResult.body.meta.status === 'FAILED') { - const jsError = new Error(pollingResult.data.error.response); - this.deps.toasts.addError(jsError, { - title: i18n.translate('queryEnhancements.sqlQueryError', { - defaultMessage: 'Could not complete the SQL async query', - }), - toastMessage: pollingResult.data.error.response, - }); - return false; - } - - this.deps.toasts.addInfo({ - title: i18n.translate('queryEnhancements.sqlQueryPolling', { - defaultMessage: 'Polling query job results...', - }), - }); - - return true; - }; - - const onPollingError = (error: Error) => { - throw new Error(error.message); - }; - - this.deps.toasts.addInfo({ - title: i18n.translate('queryEnhancements.sqlQueryInfo', { - defaultMessage: 'Starting query job...', - }), - }); - return fetchDataFrame(dfContext, queryString, dataFrame).pipe( - concatMap((jobResponse) => { - const df = jobResponse.body; - const dataFramePolling = new DataFramePolling( - () => fetchDataFramePolling(dfContext, df), - 5000, - onPollingSuccess, - onPollingError - ); - return dataFramePolling.fetch().pipe( - map(() => { - const dfPolling = dataFramePolling.data; - dfPolling.type = DATA_FRAME_TYPES.DEFAULT; - return dfPolling; - }) - ); - }) - ); - } - - public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) { - return this.runSearch(request, options.abortSignal, SEARCH_STRATEGY.SQL_ASYNC); - } -} diff --git a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts index 5a3b8278c65a..429c7f4dc543 100644 --- a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts @@ -6,8 +6,8 @@ import { trimEnd } from 'lodash'; import { Observable, throwError } from 'rxjs'; import { i18n } from '@osd/i18n'; -import { concatMap } from 'rxjs/operators'; -import { getRawDataFrame, getRawQueryString } from '../../../data/common'; +import { concatMap, map } from 'rxjs/operators'; +import { DATA_FRAME_TYPES, getRawDataFrame, getRawQueryString } from '../../../data/common'; import { DataPublicPluginStart, IOpenSearchDashboardsSearchRequest, @@ -16,18 +16,21 @@ import { SearchInterceptor, SearchInterceptorDeps, } from '../../../data/public'; -import { API, FetchDataFrameContext, SEARCH_STRATEGY, fetchDataFrame } from '../../common'; +import { + API, + DataFramePolling, + FetchDataFrameContext, + SEARCH_STRATEGY, + fetchDataFrame, + fetchDataFramePolling, +} from '../../common'; import { QueryEnhancementsPluginStartDependencies } from '../types'; -import { ConnectionsService } from '../data_source_connection'; export class SQLSearchInterceptor extends SearchInterceptor { protected queryService!: DataPublicPluginStart['query']; protected aggsService!: DataPublicPluginStart['search']['aggs']; - constructor( - deps: SearchInterceptorDeps, - private readonly connectionsService: ConnectionsService - ) { + constructor(deps: SearchInterceptorDeps) { super(deps); deps.startServices.then(([coreStart, depsStart]) => { @@ -59,9 +62,6 @@ export class SQLSearchInterceptor extends SearchInterceptor { ...dataFrame.meta, queryConfig: { ...dataFrame.meta.queryConfig, - ...(this.connectionsService.getSelectedConnection() && { - dataSourceId: this.connectionsService.getSelectedConnection()?.id, - }), }, }; @@ -81,7 +81,90 @@ export class SQLSearchInterceptor extends SearchInterceptor { return fetchDataFrame(dfContext, queryString, dataFrame); } + protected runSearchAsync( + request: IOpenSearchDashboardsSearchRequest, + signal?: AbortSignal, + strategy?: string + ): Observable { + const { id, ...searchRequest } = request; + const path = trimEnd(API.SQL_ASYNC_SEARCH); + const dfContext: FetchDataFrameContext = { + http: this.deps.http, + path, + signal, + }; + + const dataFrame = getRawDataFrame(searchRequest); + if (!dataFrame) { + return throwError(this.handleSearchError('DataFrame is not defined', request, signal!)); + } + + const queryString = getRawQueryString(searchRequest) ?? ''; + + dataFrame.meta = { + ...dataFrame.meta, + queryConfig: { + ...dataFrame.meta.queryConfig, + }, + }; + + const onPollingSuccess = (pollingResult: any) => { + if (pollingResult && pollingResult.body.meta.status === 'SUCCESS') { + return false; + } + if (pollingResult && pollingResult.body.meta.status === 'FAILED') { + const jsError = new Error(pollingResult.data.error.response); + this.deps.toasts.addError(jsError, { + title: i18n.translate('queryEnhancements.sqlQueryError', { + defaultMessage: 'Could not complete the SQL async query', + }), + toastMessage: pollingResult.data.error.response, + }); + return false; + } + + this.deps.toasts.addInfo({ + title: i18n.translate('queryEnhancements.sqlQueryPolling', { + defaultMessage: 'Polling query job results...', + }), + }); + + return true; + }; + + const onPollingError = (error: Error) => { + throw new Error(error.message); + }; + + this.deps.toasts.addInfo({ + title: i18n.translate('queryEnhancements.sqlQueryInfo', { + defaultMessage: 'Starting query job...', + }), + }); + return fetchDataFrame(dfContext, queryString, dataFrame).pipe( + concatMap((jobResponse) => { + const df = jobResponse.body; + const dataFramePolling = new DataFramePolling( + () => fetchDataFramePolling(dfContext, df), + 5000, + onPollingSuccess, + onPollingError + ); + return dataFramePolling.fetch().pipe( + map(() => { + const dfPolling = dataFramePolling.data; + dfPolling.type = DATA_FRAME_TYPES.DEFAULT; + return dfPolling; + }) + ); + }) + ); + } + public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) { + if (options.isAsync) { + return this.runSearchAsync(request, options.abortSignal, SEARCH_STRATEGY.SQL_ASYNC); + } return this.runSearch(request, options.abortSignal, SEARCH_STRATEGY.SQL); } } diff --git a/src/plugins/query_enhancements/public/services.ts b/src/plugins/query_enhancements/public/services.ts deleted file mode 100644 index d11233be2dca..000000000000 --- a/src/plugins/query_enhancements/public/services.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { createGetterSetter } from '../../opensearch_dashboards_utils/common'; -import { IStorageWrapper } from '../../opensearch_dashboards_utils/public'; -import { DataPublicPluginStart } from '../../data/public'; - -export const [getStorage, setStorage] = createGetterSetter('storage'); -export const [getData, setData] = createGetterSetter('data'); diff --git a/src/plugins/query_enhancements/public/data_source_connection/services/connections_service.ts b/src/plugins/query_enhancements/public/services/connections_service.ts similarity index 95% rename from src/plugins/query_enhancements/public/data_source_connection/services/connections_service.ts rename to src/plugins/query_enhancements/public/services/connections_service.ts index 6afec4b51a99..97a59c2cd94a 100644 --- a/src/plugins/query_enhancements/public/data_source_connection/services/connections_service.ts +++ b/src/plugins/query_enhancements/public/services/connections_service.ts @@ -6,8 +6,8 @@ import { BehaviorSubject, Observable, from } from 'rxjs'; import { DataPublicPluginStart } from 'src/plugins/data/public'; import { CoreStart } from 'opensearch-dashboards/public'; -import { API } from '../../../common'; -import { Connection, ConnectionsServiceDeps } from '../../types'; +import { API } from '../../common'; +import { Connection, ConnectionsServiceDeps } from '../types'; export class ConnectionsService { protected http!: ConnectionsServiceDeps['http']; diff --git a/src/plugins/query_enhancements/public/services/index.ts b/src/plugins/query_enhancements/public/services/index.ts new file mode 100644 index 000000000000..bb0284408faa --- /dev/null +++ b/src/plugins/query_enhancements/public/services/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createGetterSetter } from '../../../opensearch_dashboards_utils/common'; +import { IStorageWrapper } from '../../../opensearch_dashboards_utils/public'; +import { DataPublicPluginStart } from '../../../data/public'; + +export const [getStorage, setStorage] = createGetterSetter('storage'); +export const [getData, setData] = createGetterSetter('data'); + +export { ConnectionsService } from './connections_service'; diff --git a/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts b/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts index f4fe42779dae..162cc7e8f103 100644 --- a/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts +++ b/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts @@ -5,7 +5,6 @@ import { schema } from '@osd/config-schema'; import { IRouter } from 'opensearch-dashboards/server'; -import { DataSourceAttributes } from '../../../../data_source/common/data_sources'; import { API } from '../../../common'; export function registerDataSourceConnectionsRoutes(router: IRouter) { @@ -18,7 +17,7 @@ export function registerDataSourceConnectionsRoutes(router: IRouter) { }, async (context, request, response) => { const fields = ['id', 'title', 'auth.type']; - const resp = await context.core.savedObjects.client.find({ + const resp = await context.core.savedObjects.client.find({ type: 'data-source', fields, perPage: 10000, @@ -38,7 +37,7 @@ export function registerDataSourceConnectionsRoutes(router: IRouter) { }, }, async (context, request, response) => { - const resp = await context.core.savedObjects.client.get( + const resp = await context.core.savedObjects.client.get( 'data-source', request.params.dataSourceId ); diff --git a/src/plugins/query_enhancements/server/types.ts b/src/plugins/query_enhancements/server/types.ts index 1ad76c7bbf85..b6a03b672de9 100644 --- a/src/plugins/query_enhancements/server/types.ts +++ b/src/plugins/query_enhancements/server/types.ts @@ -4,7 +4,7 @@ */ import { PluginSetup } from 'src/plugins/data/server'; -import { DataSourcePluginSetup } from '../../data_source/server'; +import { DataSourcePluginSetup } from 'src/plugins/data_source/server'; import { Logger } from '../../../core/server'; import { ConfigSchema } from '../common/config';