From 4b7432583c1fa4e47dcf0246fa08f093c78a69a4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 23 Oct 2024 01:59:08 +0000 Subject: [PATCH] Add support for s3 fields in discover (#8609) * add support for s3 fields in discover Signed-off-by: Shenoy Pratik * Changeset file for PR #8609 created/updated * resolve comments, make fields fetch async Signed-off-by: Shenoy Pratik * fix unit tests Signed-off-by: Shenoy Pratik * update services to be Partial Signed-off-by: Shenoy Pratik * fix async field fetch in cachedataset Signed-off-by: Shenoy Pratik * resolve comments Signed-off-by: Shenoy Pratik --------- Signed-off-by: Shenoy Pratik Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com> (cherry picked from commit 12d072d9fd1e27142a3d695af112da66611f9ee1) Signed-off-by: github-actions[bot] --- changelogs/fragments/8609.yml | 2 + src/plugins/data/common/datasets/types.ts | 2 + .../index_patterns/index_pattern.ts | 28 ++-- .../data/common/index_patterns/types.ts | 7 +- .../dataset_service/dataset_service.ts | 81 +++++++--- .../query_string/dataset_service/types.ts | 9 +- .../ui/dataset_selector/advanced_selector.tsx | 5 +- .../ui/dataset_selector/configurator.tsx | 12 +- .../components/sidebar/discover_sidebar.tsx | 30 ++-- .../utils/use_index_pattern.ts | 15 +- .../query_enhancements/common/constants.ts | 2 + .../query_enhancements/common/types.ts | 33 ++++ .../public/datasets/s3_type.test.ts | 152 ++++++++++++++++-- .../public/datasets/s3_type.ts | 139 ++++++++++++++-- 14 files changed, 435 insertions(+), 82 deletions(-) create mode 100644 changelogs/fragments/8609.yml diff --git a/changelogs/fragments/8609.yml b/changelogs/fragments/8609.yml new file mode 100644 index 000000000000..eebae8882550 --- /dev/null +++ b/changelogs/fragments/8609.yml @@ -0,0 +1,2 @@ +fix: +- Discover 2.0 support for S3 fields ([#8609](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8609)) \ No newline at end of file diff --git a/src/plugins/data/common/datasets/types.ts b/src/plugins/data/common/datasets/types.ts index 58a5dc65932d..e777eb8a45e8 100644 --- a/src/plugins/data/common/datasets/types.ts +++ b/src/plugins/data/common/datasets/types.ts @@ -28,6 +28,8 @@ export interface DataSourceMeta { name?: string; /** Optional session ID for faster responses when utilizing async query sources */ sessionId?: string; + /** Optional supportsTimeFilter determines if a time filter is needed */ + supportsTimeFilter?: boolean; } /** diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts b/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts index 628e8c03f377..773badbde732 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts @@ -32,25 +32,25 @@ import _, { each, reject } from 'lodash'; import { SavedObjectsClientCommon } from '../..'; import { DuplicateField } from '../../../../opensearch_dashboards_utils/common'; +import { SerializedFieldFormat } from '../../../../expressions/common'; import { - OPENSEARCH_FIELD_TYPES, - OSD_FIELD_TYPES, - IIndexPattern, FieldFormatNotFoundError, IFieldType, + IIndexPattern, + OPENSEARCH_FIELD_TYPES, + OSD_FIELD_TYPES, } from '../../../common'; -import { IndexPatternField, IIndexPatternFieldList, fieldList } from '../fields'; -import { formatHitProvider } from './format_hit'; -import { flattenHitWrapper } from './flatten_hit'; -import { FieldFormatsStartCommon, FieldFormat } from '../../field_formats'; +import { FieldFormat, FieldFormatsStartCommon } from '../../field_formats'; +import { IIndexPatternFieldList, IndexPatternField, fieldList } from '../fields'; import { - IndexPatternSpec, - TypeMeta, - SourceFilter, IndexPatternFieldMap, + IndexPatternSpec, SavedObjectReference, + SourceFilter, + TypeMeta, } from '../types'; -import { SerializedFieldFormat } from '../../../../expressions/common'; +import { flattenHitWrapper } from './flatten_hit'; +import { formatHitProvider } from './format_hit'; interface IndexPatternDeps { spec?: IndexPatternSpec; @@ -94,6 +94,7 @@ export class IndexPattern implements IIndexPattern { public version: string | undefined; public sourceFilters?: SourceFilter[]; public dataSourceRef?: SavedObjectReference; + public fieldsLoading?: boolean; private originalSavedObjectBody: SavedObjectBody = {}; private shortDotsEnable: boolean = false; private fieldFormats: FieldFormatsStartCommon; @@ -137,6 +138,7 @@ export class IndexPattern implements IIndexPattern { return this.deserializeFieldFormatMap(mapping); }); this.dataSourceRef = spec.dataSourceRef; + this.fieldsLoading = spec.fieldsLoading; } /** @@ -378,6 +380,10 @@ export class IndexPattern implements IIndexPattern { : []; }; + setFieldsLoading = (status: boolean) => { + return (this.fieldsLoading = status); + }; + /** * Provide a field, get its formatter * @param field diff --git a/src/plugins/data/common/index_patterns/types.ts b/src/plugins/data/common/index_patterns/types.ts index 108a93a3725b..674a2c0a1f26 100644 --- a/src/plugins/data/common/index_patterns/types.ts +++ b/src/plugins/data/common/index_patterns/types.ts @@ -28,12 +28,12 @@ * under the License. */ -import { ToastInputFields, ErrorToastOptions } from 'src/core/public/notifications'; +import { ErrorToastOptions, ToastInputFields } from 'src/core/public/notifications'; // eslint-disable-next-line import type { SavedObject } from 'src/core/server'; -import { IFieldType } from './fields'; +import { FieldFormat, IndexPatternField, OSD_FIELD_TYPES } from '..'; import { SerializedFieldFormat } from '../../../expressions/common'; -import { OSD_FIELD_TYPES, IndexPatternField, FieldFormat } from '..'; +import { IFieldType } from './fields'; export type FieldFormatMap = Record; @@ -201,6 +201,7 @@ export interface IndexPatternSpec { typeMeta?: TypeMeta; type?: string; dataSourceRef?: SavedObjectReference; + fieldsLoading?: boolean; } export interface SourceFilter { diff --git a/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts b/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts index d8414a33779e..5012055d1b18 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts @@ -3,21 +3,22 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { CoreStart } from 'opensearch-dashboards/public'; import LRUCache from 'lru-cache'; +import { CoreStart } from 'opensearch-dashboards/public'; import { + CachedDataStructure, Dataset, + DataStorage, DataStructure, - IndexPatternSpec, DEFAULT_DATA, + IndexPatternFieldMap, + IndexPatternSpec, UI_SETTINGS, - DataStorage, - CachedDataStructure, } from '../../../../common'; -import { DatasetTypeConfig, DataStructureFetchOptions } from './types'; -import { indexPatternTypeConfig, indexTypeConfig } from './lib'; import { IndexPatternsContract } from '../../../index_patterns'; import { IDataPluginServices } from '../../../types'; +import { indexPatternTypeConfig, indexTypeConfig } from './lib'; +import { DatasetTypeConfig, DataStructureFetchOptions } from './types'; export class DatasetService { private indexPatterns?: IndexPatternsContract; @@ -91,29 +92,57 @@ export class DatasetService { } } - public async cacheDataset(dataset: Dataset): Promise { - const type = this.getType(dataset.type); - if (dataset && dataset.type !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) { - const spec = { - id: dataset.id, - title: dataset.title, - timeFieldName: dataset.timeFieldName, - fields: await type?.fetchFields(dataset), - dataSourceRef: dataset.dataSource - ? { - id: dataset.dataSource.id!, - name: dataset.dataSource.title, - type: dataset.dataSource.type, - } - : undefined, - } as IndexPatternSpec; - const temporaryIndexPattern = await this.indexPatterns?.create(spec, true); - if (temporaryIndexPattern) { - this.indexPatterns?.saveToCache(dataset.id, temporaryIndexPattern); + public async cacheDataset( + dataset: Dataset, + services: Partial + ): Promise { + const type = this.getType(dataset?.type); + try { + const asyncType = type?.meta.isFieldLoadAsync ?? false; + if (dataset && dataset.type !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) { + const fetchedFields = asyncType + ? ({} as IndexPatternFieldMap) + : await type?.fetchFields(dataset, services); + const spec = { + id: dataset.id, + title: dataset.title, + timeFieldName: dataset.timeFieldName, + fields: fetchedFields, + fieldsLoading: asyncType, + dataSourceRef: dataset.dataSource + ? { + id: dataset.dataSource.id!, + name: dataset.dataSource.title, + type: dataset.dataSource.type, + } + : undefined, + } as IndexPatternSpec; + const temporaryIndexPattern = await this.indexPatterns?.create(spec, true); + + // Load schema asynchronously if it's an async index pattern + if (asyncType && temporaryIndexPattern) { + type! + .fetchFields(dataset, services) + .then((fields) => { + temporaryIndexPattern.fields.replaceAll([...fields]); + this.indexPatterns?.saveToCache(dataset.id, temporaryIndexPattern); + }) + .catch((error) => { + throw new Error(`Error while fetching fields for dataset ${dataset.id}:`); + }) + .finally(() => { + temporaryIndexPattern.setFieldsLoading(false); + }); + } + + if (temporaryIndexPattern) { + this.indexPatterns?.saveToCache(dataset.id, temporaryIndexPattern); + } } + } catch (error) { + throw new Error(`Failed to load dataset: ${dataset?.id}`); } } - public async fetchOptions( services: IDataPluginServices, path: DataStructure[], diff --git a/src/plugins/data/public/query/query_string/dataset_service/types.ts b/src/plugins/data/public/query/query_string/dataset_service/types.ts index 43607fe49feb..020bc369ff5e 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/types.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/types.ts @@ -32,6 +32,10 @@ export interface DatasetTypeConfig { tooltip?: string; /** Optional preference for search on page load else defaulted to true */ searchOnLoad?: boolean; + /** Optional supportsTimeFilter determines if a time filter is needed */ + supportsTimeFilter?: boolean; + /** Optional isFieldLoadAsync determines if field loads are async */ + isFieldLoadAsync?: boolean; }; /** * Converts a DataStructure to a Dataset. @@ -55,7 +59,10 @@ export interface DatasetTypeConfig { * Fetches fields for the dataset. * @returns {Promise} A promise that resolves to an array of DatasetFields. */ - fetchFields: (dataset: Dataset) => Promise; + fetchFields: ( + dataset: Dataset, + services?: Partial + ) => Promise; /** * Retrieves the supported query languages for this dataset type. * @returns {Promise} A promise that resolves to an array of supported language ids. diff --git a/src/plugins/data/public/ui/dataset_selector/advanced_selector.tsx b/src/plugins/data/public/ui/dataset_selector/advanced_selector.tsx index 734153452eea..a7aa105785a0 100644 --- a/src/plugins/data/public/ui/dataset_selector/advanced_selector.tsx +++ b/src/plugins/data/public/ui/dataset_selector/advanced_selector.tsx @@ -11,10 +11,10 @@ import { DataStructure, DEFAULT_DATA, } from '../../../common'; -import { DatasetExplorer } from './dataset_explorer'; -import { Configurator } from './configurator'; import { getQueryService } from '../../services'; import { IDataPluginServices } from '../../types'; +import { Configurator } from './configurator'; +import { DatasetExplorer } from './dataset_explorer'; export const AdvancedSelector = ({ services, @@ -52,6 +52,7 @@ export const AdvancedSelector = ({ return selectedDataset ? ( void; onCancel: () => void; @@ -80,8 +83,13 @@ export const Configurator = ({ setTimeFields(dateFields || []); }; + if (baseDataset?.dataSource?.meta?.supportsTimeFilter === false && timeFields.length > 0) { + setTimeFields([]); + return; + } + fetchFields(); - }, [baseDataset, indexPatternsService, queryString]); + }, [baseDataset, indexPatternsService, queryString, timeFields.length]); return ( <> @@ -192,7 +200,7 @@ export const Configurator = ({ { - await queryString.getDatasetService().cacheDataset(dataset); + await queryString.getDatasetService().cacheDataset(dataset, services); onConfirm(dataset); }} fill diff --git a/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx b/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx index 12ba460bb5a1..bc55a3fe4a7d 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx @@ -28,31 +28,31 @@ * under the License. */ -import './discover_sidebar.scss'; -import React, { useCallback, useEffect, useState, useMemo } from 'react'; -import { i18n } from '@osd/i18n'; import { - EuiDragDropContext, DropResult, - EuiDroppable, + EuiButtonEmpty, + EuiDragDropContext, EuiDraggable, + EuiDroppable, EuiPanel, EuiSplitPanel, - EuiButtonEmpty, } from '@elastic/eui'; +import { i18n } from '@osd/i18n'; import { I18nProvider } from '@osd/i18n/react'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import { IndexPattern, IndexPatternField, UI_SETTINGS } from '../../../../../data/public'; +import { FIELDS_LIMIT_SETTING } from '../../../../common'; +import { getServices } from '../../../opensearch_dashboards_services'; import { DiscoverField } from './discover_field'; -import { DiscoverFieldSearch } from './discover_field_search'; import { DiscoverFieldDataFrame } from './discover_field_data_frame'; -import { FIELDS_LIMIT_SETTING } from '../../../../common'; -import { groupFields } from './lib/group_fields'; -import { IndexPatternField, IndexPattern, UI_SETTINGS } from '../../../../../data/public'; -import { getDetails } from './lib/get_details'; +import { DiscoverFieldSearch } from './discover_field_search'; +import './discover_sidebar.scss'; +import { displayIndexPatternCreation } from './lib/display_index_pattern_creation'; import { getDefaultFieldFilter, setFieldFilterProp } from './lib/field_filter'; +import { getDetails } from './lib/get_details'; import { getIndexPatternFieldList } from './lib/get_index_pattern_field_list'; -import { getServices } from '../../../opensearch_dashboards_services'; +import { groupFields } from './lib/group_fields'; import { FieldDetails } from './types'; -import { displayIndexPatternCreation } from './lib/display_index_pattern_creation'; export interface DiscoverSidebarProps { /** @@ -231,11 +231,12 @@ export function DiscoverSidebar(props: DiscoverSidebarProps) { /> ) : null} + - {fields.length > 0 && ( + {(fields.length > 0 || selectedIndexPattern.fieldsLoading) && ( <> {title} diff --git a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts index 7274bf125497..109fc384a071 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts @@ -3,13 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { useCallback, useEffect, useMemo, useState } from 'react'; import { i18n } from '@osd/i18n'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import { IndexPattern, useQueryStringManager } from '../../../../../data/public'; -import { useSelector, updateIndexPattern } from '../../utils/state_management'; +import { QUERY_ENHANCEMENT_ENABLED_SETTING } from '../../../../common'; import { DiscoverViewServices } from '../../../build_services'; import { getIndexPatternId } from '../../helpers/get_index_pattern_id'; -import { QUERY_ENHANCEMENT_ENABLED_SETTING } from '../../../../common'; +import { updateIndexPattern, useSelector } from '../../utils/state_management'; /** * Custom hook to fetch and manage the index pattern based on the provided services. @@ -51,7 +51,13 @@ export const useIndexPattern = (services: DiscoverViewServices) => { query.dataset.type !== 'INDEX_PATTERN' ); if (!pattern) { - await data.query.queryString.getDatasetService().cacheDataset(query.dataset); + await data.query.queryString.getDatasetService().cacheDataset(query.dataset, { + uiSettings: services.uiSettings, + savedObjects: services.savedObjects, + notifications: services.notifications, + http: services.http, + data: services.data, + }); pattern = await data.indexPatterns.get( query.dataset.id, query.dataset.type !== 'INDEX_PATTERN' @@ -98,6 +104,7 @@ export const useIndexPattern = (services: DiscoverViewServices) => { isMounted = false; }; }, [ + services, isQueryEnhancementEnabled, indexPatternIdFromState, fetchIndexPatternDetails, diff --git a/src/plugins/query_enhancements/common/constants.ts b/src/plugins/query_enhancements/common/constants.ts index 769a03e0c0d0..f729c40211aa 100644 --- a/src/plugins/query_enhancements/common/constants.ts +++ b/src/plugins/query_enhancements/common/constants.ts @@ -54,3 +54,5 @@ export const UI_SETTINGS = { }; export const ERROR_DETAILS = { GUARDRAILS_TRIGGERED: 'guardrails triggered' }; + +export const S3_PARTITION_INFO_COLUMN = '# Partition Information'; diff --git a/src/plugins/query_enhancements/common/types.ts b/src/plugins/query_enhancements/common/types.ts index 199752e17b09..1bb977527d4a 100644 --- a/src/plugins/query_enhancements/common/types.ts +++ b/src/plugins/query_enhancements/common/types.ts @@ -36,3 +36,36 @@ export interface QueryStatusOptions { } export type FetchFunction = (params?: P) => Promise; + +export interface SQLQueryResponse { + status: string; + schema: Array<{ name: string; type: string }>; + datarows: unknown[][]; + total: number; + size: number; +} + +export enum S3_FIELD_TYPES { + BOOLEAN = 'boolean', + BYTE = 'byte', + SHORT = 'short', + INTEGER = 'integer', + INT = 'int', + LONG = 'long', + FLOAT = 'float', + DOUBLE = 'double', + KEYWORD = 'keyword', + TEXT = 'text', + STRING = 'string', + TIMESTAMP = 'timestamp', + DATE = 'date', + DATE_NANOS = 'date_nanos', + TIME = 'time', + INTERVAL = 'interval', + IP = 'ip', + GEO_POINT = 'geo_point', + BINARY = 'binary', + STRUCT = 'struct', + ARRAY = 'array', + UNKNOWN = 'unknown', // For unmapped or unsupported types +} diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts index 22e44eb1293c..bbe960b5f984 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts @@ -5,16 +5,16 @@ // s3_type.test.ts -import { s3TypeConfig } from './s3_type'; import { HttpSetup, SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { IDataPluginServices, OSD_FIELD_TYPES } from 'src/plugins/data/public'; import { DATA_STRUCTURE_META_TYPES, DataStructure, DataStructureCustomMeta, Dataset, } from '../../../data/common'; -import { DATASET } from '../../common'; -import { IDataPluginServices } from 'src/plugins/data/public'; +import { DATASET, S3_FIELD_TYPES } from '../../common'; +import { castS3FieldTypeToOSDFieldType, s3TypeConfig } from './s3_type'; describe('s3TypeConfig', () => { const mockHttp = ({ @@ -69,7 +69,12 @@ describe('s3TypeConfig', () => { id: 'ds1', title: 'DataSource 1', type: 'DATA_SOURCE', - meta: { name: 'conn1', sessionId: 'session123', type: DATA_STRUCTURE_META_TYPES.CUSTOM }, + meta: { + name: 'conn1', + sessionId: 'session123', + type: DATA_STRUCTURE_META_TYPES.CUSTOM, + supportsTimeFilter: false, + }, }, }); }); @@ -84,6 +89,7 @@ describe('s3TypeConfig', () => { meta: { sessionId: 'session123', type: DATA_STRUCTURE_META_TYPES.CUSTOM, + supportsTimeFilter: false, } as DataStructureCustomMeta, }, ]; @@ -98,7 +104,11 @@ describe('s3TypeConfig', () => { id: 'ds1', title: 'DataSource 1', type: 'DATA_SOURCE', - meta: { sessionId: 'session123', type: DATA_STRUCTURE_META_TYPES.CUSTOM }, + meta: { + sessionId: 'session123', + type: DATA_STRUCTURE_META_TYPES.CUSTOM, + supportsTimeFilter: false, + }, }, }); }); @@ -144,15 +154,139 @@ describe('s3TypeConfig', () => { }); }); - test('fetchFields returns empty array', async () => { - const mockDataset: Dataset = { id: 'table1', title: 'Table 1', type: 'S3' }; - const result = await s3TypeConfig.fetchFields(mockDataset); + test('fetchFields returns table fields', async () => { + const postResponse = { + queryId: 'd09ZbTgxRHlnWW15czM=', + sessionId: 'VHg1d0Z1NXlCS215czM=', + }; + + const defaultResponse = { + status: 'SUCCESS', + schema: [ + { name: 'col_name', type: 'string' }, + { name: 'data_type', type: 'string' }, + { name: 'comment', type: 'string' }, + ], + datarows: [ + ['@timestamp', 'timestamp', null], + ['clientip', 'string', null], + ['request', 'string', null], + ['status', 'int', null], + ['size', 'int', null], + ['# Partition Information', '', ''], + ['# col_name', 'data_type', 'comment'], + ['year', 'int', null], + ['month', 'int', null], + ['day', 'int', null], + ], + total: 10, + size: 10, + }; + + const mockDataset: Dataset = { + id: '9aa4dc80-7151-11ef-8fea-1fe2265e9c7d::mys3.default.http_logs', + title: 'mys3.default.http_logs', + type: 'S3', + dataSource: { + id: '9aa4dc80-7151-11ef-8fea-1fe2265e9c7d', + title: 'flint-213', + type: 'DATA_SOURCE', + meta: { + sessionId: 'VHg1d0Z1NXlCS215czM=', + name: 'mys3', + supportsTimeFilter: false, + }, + }, + }; + + mockHttp.fetch = jest.fn(({ method }: { method: string }) => { + switch (method) { + case 'POST': + return postResponse; + default: + return [defaultResponse]; + } + }); - expect(result).toEqual([]); + const result = await s3TypeConfig.fetchFields(mockDataset, mockServices); + + expect(result).toHaveLength(5); + expect(result[0].name).toBe('@timestamp'); + expect(result[0].type).toBe('date'); + expect(result[1].name).toBe('clientip'); + expect(result[1].type).toBe('string'); + expect(result[3].name).toBe('status'); + expect(result[3].type).toBe('number'); }); test('supportedLanguages returns SQL', () => { const mockDataset: Dataset = { id: 'table1', title: 'Table 1', type: 'S3' }; expect(s3TypeConfig.supportedLanguages(mockDataset)).toEqual(['SQL']); }); + + describe('castS3FieldTypeToOSDFieldType()', () => { + it('should map BOOLEAN to OSD_FIELD_TYPES.BOOLEAN', () => { + expect(castS3FieldTypeToOSDFieldType(S3_FIELD_TYPES.BOOLEAN)).toBe(OSD_FIELD_TYPES.BOOLEAN); + }); + + it('should map BYTE, SHORT, INTEGER, INT, LONG, FLOAT, DOUBLE to OSD_FIELD_TYPES.NUMBER', () => { + const numberTypes = [ + S3_FIELD_TYPES.BYTE, + S3_FIELD_TYPES.SHORT, + S3_FIELD_TYPES.INTEGER, + S3_FIELD_TYPES.INT, + S3_FIELD_TYPES.LONG, + S3_FIELD_TYPES.FLOAT, + S3_FIELD_TYPES.DOUBLE, + ]; + numberTypes.forEach((type) => { + expect(castS3FieldTypeToOSDFieldType(type)).toBe(OSD_FIELD_TYPES.NUMBER); + }); + }); + + it('should map KEYWORD, TEXT, STRING to OSD_FIELD_TYPES.STRING', () => { + const stringTypes = [S3_FIELD_TYPES.KEYWORD, S3_FIELD_TYPES.TEXT, S3_FIELD_TYPES.STRING]; + stringTypes.forEach((type) => { + expect(castS3FieldTypeToOSDFieldType(type)).toBe(OSD_FIELD_TYPES.STRING); + }); + }); + + it('should map TIMESTAMP, DATE, DATE_NANOS, TIME, INTERVAL to OSD_FIELD_TYPES.DATE', () => { + const dateTypes = [ + S3_FIELD_TYPES.TIMESTAMP, + S3_FIELD_TYPES.DATE, + S3_FIELD_TYPES.DATE_NANOS, + S3_FIELD_TYPES.TIME, + S3_FIELD_TYPES.INTERVAL, + ]; + dateTypes.forEach((type) => { + expect(castS3FieldTypeToOSDFieldType(type)).toBe(OSD_FIELD_TYPES.DATE); + }); + }); + + it('should map IP to OSD_FIELD_TYPES.IP', () => { + expect(castS3FieldTypeToOSDFieldType(S3_FIELD_TYPES.IP)).toBe(OSD_FIELD_TYPES.IP); + }); + + it('should map GEO_POINT to OSD_FIELD_TYPES.GEO_POINT', () => { + expect(castS3FieldTypeToOSDFieldType(S3_FIELD_TYPES.GEO_POINT)).toBe( + OSD_FIELD_TYPES.GEO_POINT + ); + }); + + it('should map BINARY to OSD_FIELD_TYPES.ATTACHMENT', () => { + expect(castS3FieldTypeToOSDFieldType(S3_FIELD_TYPES.BINARY)).toBe(OSD_FIELD_TYPES.ATTACHMENT); + }); + + it('should map STRUCT and ARRAY to OSD_FIELD_TYPES.OBJECT', () => { + const objectTypes = [S3_FIELD_TYPES.STRUCT, S3_FIELD_TYPES.ARRAY]; + objectTypes.forEach((type) => { + expect(castS3FieldTypeToOSDFieldType(type)).toBe(OSD_FIELD_TYPES.OBJECT); + }); + }); + + it('should return OSD_FIELD_TYPES.UNKNOWN for unmapped types', () => { + expect(castS3FieldTypeToOSDFieldType(S3_FIELD_TYPES.UNKNOWN)).toBe(OSD_FIELD_TYPES.UNKNOWN); + }); + }); }); diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.ts b/src/plugins/query_enhancements/public/datasets/s3_type.ts index 2a26a7e5fcea..6aff9ce44a62 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.ts @@ -3,9 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { HttpSetup, SavedObjectsClientContract } from 'opensearch-dashboards/public'; -import { trimEnd } from 'lodash'; import { i18n } from '@osd/i18n'; +import { trimEnd } from 'lodash'; +import { HttpSetup, SavedObjectsClientContract } from 'opensearch-dashboards/public'; import { DATA_STRUCTURE_META_TYPES, DEFAULT_DATA, @@ -15,8 +15,15 @@ import { Dataset, DatasetField, } from '../../../data/common'; -import { DatasetTypeConfig, IDataPluginServices } from '../../../data/public'; -import { API, DATASET, handleQueryStatus } from '../../common'; +import { DatasetTypeConfig, IDataPluginServices, OSD_FIELD_TYPES } from '../../../data/public'; +import { + API, + DATASET, + S3_FIELD_TYPES, + S3_PARTITION_INFO_COLUMN, + SQLQueryResponse, + handleQueryStatus, +} from '../../common'; import S3_ICON from '../assets/s3_mark.svg'; export const s3TypeConfig: DatasetTypeConfig = { @@ -26,6 +33,8 @@ export const s3TypeConfig: DatasetTypeConfig = { icon: { type: S3_ICON }, tooltip: 'Amazon S3 Connections', searchOnLoad: true, + supportsTimeFilter: false, + isFieldLoadAsync: true, }, toDataset: (path: DataStructure[]): Dataset => { @@ -43,7 +52,10 @@ export const s3TypeConfig: DatasetTypeConfig = { id: dataSource.id, title: dataSource.title, type: dataSource.type, - meta: table.meta as DataSourceMeta, + meta: { + ...table.meta, + supportsTimeFilter: s3TypeConfig.meta.supportsTimeFilter, + } as DataSourceMeta, } : DEFAULT_DATA.STRUCTURES.LOCAL_DATASOURCE, }; @@ -96,8 +108,13 @@ export const s3TypeConfig: DatasetTypeConfig = { } }, - fetchFields: async (dataset: Dataset): Promise => { - return []; + fetchFields: async ( + dataset: Dataset, + services?: Partial + ): Promise => { + const http = services?.http; + if (!http) return []; + return await fetchFields(http, dataset); }, supportedLanguages: (dataset: Dataset): string[] => { @@ -143,7 +160,7 @@ const fetch = async ( fetchStatus: () => http.fetch({ method: 'GET', - path: trimEnd(`${API.DATA_SOURCE.ASYNC_JOBS}`), + path: trimEnd(API.DATA_SOURCE.ASYNC_JOBS), query: { id: dataSource?.id, queryId: meta.queryId, @@ -228,7 +245,7 @@ const fetchDatabases = async (http: HttpSetup, path: DataStructure[]): Promise => { + const abortController = new AbortController(); + try { + const connection = (dataset.dataSource?.meta as DataStructureCustomMeta).name; + const sessionId = (dataset.dataSource?.meta as DataStructureCustomMeta).sessionId; + const response = await http.fetch({ + method: 'POST', + path: trimEnd(API.DATA_SOURCE.ASYNC_JOBS), + body: JSON.stringify({ + lang: 'sql', + query: `DESCRIBE TABLE ${dataset.title}`, + datasource: connection, + ...(sessionId && { sessionId }), + }), + query: { + id: dataset.dataSource?.id, + }, + signal: abortController.signal, + }); + const fetchResponse = await handleQueryStatus({ + fetchStatus: () => + http.fetch({ + method: 'GET', + path: trimEnd(API.DATA_SOURCE.ASYNC_JOBS), + query: { + id: dataset.dataSource?.id, + queryId: response.queryId, + }, + }), + }); + return mapResponseToFields(fetchResponse); + } catch (error) { + throw new Error(`Failed to load table fields from ${dataset.title}: ${error}`); + } +};