Add support for s3 fields in discover

changelogs/fragments/8609.yml

@@ -0,0 +1,2 @@
+fix:
+- Discover 2.0 support for S3 fields ([#8609](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8609))

src/plugins/data/common/datasets/types.ts

@@ -28,6 +28,8 @@ export interface DataSourceMeta {
   name?: string;
   /** Optional session ID for faster responses when utilizing async query sources */
   sessionId?: string;
+  /** Optional supportsTimeFilter determines if a time filter is needed */
+  supportsTimeFilter?: boolean;
 }
 
 /**

src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts

@@ -32,25 +32,25 @@
 import { SavedObjectsClientCommon } from '../..';
 import { DuplicateField } from '../../../../opensearch_dashboards_utils/common';
 
+import { SerializedFieldFormat } from '../../../../expressions/common';
 import {
-  OPENSEARCH_FIELD_TYPES,
-  OSD_FIELD_TYPES,
-  IIndexPattern,
   FieldFormatNotFoundError,
   IFieldType,
+  IIndexPattern,
+  OPENSEARCH_FIELD_TYPES,
+  OSD_FIELD_TYPES,
 } from '../../../common';
-import { IndexPatternField, IIndexPatternFieldList, fieldList } from '../fields';
-import { formatHitProvider } from './format_hit';
-import { flattenHitWrapper } from './flatten_hit';
-import { FieldFormatsStartCommon, FieldFormat } from '../../field_formats';
+import { FieldFormat, FieldFormatsStartCommon } from '../../field_formats';
+import { IIndexPatternFieldList, IndexPatternField, fieldList } from '../fields';
 import {
-  IndexPatternSpec,
-  TypeMeta,
-  SourceFilter,
   IndexPatternFieldMap,
+  IndexPatternSpec,
   SavedObjectReference,
+  SourceFilter,
+  TypeMeta,
 } from '../types';
-import { SerializedFieldFormat } from '../../../../expressions/common';
+import { flattenHitWrapper } from './flatten_hit';
+import { formatHitProvider } from './format_hit';
 
 interface IndexPatternDeps {
   spec?: IndexPatternSpec;
@@ -94,6 +94,7 @@
   public version: string | undefined;
   public sourceFilters?: SourceFilter[];
   public dataSourceRef?: SavedObjectReference;
+  public fieldsLoading?: boolean;
   private originalSavedObjectBody: SavedObjectBody = {};
   private shortDotsEnable: boolean = false;
   private fieldFormats: FieldFormatsStartCommon;
@@ -137,6 +138,7 @@
       return this.deserializeFieldFormatMap(mapping);
     });
     this.dataSourceRef = spec.dataSourceRef;
+    this.fieldsLoading = spec.fieldsLoading;
   }
 
   /**
@@ -378,6 +380,10 @@
       : [];
   };
 
+  setFieldsLoading = (status: boolean) => {
+    return (this.fieldsLoading = status);
+  };
+
   /**
    * Provide a field, get its formatter
    * @param field

src/plugins/data/common/index_patterns/types.ts

@@ -28,12 +28,12 @@
  * under the License.
  */
 
-import { ToastInputFields, ErrorToastOptions } from 'src/core/public/notifications';
+import { ErrorToastOptions, ToastInputFields } from 'src/core/public/notifications';
 // eslint-disable-next-line
 import type { SavedObject } from 'src/core/server';
-import { IFieldType } from './fields';
+import { FieldFormat, IndexPatternField, OSD_FIELD_TYPES } from '..';
 import { SerializedFieldFormat } from '../../../expressions/common';
-import { OSD_FIELD_TYPES, IndexPatternField, FieldFormat } from '..';
+import { IFieldType } from './fields';
 
 export type FieldFormatMap = Record<string, SerializedFieldFormat>;
 
@@ -201,6 +201,7 @@ export interface IndexPatternSpec {
   typeMeta?: TypeMeta;
   type?: string;
   dataSourceRef?: SavedObjectReference;
+  fieldsLoading?: boolean;
 }
 
 export interface SourceFilter {

src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts

@@ -3,21 +3,22 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { CoreStart } from 'opensearch-dashboards/public';
 import LRUCache from 'lru-cache';
+import { CoreStart } from 'opensearch-dashboards/public';
 import {
+  CachedDataStructure,
   Dataset,
+  DataStorage,
   DataStructure,
-  IndexPatternSpec,
   DEFAULT_DATA,
+  IndexPatternFieldMap,
+  IndexPatternSpec,
   UI_SETTINGS,
-  DataStorage,
-  CachedDataStructure,
 } from '../../../../common';
-import { DatasetTypeConfig, DataStructureFetchOptions } from './types';
-import { indexPatternTypeConfig, indexTypeConfig } from './lib';
 import { IndexPatternsContract } from '../../../index_patterns';
 import { IDataPluginServices } from '../../../types';
+import { indexPatternTypeConfig, indexTypeConfig } from './lib';
+import { DatasetTypeConfig, DataStructureFetchOptions } from './types';
 
 export class DatasetService {
   private indexPatterns?: IndexPatternsContract;
@@ -91,29 +92,54 @@
     }
   }
 
-  public async cacheDataset(dataset: Dataset): Promise<void> {
-    const type = this.getType(dataset.type);
-    if (dataset && dataset.type !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) {
-      const spec = {
-        id: dataset.id,
-        title: dataset.title,
-        timeFieldName: dataset.timeFieldName,
-        fields: await type?.fetchFields(dataset),
-        dataSourceRef: dataset.dataSource
-          ? {
-              id: dataset.dataSource.id!,
-              name: dataset.dataSource.title,
-              type: dataset.dataSource.type,
-            }
-          : undefined,
-      } as IndexPatternSpec;
-      const temporaryIndexPattern = await this.indexPatterns?.create(spec, true);
-      if (temporaryIndexPattern) {
-        this.indexPatterns?.saveToCache(dataset.id, temporaryIndexPattern);
+  public async cacheDataset(
+    dataset: Dataset,
+    services: Partial<IDataPluginServices>
+  ): Promise<void> {
+    const type = this.getType(dataset?.type);
+    try {
+      const asyncType = type && type.meta.isFieldLoadAsync;
+      if (dataset && dataset.type !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) {
+        const fetchedFields = asyncType
+          ? await type?.fetchFields(dataset, services)
+          : ({} as IndexPatternFieldMap);
+        const spec = {
+          id: dataset.id,
+          title: dataset.title,
+          timeFieldName: dataset.timeFieldName,
+          fields: fetchedFields,
+          fieldsLoading: asyncType ? true : false,
+          dataSourceRef: dataset.dataSource
+            ? {
+                id: dataset.dataSource.id!,
+                name: dataset.dataSource.title,
+                type: dataset.dataSource.type,
+              }
+            : undefined,
+        } as IndexPatternSpec;
+        const temporaryIndexPattern = await this.indexPatterns?.create(spec, true);
+
+        // Load schema asynchronously if it's an async index pattern
+        if (asyncType && temporaryIndexPattern) {
+          type
+            .fetchFields(dataset, services)
+            .then((fields) => {
+              temporaryIndexPattern?.fields.replaceAll([...fields]);
+              this.indexPatterns?.saveToCache(dataset.id, temporaryIndexPattern);
+            })
+            .finally(() => {
+              temporaryIndexPattern.setFieldsLoading(false);
+            });
+        }
+
+        if (temporaryIndexPattern) {
+          this.indexPatterns?.saveToCache(dataset.id, temporaryIndexPattern);
+        }
       }
+    } catch (error) {
+      throw new Error(`Failed to load dataset: ${dataset?.id}`);
     }
   }
-
   public async fetchOptions(
     services: IDataPluginServices,
     path: DataStructure[],

src/plugins/data/public/query/query_string/dataset_service/types.ts

@@ -32,6 +32,10 @@ export interface DatasetTypeConfig {
     tooltip?: string;
     /** Optional preference for search on page load else defaulted to true */
     searchOnLoad?: boolean;
+    /** Optional supportsTimeFilter determines if a time filter is needed */
+    supportsTimeFilter?: boolean;
+    /** Optional isFieldLoadAsync determines if field loads are async */
+    isFieldLoadAsync?: boolean;
   };
   /**
    * Converts a DataStructure to a Dataset.
@@ -55,7 +59,10 @@ export interface DatasetTypeConfig {
    * Fetches fields for the dataset.
    * @returns {Promise<DatasetField[]>} A promise that resolves to an array of DatasetFields.
    */
-  fetchFields: (dataset: Dataset) => Promise<DatasetField[]>;
+  fetchFields: (
+    dataset: Dataset,
+    services?: Partial<IDataPluginServices>
+  ) => Promise<DatasetField[]>;
   /**
    * Retrieves the supported query languages for this dataset type.
    * @returns {Promise<string[]>} A promise that resolves to an array of supported language ids.

src/plugins/data/public/ui/dataset_selector/advanced_selector.tsx

@@ -11,10 +11,10 @@ import {
   DataStructure,
   DEFAULT_DATA,
 } from '../../../common';
-import { DatasetExplorer } from './dataset_explorer';
-import { Configurator } from './configurator';
 import { getQueryService } from '../../services';
 import { IDataPluginServices } from '../../types';
+import { Configurator } from './configurator';
+import { DatasetExplorer } from './dataset_explorer';
 
 export const AdvancedSelector = ({
   services,
@@ -52,6 +52,7 @@ export const AdvancedSelector = ({
 
   return selectedDataset ? (
     <Configurator
+      services={services}
       baseDataset={selectedDataset}
       onConfirm={onSelect}
       onCancel={onCancel}

src/plugins/data/public/ui/dataset_selector/configurator.tsx

@@ -21,13 +21,16 @@
 import React, { useEffect, useMemo, useState } from 'react';
 import { BaseDataset, DEFAULT_DATA, Dataset, DatasetField } from '../../../common';
 import { getIndexPatterns, getQueryService } from '../../services';
+import { IDataPluginServices } from '../../types';
 
 export const Configurator = ({
+  services,
   baseDataset,
   onConfirm,
   onCancel,
   onPrevious,
 }: {
+  services: IDataPluginServices;
   baseDataset: BaseDataset;
   onConfirm: (dataset: Dataset) => void;
   onCancel: () => void;
@@ -80,6 +83,11 @@
       setTimeFields(dateFields || []);
     };
 
+    if (baseDataset?.dataSource?.meta?.supportsTimeFilter === false) {
+      setTimeFields([]);
+      return;
+    }
+
     fetchFields();
   }, [baseDataset, indexPatternsService, queryString]);
 
@@ -192,7 +200,7 @@
         </EuiButton>
         <EuiButton
           onClick={async () => {
-            await queryString.getDatasetService().cacheDataset(dataset);
+            await queryString.getDatasetService().cacheDataset(dataset, services);
             onConfirm(dataset);
           }}
           fill

src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx

@@ -28,31 +28,31 @@
  * under the License.
  */
 
-import './discover_sidebar.scss';
-import React, { useCallback, useEffect, useState, useMemo } from 'react';
-import { i18n } from '@osd/i18n';
 import {
-  EuiDragDropContext,
   DropResult,
-  EuiDroppable,
+  EuiButtonEmpty,
+  EuiDragDropContext,
   EuiDraggable,
+  EuiDroppable,
   EuiPanel,
   EuiSplitPanel,
-  EuiButtonEmpty,
 } from '@elastic/eui';
+import { i18n } from '@osd/i18n';
 import { I18nProvider } from '@osd/i18n/react';
+import React, { useCallback, useEffect, useMemo, useState } from 'react';
+import { IndexPattern, IndexPatternField, UI_SETTINGS } from '../../../../../data/public';
+import { FIELDS_LIMIT_SETTING } from '../../../../common';
+import { getServices } from '../../../opensearch_dashboards_services';
 import { DiscoverField } from './discover_field';
-import { DiscoverFieldSearch } from './discover_field_search';
 import { DiscoverFieldDataFrame } from './discover_field_data_frame';
-import { FIELDS_LIMIT_SETTING } from '../../../../common';
-import { groupFields } from './lib/group_fields';
-import { IndexPatternField, IndexPattern, UI_SETTINGS } from '../../../../../data/public';
-import { getDetails } from './lib/get_details';
+import { DiscoverFieldSearch } from './discover_field_search';
+import './discover_sidebar.scss';
+import { displayIndexPatternCreation } from './lib/display_index_pattern_creation';
 import { getDefaultFieldFilter, setFieldFilterProp } from './lib/field_filter';
+import { getDetails } from './lib/get_details';
 import { getIndexPatternFieldList } from './lib/get_index_pattern_field_list';
-import { getServices } from '../../../opensearch_dashboards_services';
+import { groupFields } from './lib/group_fields';
 import { FieldDetails } from './types';
-import { displayIndexPatternCreation } from './lib/display_index_pattern_creation';
 
 export interface DiscoverSidebarProps {
   /**
@@ -231,11 +231,12 @@ export function DiscoverSidebar(props: DiscoverSidebarProps) {
               />
             </EuiSplitPanel.Inner>
           ) : null}
+
           <EuiSplitPanel.Inner
             className="eui-yScroll dscSideBar_fieldListContainer"
             paddingSize="none"
           >
-            {fields.length > 0 && (
+            {(fields.length > 0 || selectedIndexPattern.fieldsLoading) && (
               <>
                 <FieldList
                   category="selected"
@@ -312,6 +313,7 @@ const FieldList = ({
         size="xs"
         className="dscSideBar_fieldGroup"
         aria-label={title}
+        isLoading={selectedIndexPattern.fieldsLoading ?? false}
       >
         {title}
       </EuiButtonEmpty>

src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts

@@ -3,13 +3,13 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { useCallback, useEffect, useMemo, useState } from 'react';
 import { i18n } from '@osd/i18n';
+import { useCallback, useEffect, useMemo, useState } from 'react';
 import { IndexPattern, useQueryStringManager } from '../../../../../data/public';
-import { useSelector, updateIndexPattern } from '../../utils/state_management';
+import { QUERY_ENHANCEMENT_ENABLED_SETTING } from '../../../../common';
 import { DiscoverViewServices } from '../../../build_services';
 import { getIndexPatternId } from '../../helpers/get_index_pattern_id';
-import { QUERY_ENHANCEMENT_ENABLED_SETTING } from '../../../../common';
+import { updateIndexPattern, useSelector } from '../../utils/state_management';
 
 /**
  * Custom hook to fetch and manage the index pattern based on the provided services.
@@ -51,7 +51,13 @@
           query.dataset.type !== 'INDEX_PATTERN'
         );
         if (!pattern) {
-          await data.query.queryString.getDatasetService().cacheDataset(query.dataset);
+          await data.query.queryString.getDatasetService().cacheDataset(query.dataset, {
+            uiSettings: services.uiSettings,
+            savedObjects: services.savedObjects,
+            notifications: services.notifications,
+            http: services.http,
+            data: services.data,
+          });
           pattern = await data.indexPatterns.get(
             query.dataset.id,
             query.dataset.type !== 'INDEX_PATTERN'
@@ -98,6 +104,7 @@
       isMounted = false;
     };
   }, [
+    services,
     isQueryEnhancementEnabled,
     indexPatternIdFromState,
     fetchIndexPatternDetails,

src/plugins/query_enhancements/common/constants.ts

@@ -54,3 +54,5 @@ export const UI_SETTINGS = {
 };
 
 export const ERROR_DETAILS = { GUARDRAILS_TRIGGERED: 'guardrails triggered' };
+
+export const S3_PARTITION_INFO_COLUMN = '# Partition Information';