-
Notifications
You must be signed in to change notification settings - Fork 884
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add support for s3 fields in discover #8609
Changes from 5 commits
93128a9
5179f29
6d6a9c6
000834f
a096e61
8f87349
53198c4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
fix: | ||
- Discover 2.0 support for S3 fields ([#8609](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8609)) |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,21 +3,22 @@ | |
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
import { CoreStart } from 'opensearch-dashboards/public'; | ||
import LRUCache from 'lru-cache'; | ||
import { CoreStart } from 'opensearch-dashboards/public'; | ||
import { | ||
CachedDataStructure, | ||
Dataset, | ||
DataStorage, | ||
DataStructure, | ||
IndexPatternSpec, | ||
DEFAULT_DATA, | ||
IndexPatternFieldMap, | ||
IndexPatternSpec, | ||
UI_SETTINGS, | ||
DataStorage, | ||
CachedDataStructure, | ||
} from '../../../../common'; | ||
import { DatasetTypeConfig, DataStructureFetchOptions } from './types'; | ||
import { indexPatternTypeConfig, indexTypeConfig } from './lib'; | ||
import { IndexPatternsContract } from '../../../index_patterns'; | ||
import { IDataPluginServices } from '../../../types'; | ||
import { indexPatternTypeConfig, indexTypeConfig } from './lib'; | ||
import { DatasetTypeConfig, DataStructureFetchOptions } from './types'; | ||
|
||
export class DatasetService { | ||
private indexPatterns?: IndexPatternsContract; | ||
|
@@ -91,29 +92,54 @@ | |
} | ||
} | ||
|
||
public async cacheDataset(dataset: Dataset): Promise<void> { | ||
const type = this.getType(dataset.type); | ||
if (dataset && dataset.type !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) { | ||
const spec = { | ||
id: dataset.id, | ||
title: dataset.title, | ||
timeFieldName: dataset.timeFieldName, | ||
fields: await type?.fetchFields(dataset), | ||
dataSourceRef: dataset.dataSource | ||
? { | ||
id: dataset.dataSource.id!, | ||
name: dataset.dataSource.title, | ||
type: dataset.dataSource.type, | ||
} | ||
: undefined, | ||
} as IndexPatternSpec; | ||
const temporaryIndexPattern = await this.indexPatterns?.create(spec, true); | ||
if (temporaryIndexPattern) { | ||
this.indexPatterns?.saveToCache(dataset.id, temporaryIndexPattern); | ||
public async cacheDataset( | ||
dataset: Dataset, | ||
services: Partial<IDataPluginServices> | ||
): Promise<void> { | ||
const type = this.getType(dataset?.type); | ||
try { | ||
const asyncType = type && type.meta.isFieldLoadAsync; | ||
if (dataset && dataset.type !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) { | ||
const fetchedFields = asyncType | ||
? await type?.fetchFields(dataset, services) | ||
: ({} as IndexPatternFieldMap); | ||
const spec = { | ||
id: dataset.id, | ||
title: dataset.title, | ||
timeFieldName: dataset.timeFieldName, | ||
fields: fetchedFields, | ||
fieldsLoading: asyncType ? true : false, | ||
dataSourceRef: dataset.dataSource | ||
? { | ||
id: dataset.dataSource.id!, | ||
name: dataset.dataSource.title, | ||
type: dataset.dataSource.type, | ||
} | ||
: undefined, | ||
} as IndexPatternSpec; | ||
const temporaryIndexPattern = await this.indexPatterns?.create(spec, true); | ||
|
||
// Load schema asynchronously if it's an async index pattern | ||
if (asyncType && temporaryIndexPattern) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. does async index patterns mean the metadata like field schema is not stored in the index pattern saved object? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Async index patterns means that the fields are lazy loaded; i.e. the temporary index pattern is created with empty fields and when the fields are loaded in the index pattern gets hydrated with the fields. |
||
type | ||
.fetchFields(dataset, services) | ||
.then((fields) => { | ||
temporaryIndexPattern?.fields.replaceAll([...fields]); | ||
this.indexPatterns?.saveToCache(dataset.id, temporaryIndexPattern); | ||
Check warning on line 128 in src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts Codecov / codecov/patchsrc/plugins/data/public/query/query_string/dataset_service/dataset_service.ts#L127-L128
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Handling of this request should ideally be tested. Can we follow up on this? |
||
}) | ||
.finally(() => { | ||
temporaryIndexPattern.setFieldsLoading(false); | ||
}); | ||
} | ||
|
||
if (temporaryIndexPattern) { | ||
this.indexPatterns?.saveToCache(dataset.id, temporaryIndexPattern); | ||
} | ||
} | ||
} catch (error) { | ||
throw new Error(`Failed to load dataset: ${dataset?.id}`); | ||
} | ||
} | ||
|
||
public async fetchOptions( | ||
services: IDataPluginServices, | ||
path: DataStructure[], | ||
|
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -21,13 +21,16 @@ | |||||
import React, { useEffect, useMemo, useState } from 'react'; | ||||||
import { BaseDataset, DEFAULT_DATA, Dataset, DatasetField } from '../../../common'; | ||||||
import { getIndexPatterns, getQueryService } from '../../services'; | ||||||
import { IDataPluginServices } from '../../types'; | ||||||
|
||||||
export const Configurator = ({ | ||||||
services, | ||||||
baseDataset, | ||||||
onConfirm, | ||||||
onCancel, | ||||||
onPrevious, | ||||||
}: { | ||||||
services: IDataPluginServices; | ||||||
baseDataset: BaseDataset; | ||||||
onConfirm: (dataset: Dataset) => void; | ||||||
onCancel: () => void; | ||||||
|
@@ -80,6 +83,11 @@ | |||||
setTimeFields(dateFields || []); | ||||||
}; | ||||||
|
||||||
if (baseDataset?.dataSource?.meta?.supportsTimeFilter === false) { | ||||||
ps48 marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
setTimeFields([]); | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this
Suggested change
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Updated here: 53198c4 |
||||||
return; | ||||||
} | ||||||
|
||||||
fetchFields(); | ||||||
}, [baseDataset, indexPatternsService, queryString]); | ||||||
|
||||||
|
@@ -192,7 +200,7 @@ | |||||
</EuiButton> | ||||||
<EuiButton | ||||||
onClick={async () => { | ||||||
await queryString.getDatasetService().cacheDataset(dataset); | ||||||
await queryString.getDatasetService().cacheDataset(dataset, services); | ||||||
onConfirm(dataset); | ||||||
}} | ||||||
fill | ||||||
|
ps48 marked this conversation as resolved.
Show resolved
Hide resolved
|
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -28,31 +28,31 @@ | |||||
* under the License. | ||||||
*/ | ||||||
|
||||||
import './discover_sidebar.scss'; | ||||||
import React, { useCallback, useEffect, useState, useMemo } from 'react'; | ||||||
import { i18n } from '@osd/i18n'; | ||||||
import { | ||||||
EuiDragDropContext, | ||||||
DropResult, | ||||||
EuiDroppable, | ||||||
EuiButtonEmpty, | ||||||
EuiDragDropContext, | ||||||
EuiDraggable, | ||||||
EuiDroppable, | ||||||
EuiPanel, | ||||||
EuiSplitPanel, | ||||||
EuiButtonEmpty, | ||||||
} from '@elastic/eui'; | ||||||
import { i18n } from '@osd/i18n'; | ||||||
import { I18nProvider } from '@osd/i18n/react'; | ||||||
import React, { useCallback, useEffect, useMemo, useState } from 'react'; | ||||||
import { IndexPattern, IndexPatternField, UI_SETTINGS } from '../../../../../data/public'; | ||||||
import { FIELDS_LIMIT_SETTING } from '../../../../common'; | ||||||
import { getServices } from '../../../opensearch_dashboards_services'; | ||||||
import { DiscoverField } from './discover_field'; | ||||||
import { DiscoverFieldSearch } from './discover_field_search'; | ||||||
import { DiscoverFieldDataFrame } from './discover_field_data_frame'; | ||||||
import { FIELDS_LIMIT_SETTING } from '../../../../common'; | ||||||
import { groupFields } from './lib/group_fields'; | ||||||
import { IndexPatternField, IndexPattern, UI_SETTINGS } from '../../../../../data/public'; | ||||||
import { getDetails } from './lib/get_details'; | ||||||
import { DiscoverFieldSearch } from './discover_field_search'; | ||||||
import './discover_sidebar.scss'; | ||||||
import { displayIndexPatternCreation } from './lib/display_index_pattern_creation'; | ||||||
import { getDefaultFieldFilter, setFieldFilterProp } from './lib/field_filter'; | ||||||
import { getDetails } from './lib/get_details'; | ||||||
import { getIndexPatternFieldList } from './lib/get_index_pattern_field_list'; | ||||||
import { getServices } from '../../../opensearch_dashboards_services'; | ||||||
import { groupFields } from './lib/group_fields'; | ||||||
import { FieldDetails } from './types'; | ||||||
import { displayIndexPatternCreation } from './lib/display_index_pattern_creation'; | ||||||
|
||||||
export interface DiscoverSidebarProps { | ||||||
/** | ||||||
|
@@ -231,11 +231,12 @@ export function DiscoverSidebar(props: DiscoverSidebarProps) { | |||||
/> | ||||||
</EuiSplitPanel.Inner> | ||||||
) : null} | ||||||
|
||||||
<EuiSplitPanel.Inner | ||||||
className="eui-yScroll dscSideBar_fieldListContainer" | ||||||
paddingSize="none" | ||||||
> | ||||||
{fields.length > 0 && ( | ||||||
{(fields.length > 0 || selectedIndexPattern.fieldsLoading) && ( | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if loading and There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If loading and fields list is empty -> The UI shows the sidebar in the loading state. |
||||||
<> | ||||||
<FieldList | ||||||
category="selected" | ||||||
|
@@ -312,6 +313,7 @@ const FieldList = ({ | |||||
size="xs" | ||||||
className="dscSideBar_fieldGroup" | ||||||
aria-label={title} | ||||||
isLoading={selectedIndexPattern.fieldsLoading ?? false} | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
or coerce to boolean There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Coerce makes sense: 53198c4 |
||||||
> | ||||||
{title} | ||||||
</EuiButtonEmpty> | ||||||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
can this be controlled based on whether
timeField
is present in the dataset rather than additional flag?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Today even if
timeField
is present we don't have support for S3 datasources to automatic time filter injection in the query. This is surely where we want to go towards in future though.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Some older conversations around this: #8337 (comment), #8337 (comment)