Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MNT-24270 Disable Faceting when we have unknown nodes in the search results #2504

Original file line number Diff line number Diff line change
Expand Up @@ -674,6 +674,11 @@ function processResults(nodes, maxPageResults, startIndex, rootNode, meta)
}
}

if (failed !== 0 && logger.isWarnLoggingEnabled() === true)
{
logger.warn("Faceting disabled as hits are innacurate due to unknown nodeRefs returned");
}

if (logger.isLoggingEnabled())
logger.log("Filtered resultset to length: " + results.length + ". Discarded item count: " + failed);

Expand All @@ -684,9 +689,9 @@ function processResults(nodes, maxPageResults, startIndex, rootNode, meta)
totalRecords: results.length,
totalRecordsUpper: nodes.length - failed,
startIndex: startIndex,
numberFound: meta ? meta.numberFound : -1
numberFound: meta ? meta.numberFound - failed : -1
},
facets: meta ? meta.facets : null,
facets: (meta && failed === 0) ? meta.facets : null,
highlighting: meta ? meta.highlighting : null,
items: results,
spellcheck: meta ? meta.spellcheck : null
Expand Down Expand Up @@ -741,6 +746,11 @@ function processResultsSinglePage(nodes, startIndex, rootNode, meta)
}
}

if (failed !== 0 && logger.isWarnLoggingEnabled() === true)
{
logger.warn("Faceting disabled as hits are innacurate due to unknown nodeRefs returned");
}

if (logger.isLoggingEnabled())
logger.log("Filtered resultset to length: " + results.length + ". Discarded item count: " + failed);

Expand All @@ -751,9 +761,9 @@ function processResultsSinglePage(nodes, startIndex, rootNode, meta)
totalRecords: results.length,
totalRecordsUpper: -1,
startIndex: startIndex,
numberFound: meta ? meta.numberFound : -1
numberFound: meta ? meta.numberFound - failed : -1
},
facets: meta ? meta.facets : null,
facets: (meta && failed === 0) ? meta.facets : null,
highlighting: meta ? meta.highlighting : null,
items: results,
spellcheck: meta ? meta.spellcheck : null
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,12 +191,20 @@ public CollectionWithPagingInfo<Node> toCollectionWithPagingInfo(Params params,
}
}

// If we have unknown noderefs in the resultset, the faceting hits will be wrong. Disable facets instead of displaying incorrect hits
boolean disableFaceting = unknownNodeRefsCount.get() > 0;

if(disableFaceting)
{
logger.warn("Disabled faceting, stats and pivots as the hits are innacurate due to unknown nodeRefs returned");
}

SearchContext context =
toSearchEngineResultSet(results)
.map(resultSet -> toSearchContext(resultSet, searchRequestContext, searchQuery))
.map(resultSet -> toSearchContext(resultSet, searchRequestContext, searchQuery, disableFaceting))
.orElse(null);

return CollectionWithPagingInfo.asPaged(params.getPaging(), noderesults, results.hasMore(), setTotal(results), null, context);
return CollectionWithPagingInfo.asPaged(params.getPaging(), noderesults, results.hasMore(), getTotalAvailableResults(results, unknownNodeRefsCount.get()), null, context);
}

/**
Expand Down Expand Up @@ -288,84 +296,127 @@ public Integer setTotal(ResultSet results)
return total;
}

/**
* Sets the total number found of available nodes
* @param results
* @return An integer total
*/
public Integer getTotalAvailableResults(ResultSet results, int unavailableResultsCount)
{
Long totalItems = results.getNumberFound();
Integer total = totalItems.intValue() - unavailableResultsCount;
return total;
}

/**
* Uses the results from Solr to set the Search Context
*
* @param searchQuery
* @return SearchContext
*/
public SearchContext toSearchContext(SearchEngineResultSet resultSet, SearchRequestContext searchRequestContext, SearchQuery searchQuery)
{
return toSearchContext(resultSet, searchRequestContext, searchQuery, false);
}

public SearchContext toSearchContext(SearchEngineResultSet resultSet, SearchRequestContext searchRequestContext, SearchQuery searchQuery, boolean disableFaceting)
Fixed Show fixed Hide fixed
Fixed Show fixed Hide fixed
Fixed Show fixed Hide fixed
{
SearchContext context = null;
Map<String, Integer> facetQueries = resultSet.getFacetQueries();

List<GenericFacetResponse> facets = new ArrayList<>();
List<FacetQueryContext> facetResults = null;
SpellCheckContext spellCheckContext = null;
List<FacetFieldContext> ffcs = new ArrayList<FacetFieldContext>();

if (searchQuery == null)
{
throw new IllegalArgumentException("searchQuery can't be null");
}

//Facet queries
if(facetQueries!= null && !facetQueries.isEmpty())
SpellCheckContext spellCheckContext = getSpellCheckContext(resultSet);

if (!disableFaceting)
{
//If group by field populated in query facet return bucketing into facet field.
List<GenericFacetResponse> facetQueryForFields = getFacetBucketsFromFacetQueries(facetQueries,searchQuery);
if(hasGroup(searchQuery) || FacetFormat.V2 == searchQuery.getFacetFormat())

Map<String, Integer> facetQueries = resultSet.getFacetQueries();
Map<String, List<Pair<String, Integer>>> facetFields = resultSet.getFieldFacets();
Map<String, List<Pair<String, Integer>>> facetInterval = resultSet.getFacetIntervals();
Map<String, List<Map<String, String>>> facetRanges = resultSet.getFacetRanges();

if (useBuckets(searchQuery))
{
facets.addAll(facetQueryForFields);
facets.addAll(getFacetQueriesForFields(facetQueries, searchQuery));
facets.addAll(getFacetBucketsForFacetFieldsAsFacets(facetFields, searchQuery));
}
else
{
// Return the old way facet query with no bucketing.
facetResults = new ArrayList<>(facetQueries.size());
for (Entry<String, Integer> fq:facetQueries.entrySet())
{
String filterQuery = null;
if (searchQuery.getFacetQueries() != null)
{
Optional<FacetQuery> found = searchQuery.getFacetQueries().stream().filter(facetQuery -> fq.getKey().equals(facetQuery.getLabel())).findFirst();
filterQuery = found.isPresent()? found.get().getQuery():fq.getKey();
}
facetResults.add(new FacetQueryContext(fq.getKey(), filterQuery, fq.getValue()));
}
facetResults = getFacetResults(facetQueries, searchQuery);
ffcs.addAll(getFacetBucketsForFacetFields(facetFields, searchQuery));
}

facets.addAll(getGenericFacetsForIntervals(facetInterval, searchQuery));
facets.addAll(RangeResultMapper.getGenericFacetsForRanges(facetRanges, searchQuery.getFacetRanges()));

List<GenericFacetResponse> stats = getFieldStats(searchRequestContext, resultSet.getStats());
List<GenericFacetResponse> pimped = getPivots(searchRequestContext, resultSet.getPivotFacets(), stats);
facets.addAll(pimped);
facets.addAll(stats);
}

//Field Facets
Map<String, List<Pair<String, Integer>>> facetFields = resultSet.getFieldFacets();
if(FacetFormat.V2 == searchQuery.getFacetFormat())
// Put it all together
context = new SearchContext(resultSet.getLastIndexedTxId(), facets, facetResults, ffcs, spellCheckContext,
searchRequestContext.includeRequest() ? searchQuery : null);

return isNullContext(context) ? null : context;
}

private boolean useBuckets(SearchQuery searchQuery)
{
return hasGroup(searchQuery) || FacetFormat.V2 == searchQuery.getFacetFormat();
}

private SpellCheckContext getSpellCheckContext(SearchEngineResultSet resultSet)
{
SpellCheckContext spellCheckContext = null;
SpellCheckResult spell = resultSet.getSpellCheckResult();
if (spell != null && spell.getResultName() != null && !spell.getResults().isEmpty())
{
facets.addAll(getFacetBucketsForFacetFieldsAsFacets(facetFields, searchQuery));
spellCheckContext = new SpellCheckContext(spell.getResultName(), spell.getResults());
}
else

return spellCheckContext;
}

private List<GenericFacetResponse> getFacetQueriesForFields(Map<String, Integer> facetQueries, SearchQuery searchQuery)
{
List<GenericFacetResponse> facetQueryForFields = new ArrayList<>();
if (facetQueries != null && !facetQueries.isEmpty())
{
ffcs.addAll(getFacetBucketsForFacetFields(facetFields, searchQuery));
facetQueryForFields = getFacetBucketsFromFacetQueries(facetQueries, searchQuery);
}

Map<String, List<Pair<String, Integer>>> facetInterval = resultSet.getFacetIntervals();
facets.addAll(getGenericFacetsForIntervals(facetInterval, searchQuery));

Map<String,List<Map<String,String>>> facetRanges = resultSet.getFacetRanges();
facets.addAll(RangeResultMapper.getGenericFacetsForRanges(facetRanges, searchQuery.getFacetRanges()));

List<GenericFacetResponse> stats = getFieldStats(searchRequestContext, resultSet.getStats());
List<GenericFacetResponse> pimped = getPivots(searchRequestContext, resultSet.getPivotFacets(), stats);
facets.addAll(pimped);
facets.addAll(stats);
return facetQueryForFields;
}

//Spelling
SpellCheckResult spell = resultSet.getSpellCheckResult();
if (spell != null && spell.getResultName() != null && !spell.getResults().isEmpty())
private List<FacetQueryContext> getFacetResults(Map<String, Integer> facetQueries, SearchQuery searchQuery)
{
List<FacetQueryContext> facetResults = null;
if (facetQueries != null && !facetQueries.isEmpty())
{
spellCheckContext = new SpellCheckContext(spell.getResultName(),spell.getResults());
facetResults = new ArrayList<>(facetQueries.size());
for (Entry<String, Integer> fq : facetQueries.entrySet())
{
String filterQuery = null;
if (searchQuery.getFacetQueries() != null)
{
Optional<FacetQuery> found = searchQuery.getFacetQueries().stream()
.filter(facetQuery -> fq.getKey().equals(facetQuery.getLabel())).findFirst();
filterQuery = found.isPresent() ? found.get().getQuery() : fq.getKey();
}
facetResults.add(new FacetQueryContext(fq.getKey(), filterQuery, fq.getValue()));
}
}

//Put it all together
context = new SearchContext(resultSet.getLastIndexedTxId(), facets, facetResults, ffcs, spellCheckContext, searchRequestContext.includeRequest()?searchQuery:null);
return isNullContext(context)?null:context;
return facetResults;
}

public static boolean hasGroup(SearchQuery searchQuery)
Expand Down Expand Up @@ -399,8 +450,6 @@ protected List<GenericFacetResponse> getFacetBucketsFromFacetQueries(Map<String,
group = found.get().getGroup();
}
}
// if(group != null && !group.isEmpty() || FacetFormat.V2 == searchQuery.getFacetFormat())
// {
if(groups.containsKey(group))
{
Set<Metric> metrics = new HashSet<>(1);
Expand All @@ -416,7 +465,7 @@ protected List<GenericFacetResponse> getFacetBucketsFromFacetQueries(Map<String,
groups.put(group, l);
}
}
// }

if(!groups.isEmpty())
{
groups.forEach((a,v) -> facetResults.add(new GenericFacetResponse(FACET_TYPE.query, a, v)));
Expand Down Expand Up @@ -553,8 +602,10 @@ protected List<FacetFieldContext> getFacetBucketsForFacetFields(Map<String, List
}
return Collections.emptyList();
}

/**
* Returns generic faceting responses for Intervals
*
* @param facetFields
* @param searchQuery
* @return GenericFacetResponse
Expand All @@ -564,40 +615,14 @@ protected static List<GenericFacetResponse> getGenericFacetsForIntervals(Map<Str
if (facetFields != null && !facetFields.isEmpty())
{
List<GenericFacetResponse> ffcs = new ArrayList<>(facetFields.size());
for (Entry<String, List<Pair<String, Integer>>> facet:facetFields.entrySet())
for (Entry<String, List<Pair<String, Integer>>> facet : facetFields.entrySet())
{
if (facet.getValue() != null && !facet.getValue().isEmpty())
{
List<GenericBucket> buckets = new ArrayList<>(facet.getValue().size());
for (Pair<String, Integer> buck:facet.getValue())
for (Pair<String, Integer> buck : facet.getValue())
{
String filterQuery = null;
Map<String, String> bucketInfo = new HashMap<>();

if (searchQuery != null
&& searchQuery.getFacetIntervals() != null
&& searchQuery.getFacetIntervals().getIntervals() != null
&& !searchQuery.getFacetIntervals().getIntervals().isEmpty())
{
Optional<Interval> found = searchQuery.getFacetIntervals().getIntervals().stream().filter(
interval -> facet.getKey().equals(interval.getLabel()!=null?interval.getLabel():interval.getField())).findFirst();
if (found.isPresent())
{
if (found.get().getSets() != null)
{
Optional<IntervalSet> foundSet = found.get().getSets().stream().filter(aSet -> buck.getFirst().equals(aSet.getLabel())).findFirst();
if (foundSet.isPresent())
{
filterQuery = found.get().getField() + ":" + foundSet.get().toAFTSQuery();
bucketInfo.put(GenericFacetResponse.START, foundSet.get().getStart());
bucketInfo.put(GenericFacetResponse.END, foundSet.get().getEnd());
bucketInfo.put(GenericFacetResponse.START_INC, String.valueOf(foundSet.get().isStartInclusive()));
bucketInfo.put(GenericFacetResponse.END_INC, String.valueOf(foundSet.get().isEndInclusive()));
}
}
}
}
GenericBucket bucket = new GenericBucket(buck.getFirst(), filterQuery, null , new HashSet<Metric>(Arrays.asList(new SimpleMetric(METRIC_TYPE.count,String.valueOf(buck.getSecond())))), null, bucketInfo);
GenericBucket bucket = createBucket(facet.getKey(), buck, searchQuery);
buckets.add(bucket);
}
ffcs.add(new GenericFacetResponse(FACET_TYPE.interval, facet.getKey(), buckets));
Expand All @@ -609,6 +634,62 @@ protected static List<GenericFacetResponse> getGenericFacetsForIntervals(Map<Str
return Collections.emptyList();
}

private static GenericBucket createBucket(String facetKey, Pair<String, Integer> buck, SearchQuery searchQuery)
{
String filterQuery = null;
Map<String, String> bucketInfo = new HashMap<>();

Pair<String, IntervalSet> facetIntervalSet = getFacetIntervalSet(facetKey, buck, searchQuery);

if (facetIntervalSet != null)
{
String field = facetIntervalSet.getFirst();
IntervalSet intervalSet = facetIntervalSet.getSecond();

filterQuery = field + ":" + intervalSet.toAFTSQuery();
bucketInfo.put(GenericFacetResponse.START, intervalSet.getStart());
bucketInfo.put(GenericFacetResponse.END, intervalSet.getEnd());
bucketInfo.put(GenericFacetResponse.START_INC, String.valueOf(intervalSet.isStartInclusive()));
bucketInfo.put(GenericFacetResponse.END_INC, String.valueOf(intervalSet.isEndInclusive()));
}

return new GenericBucket(buck.getFirst(), filterQuery, null,
new HashSet<>(Arrays.asList(new SimpleMetric(METRIC_TYPE.count, String.valueOf(buck.getSecond())))), null,
bucketInfo);
}

private static Pair<String, IntervalSet> getFacetIntervalSet(String facetKey, Pair<String, Integer> buck, SearchQuery searchQuery)
{
if (!searchQueryHasFacetIntervals(searchQuery))
{
return null;
}

Optional<Interval> found = searchQuery.getFacetIntervals().getIntervals().stream()
.filter(interval -> facetKey.equals(interval.getLabel() != null ? interval.getLabel() : interval.getField()))
.findFirst();

if (found.isPresent() && found.get().getSets() != null)
{
Optional<IntervalSet> foundSet = found.get().getSets().stream()
.filter(aSet -> buck.getFirst().equals(aSet.getLabel())).findFirst();

if (foundSet.isPresent())
{
return new Pair<>(found.get().getField(), foundSet.get());
}
}

return null;
}

private static boolean searchQueryHasFacetIntervals(SearchQuery searchQuery)
{
return searchQuery != null && searchQuery.getFacetIntervals() != null
&& searchQuery.getFacetIntervals().getIntervals() != null
&& !searchQuery.getFacetIntervals().getIntervals().isEmpty();
}

/**
* Is the context null?
* @param context
Expand Down
Loading
Loading