-
Notifications
You must be signed in to change notification settings - Fork 246
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Enrich 'i' and 'ri' rows in metadata table with event date Modify the generation of 'i' (indexed rows) and 'ri' (reverse indexed rows) in the metadata table such that the column qualifier contains the event date. This is required as a first step to support efforts for issue #825 so that we can identify dates when an event was ingested and included in a frequency count for an associated 'f' row, but was not indexed. * Add counts to 'i' and 'ri' rows * Initial federated query planner implementation * code formatting * Fixed issues with FederatedQueryIterable * Fix test failures * Fix failing tests * Additional test fixes * pr feedback * Use new MetadataHelper function version * Extract fields to filter index holes * Correct logic for determining sub date ranges * Remove unnecessary check * code formatting * Add check for null query model * Limit config arg to function scope * Update metadata-utils submodule commit * code formatting * Fix failing tests * Additional test fixes * Ensure all original tests pass * Add federated planner tests and chained schedulers * pr feedback * metadata-utils 3.0.3 tag * Fixed the index hole data ingest to set appropriate time stamps on the keys Removed some of the code which I believe was trying to diagnose the test issues * Updated applyModel to use the passed in script * Remove unneeded changes * Make FederatedQueryPlanner the default * Restore original log4j.properties * code formatting * Fix QueryPlanTest * Updated to test with teardown * Test debugging edits * Updated formatting * Concatenate sub-plans * Make FederatedQueryPlanner implement Cloneable * code formatting * * Updated with metadata-utils 4.0.5 (index markers and avoid non-indexed fields for holes) * Fixed test cases with correct responses and periodic failing test cases * Updated AncestorQueryLogic to handle federate query planner * * Allow subclasses of ShardQueryConfiguration * Updated to throw a NoResultsException for am empty query. * import reorg * Updated to avoid expanding unfielded if disabled, and to assume no index holes if no query fields. * Add tests for default query planner with ne and not-eq * Revert changes to test data format * Revert changes to log4j.properties * Ensure query plan updated after any exception type * Revert all changes to test data format --------- Co-authored-by: Ivan Bella <[email protected]> Co-authored-by: hgklohr <[email protected]>
- Loading branch information
3 people
authored
Sep 9, 2024
1 parent
6ff65b9
commit 04336c4
Showing
27 changed files
with
3,105 additions
and
82 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
100 changes: 100 additions & 0 deletions
100
warehouse/query-core/src/main/java/datawave/query/planner/FederatedQueryIterable.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| package datawave.query.planner; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.ArrayList; | ||
| import java.util.Iterator; | ||
| import java.util.List; | ||
|
|
||
| import datawave.core.query.configuration.QueryData; | ||
| import datawave.query.CloseableIterable; | ||
|
|
||
| /** | ||
| * Implementation of {@link CloseableIterable} intended to be used by {@link FederatedQueryPlanner}. This iterable | ||
| */ | ||
| public class FederatedQueryIterable implements CloseableIterable<QueryData> { | ||
|
|
||
| private final List<CloseableIterable<QueryData>> iterables = new ArrayList<>(); | ||
|
|
||
| /** | ||
| * Add an iterable to this {@link FederatedQueryIterable}. | ||
| * | ||
| * @param iterable | ||
| * the iterable to add | ||
| */ | ||
| public void addIterable(CloseableIterable<QueryData> iterable) { | ||
| if (iterable != null) { | ||
| iterables.add(iterable); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Closes and clears each iterable in this {@link FederatedQueryIterable}. | ||
| * | ||
| * @throws IOException | ||
| * if an error occurred when closing an iterable | ||
| */ | ||
| @Override | ||
| public void close() throws IOException { | ||
| for (CloseableIterable<QueryData> iterable : iterables) { | ||
| iterable.close(); | ||
| } | ||
| iterables.clear(); | ||
| } | ||
|
|
||
| /** | ||
| * Returns an iterator that will iterate over the {@link QueryData} returned by each iterable in this {@link FederatedQueryIterable}. | ||
| * | ||
| * @return the iterator | ||
| */ | ||
| @Override | ||
| public Iterator<QueryData> iterator() { | ||
| return new Iter(); | ||
| } | ||
|
|
||
| /** | ||
| * Iterator implementation that provides the ability to iterate over each {@link QueryData} of the iterables in {@link #iterables}. | ||
| */ | ||
| private class Iter implements Iterator<QueryData> { | ||
|
|
||
| // Iterator that traverses over the iterables. | ||
| private final Iterator<CloseableIterable<QueryData>> iterableIterator = iterables.iterator(); | ||
|
|
||
| // The current sub iterator. | ||
| private Iterator<QueryData> currentSubIterator = null; | ||
|
|
||
| @Override | ||
| public boolean hasNext() { | ||
| seekToNextAvailableQueryData(); | ||
| return currentSubIterator != null && currentSubIterator.hasNext(); | ||
| } | ||
|
|
||
| @Override | ||
| public QueryData next() { | ||
| return currentSubIterator.next(); | ||
| } | ||
|
|
||
| /** | ||
| * Seek to the next sub-iterator that has a {@link QueryData} remaining in it. | ||
| */ | ||
| private void seekToNextAvailableQueryData() { | ||
| // If the current sub iterator is null, attempt to get the next available iterator, or return early if there are no more iterators. | ||
| if (currentSubIterator == null) { | ||
| if (iterableIterator.hasNext()) { | ||
| currentSubIterator = iterableIterator.next().iterator(); | ||
| } else { | ||
| return; | ||
| } | ||
| } | ||
| // If the current sub iterator does not have any more elements remaining, move to the next sub iterator that does have elements. | ||
| if (!currentSubIterator.hasNext()) { | ||
| while (iterableIterator.hasNext()) { | ||
| // We must ensure we only ever call iterator() once on each sub-iterator. | ||
| currentSubIterator = iterableIterator.next().iterator(); | ||
| if (currentSubIterator.hasNext()) { | ||
| return; | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.