Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

replace Map<String,Object> with IntObjectHashMap for DV producer #13961

Merged
merged 2 commits into from
Oct 29, 2024
Merged

replace Map<String,Object> with IntObjectHashMap for DV producer #13961

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
355e57c
replace Map<String,Object> with IntObjectHashMap for DV producer
bugmakerrrrrr Aug 26, 2024
e3fdaa1
apply review comments
bugmakerrrrrr Oct 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions lucene/CHANGES.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ Optimizations
* GITHUB#13943: Removed shared `HitsThresholdChecker`, which reduces overhead
but may delay a bit when dynamic pruning kicks in. (Adrien Grand)

* GITHUB#13961: Replace Map<String,Object> with IntObjectHashMap for DV producer. (Pan Guixin)

Bug Fixes
---------------------
* GITHUB#13832: Fixed an issue where the DefaultPassageFormatter.format method did not format passages as intended
Expand Down
33 changes: 16 additions & 17 deletions ...codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80DocValuesProducer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@
package org.apache.lucene.backward_codecs.lucene80;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicReader;
import org.apache.lucene.backward_codecs.packed.LegacyDirectReader;
import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
Expand All @@ -41,6 +39,7 @@
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
Expand All @@ -53,11 +52,11 @@

/** reader for {@link Lucene80DocValuesFormat} */
final class Lucene80DocValuesProducer extends DocValuesProducer {
private final Map<String, NumericEntry> numerics = new HashMap<>();
private final Map<String, BinaryEntry> binaries = new HashMap<>();
private final Map<String, SortedEntry> sorted = new HashMap<>();
private final Map<String, SortedSetEntry> sortedSets = new HashMap<>();
private final Map<String, SortedNumericEntry> sortedNumerics = new HashMap<>();
private final IntObjectHashMap<NumericEntry> numerics = new IntObjectHashMap<>();
private final IntObjectHashMap<BinaryEntry> binaries = new IntObjectHashMap<>();
private final IntObjectHashMap<SortedEntry> sorted = new IntObjectHashMap<>();
private final IntObjectHashMap<SortedSetEntry> sortedSets = new IntObjectHashMap<>();
private final IntObjectHashMap<SortedNumericEntry> sortedNumerics = new IntObjectHashMap<>();
private final IndexInput data;
private final int maxDoc;
private int version = -1;
Expand Down Expand Up @@ -139,7 +138,7 @@ private void readFields(String segmentName, IndexInput meta, FieldInfos infos)
}
byte type = meta.readByte();
if (type == Lucene80DocValuesFormat.NUMERIC) {
numerics.put(info.name, readNumeric(meta));
numerics.put(info.number, readNumeric(meta));
} else if (type == Lucene80DocValuesFormat.BINARY) {
final boolean compressed;
if (version >= Lucene80DocValuesFormat.VERSION_CONFIGURABLE_COMPRESSION) {
Expand All @@ -158,13 +157,13 @@ private void readFields(String segmentName, IndexInput meta, FieldInfos infos)
} else {
compressed = version >= Lucene80DocValuesFormat.VERSION_BIN_COMPRESSED;
}
binaries.put(info.name, readBinary(meta, compressed));
binaries.put(info.number, readBinary(meta, compressed));
} else if (type == Lucene80DocValuesFormat.SORTED) {
sorted.put(info.name, readSorted(meta));
sorted.put(info.number, readSorted(meta));
} else if (type == Lucene80DocValuesFormat.SORTED_SET) {
sortedSets.put(info.name, readSortedSet(meta));
sortedSets.put(info.number, readSortedSet(meta));
} else if (type == Lucene80DocValuesFormat.SORTED_NUMERIC) {
sortedNumerics.put(info.name, readSortedNumeric(meta));
sortedNumerics.put(info.number, readSortedNumeric(meta));
} else {
throw new CorruptIndexException("invalid type: " + type, meta);
}
Expand Down Expand Up @@ -426,7 +425,7 @@ private static class SortedNumericEntry extends NumericEntry {

@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
NumericEntry entry = numerics.get(field.name);
NumericEntry entry = numerics.get(field.number);
return getNumeric(entry);
}

Expand Down Expand Up @@ -915,7 +914,7 @@ BytesRef decode(int docNumber) throws IOException {

@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
BinaryEntry entry = binaries.get(field.name);
BinaryEntry entry = binaries.get(field.number);
if (entry.compressed) {
return getCompressedBinary(entry);
} else {
Expand Down Expand Up @@ -973,7 +972,7 @@ public BytesRef binaryValue() throws IOException {

@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
SortedEntry entry = sorted.get(field.name);
SortedEntry entry = sorted.get(field.number);
return getSorted(entry);
}

Expand Down Expand Up @@ -1407,7 +1406,7 @@ public int docFreq() throws IOException {

@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
SortedNumericEntry entry = sortedNumerics.get(field.name);
SortedNumericEntry entry = sortedNumerics.get(field.number);
if (entry.numValues == entry.numDocsWithField) {
return DocValues.singleton(getNumeric(entry));
}
Expand Down Expand Up @@ -1543,7 +1542,7 @@ private void set() {

@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
SortedSetEntry entry = sortedSets.get(field.name);
SortedSetEntry entry = sortedSets.get(field.number);
if (entry.singleValueEntry != null) {
return DocValues.singleton(getSorted(entry.singleValueEntry));
}
Expand Down
63 changes: 31 additions & 32 deletions lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.TERMS_DICT_BLOCK_LZ4_SHIFT;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.BaseTermsEnum;
Expand All @@ -43,6 +41,7 @@
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
Expand All @@ -59,12 +58,12 @@

/** reader for {@link Lucene90DocValuesFormat} */
final class Lucene90DocValuesProducer extends DocValuesProducer {
private final Map<String, NumericEntry> numerics;
private final Map<String, BinaryEntry> binaries;
private final Map<String, SortedEntry> sorted;
private final Map<String, SortedSetEntry> sortedSets;
private final Map<String, SortedNumericEntry> sortedNumerics;
private final Map<String, DocValuesSkipperEntry> skippers;
private final IntObjectHashMap<NumericEntry> numerics;
private final IntObjectHashMap<BinaryEntry> binaries;
private final IntObjectHashMap<SortedEntry> sorted;
private final IntObjectHashMap<SortedSetEntry> sortedSets;
private final IntObjectHashMap<SortedNumericEntry> sortedNumerics;
private final IntObjectHashMap<DocValuesSkipperEntry> skippers;
private final IndexInput data;
private final int maxDoc;
private int version = -1;
Expand All @@ -81,12 +80,12 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
String metaName =
IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
this.maxDoc = state.segmentInfo.maxDoc();
numerics = new HashMap<>();
binaries = new HashMap<>();
sorted = new HashMap<>();
sortedSets = new HashMap<>();
sortedNumerics = new HashMap<>();
skippers = new HashMap<>();
numerics = new IntObjectHashMap<>();
binaries = new IntObjectHashMap<>();
sorted = new IntObjectHashMap<>();
sortedSets = new IntObjectHashMap<>();
sortedNumerics = new IntObjectHashMap<>();
skippers = new IntObjectHashMap<>();
merging = false;

// read in the entries from the metadata file.
Expand Down Expand Up @@ -149,12 +148,12 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {

// Used for cloning
private Lucene90DocValuesProducer(
Map<String, NumericEntry> numerics,
Map<String, BinaryEntry> binaries,
Map<String, SortedEntry> sorted,
Map<String, SortedSetEntry> sortedSets,
Map<String, SortedNumericEntry> sortedNumerics,
Map<String, DocValuesSkipperEntry> skippers,
IntObjectHashMap<NumericEntry> numerics,
IntObjectHashMap<BinaryEntry> binaries,
IntObjectHashMap<SortedEntry> sorted,
IntObjectHashMap<SortedSetEntry> sortedSets,
IntObjectHashMap<SortedNumericEntry> sortedNumerics,
IntObjectHashMap<DocValuesSkipperEntry> skippers,
IndexInput data,
int maxDoc,
int version,
Expand Down Expand Up @@ -194,18 +193,18 @@ private void readFields(IndexInput meta, FieldInfos infos) throws IOException {
}
byte type = meta.readByte();
if (info.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
skippers.put(info.name, readDocValueSkipperMeta(meta));
skippers.put(info.number, readDocValueSkipperMeta(meta));
}
if (type == Lucene90DocValuesFormat.NUMERIC) {
numerics.put(info.name, readNumeric(meta));
numerics.put(info.number, readNumeric(meta));
} else if (type == Lucene90DocValuesFormat.BINARY) {
binaries.put(info.name, readBinary(meta));
binaries.put(info.number, readBinary(meta));
} else if (type == Lucene90DocValuesFormat.SORTED) {
sorted.put(info.name, readSorted(meta));
sorted.put(info.number, readSorted(meta));
} else if (type == Lucene90DocValuesFormat.SORTED_SET) {
sortedSets.put(info.name, readSortedSet(meta));
sortedSets.put(info.number, readSortedSet(meta));
} else if (type == Lucene90DocValuesFormat.SORTED_NUMERIC) {
sortedNumerics.put(info.name, readSortedNumeric(meta));
sortedNumerics.put(info.number, readSortedNumeric(meta));
} else {
throw new CorruptIndexException("invalid type: " + type, meta);
}
Expand Down Expand Up @@ -430,7 +429,7 @@ private static class SortedNumericEntry extends NumericEntry {

@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
NumericEntry entry = numerics.get(field.name);
NumericEntry entry = numerics.get(field.number);
return getNumeric(entry);
}

Expand Down Expand Up @@ -786,7 +785,7 @@ public boolean advanceExact(int target) throws IOException {

@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
BinaryEntry entry = binaries.get(field.name);
BinaryEntry entry = binaries.get(field.number);

if (entry.docsWithFieldOffset == -2) {
return DocValues.emptyBinary();
Expand Down Expand Up @@ -887,7 +886,7 @@ public BytesRef binaryValue() throws IOException {

@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
SortedEntry entry = sorted.get(field.name);
SortedEntry entry = sorted.get(field.number);
return getSorted(entry);
}

Expand Down Expand Up @@ -1363,7 +1362,7 @@ public int docFreq() throws IOException {

@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
SortedNumericEntry entry = sortedNumerics.get(field.name);
SortedNumericEntry entry = sortedNumerics.get(field.number);
return getSortedNumeric(entry);
}

Expand Down Expand Up @@ -1508,7 +1507,7 @@ private void set() {

@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
SortedSetEntry entry = sortedSets.get(field.name);
SortedSetEntry entry = sortedSets.get(field.number);
if (entry.singleValueEntry != null) {
return DocValues.singleton(getSorted(entry.singleValueEntry));
}
Expand Down Expand Up @@ -1782,7 +1781,7 @@ long getLongValue(long index) throws IOException {

@Override
public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {
final DocValuesSkipperEntry entry = skippers.get(field.name);
final DocValuesSkipperEntry entry = skippers.get(field.number);

final IndexInput input = data.slice("doc value skipper", entry.offset, entry.length);
// Prefetch the first page of data. Following pages are expected to get prefetched through
Expand Down
23 changes: 12 additions & 11 deletions lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.util.IOUtils;

/**
Expand Down Expand Up @@ -256,7 +257,7 @@ static String getFullSegmentSuffix(String outerSegmentSuffix, String segmentSuff

private static class FieldsReader extends DocValuesProducer {

private final Map<String, DocValuesProducer> fields = new HashMap<>();
private final IntObjectHashMap<DocValuesProducer> fields = new IntObjectHashMap<>();
private final Map<String, DocValuesProducer> formats = new HashMap<>();

// clone for merge
Expand All @@ -270,10 +271,10 @@ private static class FieldsReader extends DocValuesProducer {
}

// Then rebuild fields:
for (Map.Entry<String, DocValuesProducer> ent : other.fields.entrySet()) {
DocValuesProducer producer = oldToNew.get(ent.getValue());
for (IntObjectHashMap.IntObjectCursor<DocValuesProducer> ent : other.fields) {
DocValuesProducer producer = oldToNew.get(ent.value);
assert producer != null;
fields.put(ent.getKey(), producer);
fields.put(ent.key, producer);
}
}

Expand Down Expand Up @@ -302,7 +303,7 @@ public FieldsReader(final SegmentReadState readState) throws IOException {
segmentSuffix,
format.fieldsProducer(new SegmentReadState(readState, segmentSuffix)));
}
fields.put(fieldName, formats.get(segmentSuffix));
fields.put(fi.number, formats.get(segmentSuffix));
}
}
}
Expand All @@ -316,37 +317,37 @@ public FieldsReader(final SegmentReadState readState) throws IOException {

@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
DocValuesProducer producer = fields.get(field.name);
DocValuesProducer producer = fields.get(field.number);
return producer == null ? null : producer.getNumeric(field);
}

@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
DocValuesProducer producer = fields.get(field.name);
DocValuesProducer producer = fields.get(field.number);
return producer == null ? null : producer.getBinary(field);
}

@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
DocValuesProducer producer = fields.get(field.name);
DocValuesProducer producer = fields.get(field.number);
return producer == null ? null : producer.getSorted(field);
}

@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
DocValuesProducer producer = fields.get(field.name);
DocValuesProducer producer = fields.get(field.number);
return producer == null ? null : producer.getSortedNumeric(field);
}

@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
DocValuesProducer producer = fields.get(field.name);
DocValuesProducer producer = fields.get(field.number);
return producer == null ? null : producer.getSortedSet(field);
}

@Override
public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {
DocValuesProducer producer = fields.get(field.name);
DocValuesProducer producer = fields.get(field.number);
return producer == null ? null : producer.getSkipper(field);
}

Expand Down
Loading