Skip to content

Commit

Permalink
Remove IOContext from Directory#openChecksumInput (#12027)
Browse files Browse the repository at this point in the history
  • Loading branch information
zacharymorn authored Dec 26, 2022
1 parent c9401bf commit 008a0d4
Show file tree
Hide file tree
Showing 49 changed files with 62 additions and 78 deletions.
2 changes: 2 additions & 0 deletions lucene/CHANGES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ API Changes
* GITHUB#11840: Query rewrite now takes an IndexSearcher instead of IndexReader to enable concurrent
rewriting. (Patrick Zhai)

* GITHUB#11933: Remove IOContext from Directory#openChecksumInput. (Zach Chen)

* GITHUB#11814: Support deletions in IndexRearranger. (Stefan Vodita)

New Features
Expand Down
6 changes: 6 additions & 0 deletions lucene/MIGRATE.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,12 @@ determine the number of valid ordinals for the currently-positioned document up-
illegal to call `SortedSetDocValues#nextOrd()` more than `SortedSetDocValues#docValueCount()` times
for the currently-positioned document (doing so will result in undefined behavior).

### IOContext removed from Directory#openChecksumInput (GITHUB-12027)

`Directory#openChecksumInput` no longer takes in `IOContext` as a parameter, and will always use value
`IOContext.READONCE` for opening internally, as that's the only valid usage pattern for checksum input.
Callers should remove the parameter when calling this method.

## Migration from Lucene 9.0 to Lucene 9.1

### Test framework package migration and module (LUCENE-10301)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1136,7 +1136,7 @@ private WordStorage readSortedDictionaries(
new WordStorage.Builder(wordCount, hasCustomMorphData, flags, allNonSuggestibleFlags());

try (ByteSequencesReader reader =
new ByteSequencesReader(tempDir.openChecksumInput(sorted, IOContext.READONCE), sorted)) {
new ByteSequencesReader(tempDir.openChecksumInput(sorted), sorted)) {

// TODO: the flags themselves can be double-chars (long) or also numeric
// either way the trick is to encode them as char... but they must be parsed differently
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ public FieldInfos read(
throws IOException {
final String fileName =
IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
try (ChecksumIndexInput input = directory.openChecksumInput(fileName, context)) {
try (ChecksumIndexInput input = directory.openChecksumInput(fileName)) {
Throwable priorE = null;
FieldInfo[] infos = null;
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ private int readMetadata(SegmentReadState state) throws IOException {
IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, Lucene90HnswVectorsFormat.META_EXTENSION);
int versionMeta = -1;
try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName, state.context)) {
try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) {
Throwable priorE = null;
try {
versionMeta =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ private int readMetadata(SegmentReadState state) throws IOException {
IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, Lucene91HnswVectorsFormat.META_EXTENSION);
int versionMeta = -1;
try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName, state.context)) {
try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) {
Throwable priorE = null;
try {
versionMeta =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ private int readMetadata(SegmentReadState state) throws IOException {
IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, Lucene92HnswVectorsFormat.META_EXTENSION);
int versionMeta = -1;
try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName, state.context)) {
try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) {
Throwable priorE = null;
try {
versionMeta =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -89,8 +88,7 @@ private int readMetadata(SegmentReadState state) throws IOException {
IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, Lucene94HnswVectorsFormat.META_EXTENSION);
int versionMeta = -1;
try (ChecksumIndexInput meta =
state.directory.openChecksumInput(metaFileName, IOContext.READONCE)) {
try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) {
Throwable priorE = null;
try {
versionMeta =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public static IndexInput openInput(Directory directory, String name, IOContext c
/** Open a checksum index input */
public static ChecksumIndexInput openChecksumInput(
Directory directory, String name, IOContext context) throws IOException {
return new EndiannessReverserChecksumIndexInput(directory.openChecksumInput(name, context));
return new EndiannessReverserChecksumIndexInput(directory.openChecksumInput(name));
}

/** Open an index output */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.TrackingDirectoryWrapper;
import org.apache.lucene.util.ArrayUtil;
Expand Down Expand Up @@ -1577,7 +1576,7 @@ private Error verifyChecksum(Throwable priorException, PointWriter writer) throw
// We are reading from a temp file; go verify the checksum:
String tempFileName = ((OfflinePointWriter) writer).name;
if (tempDir.getCreatedFiles().contains(tempFileName)) {
try (ChecksumIndexInput in = tempDir.openChecksumInput(tempFileName, IOContext.READONCE)) {
try (ChecksumIndexInput in = tempDir.openChecksumInput(tempFileName)) {
CodecUtil.checkFooter(in, priorException);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public class TestEndiannessReverserCheckSumIndexInput extends EndiannessReverser
@Override
protected IndexInput getEndiannessReverserInput(Directory dir, String name, IOContext context)
throws IOException {
return new EndiannessReverserChecksumIndexInput(dir.openChecksumInput(name, context));
return new EndiannessReverserChecksumIndexInput(dir.openChecksumInput(name));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ public BloomFilteredFieldsProducer(SegmentReadState state) throws IOException {
ChecksumIndexInput bloomIn = null;
boolean success = false;
try {
bloomIn = state.directory.openChecksumInput(bloomFileName, state.context);
bloomIn = state.directory.openChecksumInput(bloomFileName);
CodecUtil.checkIndexHeader(
bloomIn,
BLOOM_CODEC_NAME,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
import org.apache.lucene.index.PointValues.Relation;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.TrackingDirectoryWrapper;
import org.apache.lucene.util.ArrayUtil;
Expand Down Expand Up @@ -841,7 +840,7 @@ private Error verifyChecksum(Throwable priorException, PointWriter writer) throw
if (writer instanceof OfflinePointWriter) {
// We are reading from a temp file; go verify the checksum:
String tempFileName = ((OfflinePointWriter) writer).name;
try (ChecksumIndexInput in = tempDir.openChecksumInput(tempFileName, IOContext.READONCE)) {
try (ChecksumIndexInput in = tempDir.openChecksumInput(tempFileName)) {
CodecUtil.checkFooter(in, priorException);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public FieldInfos read(
throws IOException {
final String fileName =
IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, FIELD_INFOS_EXTENSION);
ChecksumIndexInput input = directory.openChecksumInput(fileName, iocontext);
ChecksumIndexInput input = directory.openChecksumInput(fileName);
BytesRefBuilder scratch = new BytesRefBuilder();

boolean success = false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,7 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {
SimpleTextKnnVectorsFormat.VECTOR_EXTENSION);

boolean success = false;
try (ChecksumIndexInput in =
readState.directory.openChecksumInput(metaFileName, IOContext.DEFAULT)) {
try (ChecksumIndexInput in = readState.directory.openChecksumInput(metaFileName)) {
int fieldNumber = readInt(in, FIELD_NUMBER);
while (fieldNumber != -1) {
String fieldName = readString(in, FIELD_NAME);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext contex
ChecksumIndexInput in = null;
boolean success = false;
try {
in = dir.openChecksumInput(fileName, context);
in = dir.openChecksumInput(fileName);

SimpleTextUtil.readLine(in, scratch);
assert StringHelper.startsWith(scratch.get(), SIZE);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,7 @@ public SimpleTextPointsReader(SegmentReadState readState) throws IOException {
readState.segmentInfo.name,
readState.segmentSuffix,
SimpleTextPointsFormat.POINT_INDEX_EXTENSION);
try (ChecksumIndexInput in =
readState.directory.openChecksumInput(indexFileName, IOContext.DEFAULT)) {
try (ChecksumIndexInput in = readState.directory.openChecksumInput(indexFileName)) {
readLine(in);
int count = parseInt(FIELD_COUNT);
for (int i = 0; i < count; i++) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public SegmentInfo read(
BytesRefBuilder scratch = new BytesRefBuilder();
String segFileName =
IndexFileNames.segmentFileName(segmentName, "", SimpleTextSegmentInfoFormat.SI_EXTENSION);
try (ChecksumIndexInput input = directory.openChecksumInput(segFileName, context)) {
try (ChecksumIndexInput input = directory.openChecksumInput(segFileName)) {
SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), SI_VERSION);
final Version version;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ private void writeCompoundFile(
// align file start offset
long startOffset = data.alignFilePointer(Long.BYTES);
// write bytes for file
try (ChecksumIndexInput in = dir.openChecksumInput(file, IOContext.READONCE)) {
try (ChecksumIndexInput in = dir.openChecksumInput(file)) {

// just copies the index header, verifying that its id matches what we expect
CodecUtil.verifyAndCopyIndexHeader(in, data, si.getId());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,7 @@ public Lucene90CompoundReader(Directory directory, SegmentInfo si, IOContext con
private Map<String, FileEntry> readEntries(
byte[] segmentID, Directory dir, String entriesFileName) throws IOException {
Map<String, FileEntry> mapping = null;
try (ChecksumIndexInput entriesStream =
dir.openChecksumInput(entriesFileName, IOContext.READONCE)) {
try (ChecksumIndexInput entriesStream = dir.openChecksumInput(entriesFileName)) {
Throwable priorE = null;
try {
version =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -83,7 +82,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
merging = false;

// read in the entries from the metadata file.
try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName, IOContext.READONCE)) {
try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName)) {
Throwable priorE = null;

try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext contex
long gen = info.getDelGen();
String name = IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, gen);
final int length = info.info.maxDoc();
try (ChecksumIndexInput input = dir.openChecksumInput(name, context)) {
try (ChecksumIndexInput input = dir.openChecksumInput(name)) {
Throwable priorE = null;
try {
CodecUtil.checkIndexHeader(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.IOUtils;
Expand Down Expand Up @@ -61,7 +60,7 @@ final class Lucene90NormsProducer extends NormsProducer implements Cloneable {
int version = -1;

// read in the entries from the metadata file.
try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName, IOContext.READONCE)) {
try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName)) {
Throwable priorE = null;
try {
version =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,7 @@ public Lucene90PointsReader(SegmentReadState readState) throws IOException {
CodecUtil.retrieveChecksum(dataIn);

long indexLength = -1, dataLength = -1;
try (ChecksumIndexInput metaIn =
readState.directory.openChecksumInput(metaFileName, IOContext.READONCE)) {
try (ChecksumIndexInput metaIn = readState.directory.openChecksumInput(metaFileName)) {
Throwable priorE = null;
try {
CodecUtil.checkIndexHeader(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ public Lucene90SegmentInfoFormat() {}
public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context)
throws IOException {
final String fileName = IndexFileNames.segmentFileName(segment, "", SI_EXTENSION);
try (ChecksumIndexInput input = dir.openChecksumInput(fileName, context)) {
try (ChecksumIndexInput input = dir.openChecksumInput(fileName)) {
Throwable priorE = null;
SegmentInfo si = null;
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,7 @@ public Lucene90BlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentRe
Map<String, FieldReader> fieldMap = null;
Throwable priorE = null;
long indexLength = -1, termsLength = -1;
try (ChecksumIndexInput metaIn =
state.directory.openChecksumInput(metaName, IOContext.READONCE)) {
try (ChecksumIndexInput metaIn = state.directory.openChecksumInput(metaName)) {
try {
CodecUtil.checkIndexHeader(
metaIn,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,7 @@ void finish(int numDocs, long maxPointer, IndexOutput metaOut) throws IOExceptio
metaOut.writeInt(totalChunks + 1);
metaOut.writeLong(dataOut.getFilePointer());

try (ChecksumIndexInput docsIn =
dir.openChecksumInput(docsOut.getName(), IOContext.READONCE)) {
try (ChecksumIndexInput docsIn = dir.openChecksumInput(docsOut.getName())) {
CodecUtil.checkHeader(docsIn, codecName + "Docs", VERSION_CURRENT, VERSION_CURRENT);
Throwable priorE = null;
try {
Expand All @@ -147,8 +146,7 @@ void finish(int numDocs, long maxPointer, IndexOutput metaOut) throws IOExceptio
docsOut = null;

metaOut.writeLong(dataOut.getFilePointer());
try (ChecksumIndexInput filePointersIn =
dir.openChecksumInput(filePointersOut.getName(), IOContext.READONCE)) {
try (ChecksumIndexInput filePointersIn = dir.openChecksumInput(filePointersOut.getName())) {
CodecUtil.checkHeader(
filePointersIn, codecName + "FilePointers", VERSION_CURRENT, VERSION_CURRENT);
Throwable priorE = null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ public Lucene90CompressingStoredFieldsReader(

final String metaStreamFN =
IndexFileNames.segmentFileName(segment, segmentSuffix, META_EXTENSION);
metaIn = d.openChecksumInput(metaStreamFN, IOContext.READONCE);
metaIn = d.openChecksumInput(metaStreamFN);
CodecUtil.checkIndexHeader(
metaIn,
INDEX_CODEC_NAME + "Meta",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ public Lucene90CompressingTermVectorsReader(

final String metaStreamFN =
IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_META_EXTENSION);
metaIn = d.openChecksumInput(metaStreamFN, IOContext.READONCE);
metaIn = d.openChecksumInput(metaStreamFN);
CodecUtil.checkIndexHeader(
metaIn,
VECTORS_INDEX_CODEC_NAME + "Meta",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ public FieldInfos read(
throws IOException {
final String fileName =
IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
try (ChecksumIndexInput input = directory.openChecksumInput(fileName, context)) {
try (ChecksumIndexInput input = directory.openChecksumInput(fileName)) {
Throwable priorE = null;
FieldInfo[] infos = null;
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.Accountable;
Expand Down Expand Up @@ -96,8 +95,7 @@ private int readMetadata(SegmentReadState state) throws IOException {
IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, Lucene95HnswVectorsFormat.META_EXTENSION);
int versionMeta = -1;
try (ChecksumIndexInput meta =
state.directory.openChecksumInput(metaFileName, IOContext.READONCE)) {
try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) {
Throwable priorE = null;
try {
versionMeta =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ static final SegmentInfos readCommit(

long generation = generationFromSegmentsFileName(segmentFileName);
// System.out.println(Thread.currentThread() + ": SegmentInfos.readCommit " + segmentFileName);
try (ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READ)) {
try (ChecksumIndexInput input = directory.openChecksumInput(segmentFileName)) {
try {
return readCommit(directory, input, generation, minSupportedMajorVersion);
} catch (EOFException | NoSuchFileException | FileNotFoundException e) {
Expand Down
4 changes: 2 additions & 2 deletions lucene/core/src/java/org/apache/lucene/store/Directory.java
Original file line number Diff line number Diff line change
Expand Up @@ -152,8 +152,8 @@ public abstract IndexOutput createTempOutput(String prefix, String suffix, IOCon
* @param name the name of an existing file.
* @throws IOException in case of I/O error
*/
public ChecksumIndexInput openChecksumInput(String name, IOContext context) throws IOException {
return new BufferedChecksumIndexInput(openInput(name, context));
public ChecksumIndexInput openChecksumInput(String name) throws IOException {
return new BufferedChecksumIndexInput(openInput(name, IOContext.READONCE));
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -283,8 +283,7 @@ public String sort(String inputFileName) throws IOException {
TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(dir);

boolean success = false;
try (ByteSequencesReader is =
getReader(dir.openChecksumInput(inputFileName, IOContext.READONCE), inputFileName)) {
try (ByteSequencesReader is = getReader(dir.openChecksumInput(inputFileName), inputFileName)) {
while (true) {
Partition part = readPartition(is);
if (part.count == 0) {
Expand Down Expand Up @@ -367,7 +366,7 @@ public String sort(String inputFileName) throws IOException {
*/
private void verifyChecksum(Throwable priorException, ByteSequencesReader reader)
throws IOException {
try (ChecksumIndexInput in = dir.openChecksumInput(reader.name, IOContext.READONCE)) {
try (ChecksumIndexInput in = dir.openChecksumInput(reader.name)) {
CodecUtil.checkFooter(in, priorException);
}
}
Expand Down Expand Up @@ -701,9 +700,7 @@ protected boolean lessThan(FileAndTop a, FileAndTop b) {
// Open streams and read the top for each file
for (int i = 0; i < segmentsToMerge.size(); i++) {
Partition segment = getPartition(segmentsToMerge.get(i));
streams[i] =
getReader(
dir.openChecksumInput(segment.fileName, IOContext.READONCE), segment.fileName);
streams[i] = getReader(dir.openChecksumInput(segment.fileName), segment.fileName);

BytesRef item = null;
try {
Expand Down
Loading

0 comments on commit 008a0d4

Please sign in to comment.