Merge branch 'main' into simplify_codec_setup_in_vector_tests

apache · Nov 4, 2024 · 56d1f2e · 56d1f2e
2 parents b0de85c + 1328527
commit 56d1f2e
Show file tree

Hide file tree

Showing 93 changed files with 6,846 additions and 923 deletions.
diff --git a/gradle/generation/forUtil.gradle b/gradle/generation/forUtil.gradle
@@ -23,7 +23,7 @@ configure(project(":lucene:core")) {
     description "Regenerate gen_ForUtil.py"
     group "generation"
 
-    def genDir = file("src/java/org/apache/lucene/codecs/lucene912")
+    def genDir = file("src/java/org/apache/lucene/codecs/lucene101")
     def genScript = file("${genDir}/gen_ForUtil.py")
     def genOutput = file("${genDir}/ForUtil.java")
 
@@ -48,7 +48,7 @@ configure(project(":lucene:core")) {
     description "Regenerate gen_ForDeltaUtil.py"
     group "generation"
 
-    def genDir = file("src/java/org/apache/lucene/codecs/lucene912")
+    def genDir = file("src/java/org/apache/lucene/codecs/lucene101")
     def genScript = file("${genDir}/gen_ForDeltaUtil.py")
     def genOutput = file("${genDir}/ForDeltaUtil.java")
 
@@ -68,6 +68,7 @@ configure(project(":lucene:core")) {
           andThenTasks: ["spotlessJava", "spotlessJavaApply"],
           mustRunBefore: [ "compileJava" ]
   ])
+
 }
 
 configure(project(":lucene:backward-codecs")) {
@@ -146,5 +147,55 @@ configure(project(":lucene:backward-codecs")) {
           andThenTasks: ["spotlessJava", "spotlessJavaApply"],
           mustRunBefore: [ "compileJava" ]
   ])
+
+  task generateForUtil912Internal() {
+    description "Regenerate gen_ForUtil.py"
+    group "generation"
+
+    def genDir = file("src/java/org/apache/lucene/backward_codecs/lucene912")
+    def genScript = file("${genDir}/gen_ForUtil.py")
+    def genOutput = file("${genDir}/ForUtil.java")
+
+    inputs.file genScript
+    outputs.file genOutput
+
+    doLast {
+      quietExec {
+        workingDir genDir
+        executable project.externalTool("python3")
+        args = [ '-B', genScript ]
+      }
+    }
+  }
+
+  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil912Internal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
+
+  task generateForDeltaUtil912Internal() {
+    description "Regenerate gen_ForDeltaUtil.py"
+    group "generation"
+
+    def genDir = file("src/java/org/apache/lucene/backward_codecs/lucene912")
+    def genScript = file("${genDir}/gen_ForDeltaUtil.py")
+    def genOutput = file("${genDir}/ForDeltaUtil.java")
+
+    inputs.file genScript
+    outputs.file genOutput
+
+    doLast {
+      quietExec {
+        workingDir genDir
+        executable project.externalTool("python3")
+        args = [ '-B', genScript ]
+      }
+    }
+  }
+
+  regenerate.dependsOn wrapWithPersistentChecksums(generateForDeltaUtil912Internal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 }
 
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
@@ -84,6 +84,11 @@ Optimizations
 
 * GITHUB#13763: Replace Map<String,Object> with IntObjectHashMap for KnnVectorsReader (Pan Guixin)
 
+* GITHUB#13968: Switch postings from storing doc IDs in a long[] to an int[].
+  Lucene 8.4 had moved to a long[] to help speed up block decoding by using
+  longs that would pack two integers. We are now moving back to integers to be
+  able to take advantage of 2x more lanes with the vector API. (Adrien Grand)
+
 Bug Fixes
 ---------------------
 * GITHUB#13832: Fixed an issue where the DefaultPassageFormatter.format method did not format passages as intended

diff --git a/lucene/backward-codecs/src/generated/checksums/generateForDeltaUtil912.json b/lucene/backward-codecs/src/generated/checksums/generateForDeltaUtil912.json
@@ -0,0 +1,4 @@
+{
+    "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene912/ForDeltaUtil.java": "b81961f0b277b1458ca259e0d23ccc4eeeb47fe7",
+    "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene912/gen_ForDeltaUtil.py": "3191d7591309b7876c5c709fb9375af5b87c2ef8"
+}
diff --git a/lucene/backward-codecs/src/generated/checksums/generateForUtil912.json b/lucene/backward-codecs/src/generated/checksums/generateForUtil912.json
@@ -0,0 +1,4 @@
+{
+    "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene912/ForUtil.java": "e6db3c665dfebca8b93eb6b4651d2eb3af637b02",
+    "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene912/gen_ForUtil.py": "993ecc9cf7ea821963384070669695257b16e040"
+}
diff --git a/lucene/backward-codecs/src/java/module-info.java b/lucene/backward-codecs/src/java/module-info.java
@@ -37,6 +37,7 @@
   exports org.apache.lucene.backward_codecs.lucene95;
   exports org.apache.lucene.backward_codecs.lucene99;
   exports org.apache.lucene.backward_codecs.lucene912;
+  exports org.apache.lucene.backward_codecs.lucene100;
   exports org.apache.lucene.backward_codecs.packed;
   exports org.apache.lucene.backward_codecs.store;
 
@@ -46,7 +47,8 @@
       org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat,
       org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat,
       org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat,
-      org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat;
+      org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat,
+      org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat;
   provides org.apache.lucene.codecs.KnnVectorsFormat with
       org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsFormat,
       org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat,
@@ -64,5 +66,6 @@
       org.apache.lucene.backward_codecs.lucene94.Lucene94Codec,
       org.apache.lucene.backward_codecs.lucene95.Lucene95Codec,
       org.apache.lucene.backward_codecs.lucene99.Lucene99Codec,
-      org.apache.lucene.backward_codecs.lucene912.Lucene912Codec;
+      org.apache.lucene.backward_codecs.lucene912.Lucene912Codec,
+      org.apache.lucene.backward_codecs.lucene100.Lucene100Codec;
 }
diff --git a/...cene/codecs/lucene100/Lucene100Codec.java → ...ward_codecs/lucene100/Lucene100Codec.java b/...cene/codecs/lucene100/Lucene100Codec.java → ...ward_codecs/lucene100/Lucene100Codec.java
@@ -14,9 +14,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.lucene.codecs.lucene100;
+package org.apache.lucene.backward_codecs.lucene100;
 
 import java.util.Objects;
+import org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.CompoundFormat;
 import org.apache.lucene.codecs.DocValuesFormat;
@@ -37,7 +38,6 @@
 import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;
-import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
 import org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat;
 import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
 import org.apache.lucene.codecs.lucene99.Lucene99SegmentInfoFormat;
@@ -50,7 +50,7 @@
  *
  * <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.
  *
- * @see org.apache.lucene.codecs.lucene100 package documentation for file format details.
+ * @see org.apache.lucene.backward_codecs.lucene100 package documentation for file format details.
  * @lucene.experimental
  */
 public class Lucene100Codec extends Codec {

diff --git a/...lucene/codecs/lucene912/package-info.java → ...ckward_codecs/lucene100/package-info.java b/...lucene/codecs/lucene912/package-info.java → ...ckward_codecs/lucene100/package-info.java
@@ -15,5 +15,5 @@
  * limitations under the License.
  */
 
-/** Lucene 9.12 file format. */
-package org.apache.lucene.codecs.lucene912;
+/** Lucene 10.0 file format. */
+package org.apache.lucene.backward_codecs.lucene100;