Update Apache Lucene to 10.1.0

Signed-off-by: Andriy Redko <[email protected]>
opensearch-project · Dec 20, 2024 · 5588a62 · 5588a62
1 parent 0874389
commit 5588a62
Show file tree

Hide file tree

Showing 70 changed files with 113 additions and 117 deletions.
diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md
@@ -15,7 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957))
 
 ### Dependencies
-- Update Apache Lucene to 10.0.0 ([#16366](https://github.com/opensearch-project/OpenSearch/pull/16366))
+- Update Apache Lucene to 10.1.0 ([#16366](https://github.com/opensearch-project/OpenSearch/pull/16366))
 
 ### Changed
 - Changed locale provider from COMPAT to CLDR  ([#14345](https://github.com/opensearch-project/OpenSearch/pull/14345))

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -1,6 +1,6 @@
 [versions]
 opensearch        = "3.0.0"
-lucene            = "10.0.0"
+lucene            = "10.1.0"
 
 bundled_jdk_vendor = "adoptium"
 bundled_jdk = "23.0.1+11"

diff --git a/libs/core/licenses/lucene-core-10.0.0.jar.sha1 b/libs/core/licenses/lucene-core-10.0.0.jar.sha1
diff --git a/libs/core/licenses/lucene-core-10.1.0.jar.sha1 b/libs/core/licenses/lucene-core-10.1.0.jar.sha1
@@ -0,0 +1 @@
+65d7670de89a72433ef374b332da679a484d3a1e
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
@@ -112,9 +112,9 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_17_1 = new Version(2170199, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_2_17_2 = new Version(2170299, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_12_0);
-    public static final Version V_2_18_1 = new Version(2180199, org.apache.lucene.util.Version.fromBits(9, 12, 1));
-    public static final Version V_2_19_0 = new Version(2190099, org.apache.lucene.util.Version.fromBits(9, 12, 1));
-    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_10_0_0);
+    public static final Version V_2_18_1 = new Version(2180199, org.apache.lucene.util.Version.LUCENE_9_12_1);
+    public static final Version V_2_19_0 = new Version(2190099, org.apache.lucene.util.Version.LUCENE_9_12_1);
+    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_10_1_0);
     public static final Version CURRENT = V_3_0_0;
 
     public static Version fromId(int id) {

diff --git a/modules/lang-expression/licenses/lucene-expressions-10.0.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-10.0.0.jar.sha1
diff --git a/modules/lang-expression/licenses/lucene-expressions-10.1.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-10.1.0.jar.sha1
@@ -0,0 +1 @@
+5b2af7e4940909058dde5c707be7ddb266590309
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-10.0.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-10.0.0.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-10.1.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-10.1.0.jar.sha1
@@ -0,0 +1 @@
+93c315f11e5f6901595a5bd41e35224e081f4c84
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-10.0.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-10.0.0.jar.sha1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-10.1.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-10.1.0.jar.sha1
@@ -0,0 +1 @@
+5cbe34565be95b4dfe4386e7b8dc62eb9e9020e6
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-10.0.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-10.0.0.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-10.1.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-10.1.0.jar.sha1
@@ -0,0 +1 @@
+ae634b73ddd2e5de044993bf2a5ab36aeeadbf99
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-10.0.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-10.0.0.jar.sha1
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-10.1.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-10.1.0.jar.sha1
@@ -0,0 +1 @@
+62f9351b6ba692e948f0e4f718cf97e16962f441
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-10.0.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-10.0.0.jar.sha1
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-10.1.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-10.1.0.jar.sha1
@@ -0,0 +1 @@
+277221f0ddfb8b21053cd154cb973c61631ec669
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-10.0.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-10.0.0.jar.sha1
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-10.1.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-10.1.0.jar.sha1
@@ -0,0 +1 @@
+980b7136dac6b375d621237488930e8bc3a664c7
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-10.0.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-10.0.0.jar.sha1
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-10.1.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-10.1.0.jar.sha1
@@ -0,0 +1 @@
+b641b4a674c4c89d6cee7b28018aa378b933398b
diff --git a/server/licenses/lucene-analysis-common-10.0.0.jar.sha1 b/server/licenses/lucene-analysis-common-10.0.0.jar.sha1
diff --git a/server/licenses/lucene-analysis-common-10.1.0.jar.sha1 b/server/licenses/lucene-analysis-common-10.1.0.jar.sha1
@@ -0,0 +1 @@
+ddbc824a311d49a54f5808d5a01d5c52424c48b8
diff --git a/server/licenses/lucene-backward-codecs-10.0.0.jar.sha1 b/server/licenses/lucene-backward-codecs-10.0.0.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-10.1.0.jar.sha1 b/server/licenses/lucene-backward-codecs-10.1.0.jar.sha1
@@ -0,0 +1 @@
+fbbebd58f1505cc70d73dbdbb8196bfc29b9cf08
diff --git a/server/licenses/lucene-core-10.0.0.jar.sha1 b/server/licenses/lucene-core-10.0.0.jar.sha1
diff --git a/server/licenses/lucene-core-10.1.0.jar.sha1 b/server/licenses/lucene-core-10.1.0.jar.sha1
@@ -0,0 +1 @@
+65d7670de89a72433ef374b332da679a484d3a1e
diff --git a/server/licenses/lucene-grouping-10.0.0.jar.sha1 b/server/licenses/lucene-grouping-10.0.0.jar.sha1
diff --git a/server/licenses/lucene-grouping-10.1.0.jar.sha1 b/server/licenses/lucene-grouping-10.1.0.jar.sha1
@@ -0,0 +1 @@
+2f6409cbe456d5a9840969038173f0826f7ce4ff
diff --git a/server/licenses/lucene-highlighter-10.0.0.jar.sha1 b/server/licenses/lucene-highlighter-10.0.0.jar.sha1
diff --git a/server/licenses/lucene-highlighter-10.1.0.jar.sha1 b/server/licenses/lucene-highlighter-10.1.0.jar.sha1
@@ -0,0 +1 @@
+db9c72d8606788e5cd3a5c97e0a8b57648c8c6d4
diff --git a/server/licenses/lucene-join-10.0.0.jar.sha1 b/server/licenses/lucene-join-10.0.0.jar.sha1
diff --git a/server/licenses/lucene-join-10.1.0.jar.sha1 b/server/licenses/lucene-join-10.1.0.jar.sha1
@@ -0,0 +1 @@
+8ae87dbc1ee0a044b72366eb750c40b269c52fd1
diff --git a/server/licenses/lucene-memory-10.0.0.jar.sha1 b/server/licenses/lucene-memory-10.0.0.jar.sha1
diff --git a/server/licenses/lucene-memory-10.1.0.jar.sha1 b/server/licenses/lucene-memory-10.1.0.jar.sha1
@@ -0,0 +1 @@
+8f6a402ff8c40f28fe2debe3ca01c441ccbe5b2d
diff --git a/server/licenses/lucene-misc-10.0.0.jar.sha1 b/server/licenses/lucene-misc-10.0.0.jar.sha1
diff --git a/server/licenses/lucene-misc-10.1.0.jar.sha1 b/server/licenses/lucene-misc-10.1.0.jar.sha1
@@ -0,0 +1 @@
+ae1104521d00501e18e3c18c2b326f15589cc873
diff --git a/server/licenses/lucene-queries-10.0.0.jar.sha1 b/server/licenses/lucene-queries-10.0.0.jar.sha1
diff --git a/server/licenses/lucene-queries-10.1.0.jar.sha1 b/server/licenses/lucene-queries-10.1.0.jar.sha1
@@ -0,0 +1 @@
+48584c8a650cb4ce33a7e25864474e0da8c9eb53
diff --git a/server/licenses/lucene-queryparser-10.0.0.jar.sha1 b/server/licenses/lucene-queryparser-10.0.0.jar.sha1
diff --git a/server/licenses/lucene-queryparser-10.1.0.jar.sha1 b/server/licenses/lucene-queryparser-10.1.0.jar.sha1
@@ -0,0 +1 @@
+2774ab95128b0615568a4861b9a56f24511f774a
diff --git a/server/licenses/lucene-sandbox-10.0.0.jar.sha1 b/server/licenses/lucene-sandbox-10.0.0.jar.sha1
diff --git a/server/licenses/lucene-sandbox-10.1.0.jar.sha1 b/server/licenses/lucene-sandbox-10.1.0.jar.sha1
@@ -0,0 +1 @@
+1c10f36c5c2bb39ce5dd52af87834d5f0172b31d
diff --git a/server/licenses/lucene-spatial-extras-10.0.0.jar.sha1 b/server/licenses/lucene-spatial-extras-10.0.0.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-10.1.0.jar.sha1 b/server/licenses/lucene-spatial-extras-10.1.0.jar.sha1
@@ -0,0 +1 @@
+601a8e6f9be21ad99fbc43d8fe222553e55ccf80
diff --git a/server/licenses/lucene-spatial3d-10.0.0.jar.sha1 b/server/licenses/lucene-spatial3d-10.0.0.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-10.1.0.jar.sha1 b/server/licenses/lucene-spatial3d-10.1.0.jar.sha1
@@ -0,0 +1 @@
+59819cdc2b1af71dcb215284a2f48076b6d4235a
diff --git a/server/licenses/lucene-suggest-10.0.0.jar.sha1 b/server/licenses/lucene-suggest-10.0.0.jar.sha1
diff --git a/server/licenses/lucene-suggest-10.1.0.jar.sha1 b/server/licenses/lucene-suggest-10.1.0.jar.sha1
@@ -0,0 +1 @@
+84c212cf954bb48b022511d63b362d9a0a282ecf
diff --git a/server/src/main/java/org/opensearch/common/lucene/Lucene.java b/server/src/main/java/org/opensearch/common/lucene/Lucene.java
@@ -110,7 +110,7 @@
  * @opensearch.internal
  */
 public class Lucene {
-    public static final String LATEST_CODEC = "Lucene100";
+    public static final String LATEST_CODEC = "Lucene101";
 
     public static final String SOFT_DELETES_FIELD = "__soft_deletes";
 

diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java
@@ -34,8 +34,8 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene100.Lucene100Codec;
-import org.apache.lucene.codecs.lucene100.Lucene100Codec.Mode;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec.Mode;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.index.IndexSettings;
@@ -70,10 +70,10 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
         assert null != indexSettings;
         if (mapperService == null) {
-            codecs.put(DEFAULT_CODEC, new Lucene100Codec());
-            codecs.put(LZ4, new Lucene100Codec());
-            codecs.put(BEST_COMPRESSION_CODEC, new Lucene100Codec(Mode.BEST_COMPRESSION));
-            codecs.put(ZLIB, new Lucene100Codec(Mode.BEST_COMPRESSION));
+            codecs.put(DEFAULT_CODEC, new Lucene101Codec());
+            codecs.put(LZ4, new Lucene101Codec());
+            codecs.put(BEST_COMPRESSION_CODEC, new Lucene101Codec(Mode.BEST_COMPRESSION));
+            codecs.put(ZLIB, new Lucene101Codec(Mode.BEST_COMPRESSION));
         } else {
             // CompositeCodec still delegates to PerFieldMappingPostingFormatCodec
             // We can still support all the compression codecs when composite index is present

diff --git a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java
@@ -36,7 +36,7 @@
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene100.Lucene100Codec;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
 import org.opensearch.common.lucene.Lucene;
 import org.opensearch.index.codec.fuzzy.FuzzyFilterPostingsFormat;
@@ -59,7 +59,7 @@
  *
  * @opensearch.internal
  */
-public class PerFieldMappingPostingFormatCodec extends Lucene100Codec {
+public class PerFieldMappingPostingFormatCodec extends Lucene101Codec {
     private final Logger logger;
     private final MapperService mapperService;
     private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat();

diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java b/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java
@@ -10,10 +10,9 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene100.Lucene100Codec;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
 import org.opensearch.common.annotation.ExperimentalApi;
-import org.opensearch.index.codec.composite.composite100.Composite100Codec;
-import org.opensearch.index.codec.composite.composite912.Composite912Codec;
+import org.opensearch.index.codec.composite.composite101.Composite101Codec;
 import org.opensearch.index.mapper.MapperService;
 
 import java.util.HashMap;
@@ -33,16 +32,16 @@
 public class CompositeCodecFactory {
 
     // we can use this to track the latest composite codec
-    public static final String COMPOSITE_CODEC = Composite912Codec.COMPOSITE_INDEX_CODEC_NAME;
+    public static final String COMPOSITE_CODEC = Composite101Codec.COMPOSITE_INDEX_CODEC_NAME;
 
     public CompositeCodecFactory() {}
 
     public Map<String, Codec> getCompositeIndexCodecs(MapperService mapperService, Logger logger) {
         Map<String, Codec> codecs = new HashMap<>();
-        codecs.put(DEFAULT_CODEC, new Composite100Codec(Lucene100Codec.Mode.BEST_SPEED, mapperService, logger));
-        codecs.put(LZ4, new Composite100Codec(Lucene100Codec.Mode.BEST_SPEED, mapperService, logger));
-        codecs.put(BEST_COMPRESSION_CODEC, new Composite100Codec(Lucene100Codec.Mode.BEST_COMPRESSION, mapperService, logger));
-        codecs.put(ZLIB, new Composite100Codec(Lucene100Codec.Mode.BEST_COMPRESSION, mapperService, logger));
+        codecs.put(DEFAULT_CODEC, new Composite101Codec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger));
+        codecs.put(LZ4, new Composite101Codec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger));
+        codecs.put(BEST_COMPRESSION_CODEC, new Composite101Codec(Lucene101Codec.Mode.BEST_COMPRESSION, mapperService, logger));
+        codecs.put(ZLIB, new Composite101Codec(Lucene101Codec.Mode.BEST_COMPRESSION, mapperService, logger));
         return codecs;
     }
 }
diff --git a/...osite/composite100/Composite100Codec.java → ...osite/composite101/Composite101Codec.java b/...osite/composite100/Composite100Codec.java → ...osite/composite101/Composite101Codec.java
@@ -6,13 +6,13 @@
  * compatible open source license.
  */
 
-package org.opensearch.index.codec.composite.composite100;
+package org.opensearch.index.codec.composite.composite101;
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.lucene100.Lucene100Codec;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
 import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat;
@@ -25,16 +25,16 @@
  * @opensearch.experimental
  */
 @ExperimentalApi
-public class Composite100Codec extends FilterCodec {
-    public static final String COMPOSITE_INDEX_CODEC_NAME = "Composite100Codec";
+public class Composite101Codec extends FilterCodec {
+    public static final String COMPOSITE_INDEX_CODEC_NAME = "Composite101Codec";
     private final MapperService mapperService;
 
     // needed for SPI - this is used in reader path
-    public Composite100Codec() {
-        this(COMPOSITE_INDEX_CODEC_NAME, new Lucene100Codec(), null);
+    public Composite101Codec() {
+        this(COMPOSITE_INDEX_CODEC_NAME, new Lucene101Codec(), null);
     }
 
-    public Composite100Codec(Lucene100Codec.Mode compressionMode, MapperService mapperService, Logger logger) {
+    public Composite101Codec(Lucene101Codec.Mode compressionMode, MapperService mapperService, Logger logger) {
         this(COMPOSITE_INDEX_CODEC_NAME, new PerFieldMappingPostingFormatCodec(compressionMode, mapperService, logger), mapperService);
     }
 
@@ -46,7 +46,7 @@ public Composite100Codec(Lucene100Codec.Mode compressionMode, MapperService mapp
      * @param delegate codec delegate
      * @param mapperService mapper service instance
      */
-    protected Composite100Codec(String name, Codec delegate, MapperService mapperService) {
+    protected Composite101Codec(String name, Codec delegate, MapperService mapperService) {
         super(name, delegate);
         this.mapperService = mapperService;
     }

diff --git a/.../composite/composite100/package-info.java → .../composite/composite101/package-info.java b/.../composite/composite100/package-info.java → .../composite/composite101/package-info.java
diff --git a/server/src/main/java/org/opensearch/index/engine/Engine.java b/server/src/main/java/org/opensearch/index/engine/Engine.java
@@ -57,7 +57,6 @@
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
 import org.opensearch.ExceptionsHelper;
 import org.opensearch.action.index.IndexRequest;
 import org.opensearch.common.Nullable;
@@ -981,9 +980,7 @@ private Map<String, Long> getSegmentFileSizes(SegmentReader segmentReader) {
         boolean useCompoundFile = segmentCommitInfo.info.getUseCompoundFile();
         if (useCompoundFile) {
             try {
-                directory = engineConfig.getCodec()
-                    .compoundFormat()
-                    .getCompoundReader(segmentReader.directory(), segmentCommitInfo.info, IOContext.READONCE);
+                directory = engineConfig.getCodec().compoundFormat().getCompoundReader(segmentReader.directory(), segmentCommitInfo.info);
             } catch (IOException e) {
                 logger.warn(
                     () -> new ParameterizedMessage(

diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/CardinalityAggregator.java
@@ -354,7 +354,7 @@ private static class PruningCollector extends Collector {
 
             this.queue = new DisiPriorityQueue(postingMap.size());
             for (Scorer scorer : postingMap.values()) {
-                queue.add(new DisiWrapper(scorer));
+                queue.add(new DisiWrapper(scorer, false));
             }
 
             competitiveIterator = new DisjunctionDISI(queue);

diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -1,2 +1,2 @@
 org.opensearch.index.codec.composite.composite912.Composite912Codec
-org.opensearch.index.codec.composite.composite100.Composite100Codec
+org.opensearch.index.codec.composite.composite101.Composite101Codec