Update to Apache Lucene 10 for 3.0.0

Signed-off-by: Andriy Redko <[email protected]>
opensearch-project · Nov 1, 2024 · 0cfa7c8 · 0cfa7c8
1 parent 4b284c5
commit 0cfa7c8
Show file tree

Hide file tree

Showing 160 changed files with 782 additions and 381 deletions.
diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        java: [ 11, 17, 21 ]
+        java: [ 21, 23 ]
         os: [ubuntu-latest, windows-latest, macos-13]
     steps:
       - uses: actions/checkout@v4
@@ -16,17 +16,6 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
           distribution: temurin
-      - name: Set up JDK 17
-        # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
-        if: matrix.java == 11
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      - name: Set JAVA${{ matrix.java }}_HOME
-        shell: bash
-        run: |
-          echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
       - name: Setup docker (missing on MacOS)
         id: setup_docker
         if: runner.os == 'macos'
@@ -47,8 +36,8 @@ jobs:
         shell: bash
         if: runner.os != 'macos'
         run: |
-          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }} 
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE 
       - name: Run Gradle (assemble)
         if: runner.os == 'macos' && steps.setup_docker.outcome == 'success'
         run: |
-          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }}
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        java: [ 11, 17, 21, 23 ]
+        java: [ 21, 23 ]
         os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
     steps:
       - uses: actions/checkout@v4
@@ -17,18 +17,7 @@ jobs:
           java-version: ${{ matrix.java }}
           distribution: temurin
           cache: gradle
-      - name: Set up JDK 17
-        # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
-        if: matrix.java == 11
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      - name: Set JAVA${{ matrix.java }}_HOME
-        shell: bash
-        run: |
-          echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
       - name: Run Gradle (precommit)
         shell: bash
         run: |
-          ./gradlew javadoc precommit --parallel -Druntime.java=${{ matrix.java }}
+          ./gradlew javadoc precommit --parallel
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -171,8 +171,8 @@ if (project != rootProject) {
 
   allprojects {
     java {
-        targetCompatibility = JavaVersion.VERSION_11
-        sourceCompatibility = JavaVersion.VERSION_11
+        targetCompatibility = JavaVersion.VERSION_21
+        sourceCompatibility = JavaVersion.VERSION_21
     }
   }
 

diff --git a/buildSrc/reaper/build.gradle b/buildSrc/reaper/build.gradle
@@ -12,8 +12,8 @@
 apply plugin: 'java'
 
 java {
-  targetCompatibility = JavaVersion.VERSION_11
-  sourceCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_21
+  sourceCompatibility = JavaVersion.VERSION_21
 }
 
 jar {

diff --git a/buildSrc/src/testKit/thirdPartyAudit/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/build.gradle
@@ -46,11 +46,11 @@ dependencies {
 }
 
 tasks.register("empty", ThirdPartyAuditTask) {
-  targetCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_21
   signatureFile = file('third-party-audit-empty.txt')
 }
 
 tasks.register("absurd", ThirdPartyAuditTask) {
-  targetCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_21
   signatureFile = file('third-party-audit-absurd.txt')
 }
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -1,6 +1,6 @@
 [versions]
 opensearch        = "3.0.0"
-lucene            = "9.12.0"
+lucene            = "10.0.0"
 
 bundled_jdk_vendor = "adoptium"
 bundled_jdk = "23.0.1+11"

diff --git a/libs/core/licenses/lucene-core-10.0.0.jar.sha1 b/libs/core/licenses/lucene-core-10.0.0.jar.sha1
@@ -0,0 +1 @@
+5a9b3f728041df5b054aaaed3c3fd7ff0fed8ee7
diff --git a/libs/core/licenses/lucene-core-9.12.0.jar.sha1 b/libs/core/licenses/lucene-core-9.12.0.jar.sha1
diff --git a/libs/core/src/main/java/org/opensearch/LegacyESVersion.java b/libs/core/src/main/java/org/opensearch/LegacyESVersion.java
@@ -50,7 +50,7 @@ public class LegacyESVersion extends Version {
 
     public static final LegacyESVersion V_6_0_0 = new LegacyESVersion(6000099, org.apache.lucene.util.Version.fromBits(7, 0, 0));
     public static final LegacyESVersion V_6_5_0 = new LegacyESVersion(6050099, org.apache.lucene.util.Version.fromBits(7, 0, 0));
-    public static final LegacyESVersion V_7_2_0 = new LegacyESVersion(7020099, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final LegacyESVersion V_7_2_0 = new LegacyESVersion(7020099, org.apache.lucene.util.Version.fromBits(8, 0, 0));
 
     // todo move back to Version.java if retiring legacy version support
     protected static final Map<Integer, Version> idToVersion;

diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
@@ -112,8 +112,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_17_1 = new Version(2170199, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_2_17_2 = new Version(2170299, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_12_0);
-    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0);
     public static final Version V_2_19_0 = new Version(2190099, org.apache.lucene.util.Version.LUCENE_9_12_0);
+    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_10_0_0);
     public static final Version CURRENT = V_3_0_0;
 
     public static Version fromId(int id) {

diff --git a/modules/lang-expression/licenses/lucene-expressions-10.0.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-10.0.0.jar.sha1
@@ -0,0 +1 @@
+89b26348ec305598fc224cc9583939564b67b2cf
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.12.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.12.0.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-10.0.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-10.0.0.jar.sha1
@@ -0,0 +1 @@
+082c38c1335c069a73622c37ca3d39e64c1b2d33
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0.jar.sha1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-10.0.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-10.0.0.jar.sha1
@@ -0,0 +1 @@
+26071742008630779523d08c0b46b2f371ef23a0
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-10.0.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-10.0.0.jar.sha1
@@ -0,0 +1 @@
+4a6ff02a1bd34a3c0165da05f714bb8188074bdc
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0.jar.sha1
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-10.0.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-10.0.0.jar.sha1
@@ -0,0 +1 @@
+4c488697df5038a78e5e65bb9b6da120af62d824
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0.jar.sha1
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-10.0.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-10.0.0.jar.sha1
@@ -0,0 +1 @@
+1137b9846ec000b49c70c3fe5f8cd79b7129be22
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0.jar.sha1
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-10.0.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-10.0.0.jar.sha1
@@ -0,0 +1 @@
+3fd86db5e9748063369db4bed84f1bd2ca62d387
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0.jar.sha1
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-10.0.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-10.0.0.jar.sha1
@@ -0,0 +1 @@
+4e6b940b3b934d6de174fedaaeaefd647698648d
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0.jar.sha1
diff --git a/server/licenses/lucene-analysis-common-10.0.0.jar.sha1 b/server/licenses/lucene-analysis-common-10.0.0.jar.sha1
@@ -0,0 +1 @@
+13eb016bab14973158554a2e6cdf2abbc5c3eda1
diff --git a/server/licenses/lucene-analysis-common-9.12.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.12.0.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-10.0.0.jar.sha1 b/server/licenses/lucene-backward-codecs-10.0.0.jar.sha1
@@ -0,0 +1 @@
+8e21f708eb1bbb71ce79cbfea093b6ca913f4abf
diff --git a/server/licenses/lucene-backward-codecs-9.12.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.12.0.jar.sha1
diff --git a/server/licenses/lucene-core-10.0.0.jar.sha1 b/server/licenses/lucene-core-10.0.0.jar.sha1
@@ -0,0 +1 @@
+5a9b3f728041df5b054aaaed3c3fd7ff0fed8ee7
diff --git a/server/licenses/lucene-core-9.12.0.jar.sha1 b/server/licenses/lucene-core-9.12.0.jar.sha1
diff --git a/server/licenses/lucene-grouping-10.0.0.jar.sha1 b/server/licenses/lucene-grouping-10.0.0.jar.sha1
@@ -0,0 +1 @@
+17145d786d31e7ecd68d149ccc3e7ab83270f282
diff --git a/server/licenses/lucene-grouping-9.12.0.jar.sha1 b/server/licenses/lucene-grouping-9.12.0.jar.sha1
diff --git a/server/licenses/lucene-highlighter-10.0.0.jar.sha1 b/server/licenses/lucene-highlighter-10.0.0.jar.sha1
@@ -0,0 +1 @@
+b8324f1b859620912c186b27d9666215ce3d258b
diff --git a/server/licenses/lucene-highlighter-9.12.0.jar.sha1 b/server/licenses/lucene-highlighter-9.12.0.jar.sha1
diff --git a/server/licenses/lucene-join-10.0.0.jar.sha1 b/server/licenses/lucene-join-10.0.0.jar.sha1
@@ -0,0 +1 @@
+3a4c5bf84c855b011e740f30cb8a23f2ee85e1c1
diff --git a/server/licenses/lucene-join-9.12.0.jar.sha1 b/server/licenses/lucene-join-9.12.0.jar.sha1
diff --git a/server/licenses/lucene-memory-10.0.0.jar.sha1 b/server/licenses/lucene-memory-10.0.0.jar.sha1
@@ -0,0 +1 @@
+bc0f37a0a06b445555d07e5fe199d73436d51352
diff --git a/server/licenses/lucene-memory-9.12.0.jar.sha1 b/server/licenses/lucene-memory-9.12.0.jar.sha1
diff --git a/server/licenses/lucene-misc-10.0.0.jar.sha1 b/server/licenses/lucene-misc-10.0.0.jar.sha1
@@ -0,0 +1 @@
+5f619b32c62bb9405e7af595cf7311113ed62e33
diff --git a/server/licenses/lucene-misc-9.12.0.jar.sha1 b/server/licenses/lucene-misc-9.12.0.jar.sha1
diff --git a/server/licenses/lucene-queries-10.0.0.jar.sha1 b/server/licenses/lucene-queries-10.0.0.jar.sha1
@@ -0,0 +1 @@
+3d2f98787e27e8e2a65d994c86563edf16dd92f1
diff --git a/server/licenses/lucene-queries-9.12.0.jar.sha1 b/server/licenses/lucene-queries-9.12.0.jar.sha1
diff --git a/server/licenses/lucene-queryparser-10.0.0.jar.sha1 b/server/licenses/lucene-queryparser-10.0.0.jar.sha1
@@ -0,0 +1 @@
+e11886c913058ef20378715dee715d942d04babc
diff --git a/server/licenses/lucene-queryparser-9.12.0.jar.sha1 b/server/licenses/lucene-queryparser-9.12.0.jar.sha1
diff --git a/server/licenses/lucene-sandbox-10.0.0.jar.sha1 b/server/licenses/lucene-sandbox-10.0.0.jar.sha1
@@ -0,0 +1 @@
+1795ce5b066bda61483c375f6b8e358aaa4f6348
diff --git a/server/licenses/lucene-sandbox-9.12.0.jar.sha1 b/server/licenses/lucene-sandbox-9.12.0.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-10.0.0.jar.sha1 b/server/licenses/lucene-spatial-extras-10.0.0.jar.sha1
@@ -0,0 +1 @@
+a701eb363cf0a75ebacd1844398314250abcf592
diff --git a/server/licenses/lucene-spatial-extras-9.12.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.12.0.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-10.0.0.jar.sha1 b/server/licenses/lucene-spatial3d-10.0.0.jar.sha1
@@ -0,0 +1 @@
+d9e0a8a6084d7657a633c1aa94d750414f5288c4
diff --git a/server/licenses/lucene-spatial3d-9.12.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.12.0.jar.sha1
diff --git a/server/licenses/lucene-suggest-10.0.0.jar.sha1 b/server/licenses/lucene-suggest-10.0.0.jar.sha1
@@ -0,0 +1 @@
+d79099abc148a1906e129abbabd5e1b18a20c117
diff --git a/server/licenses/lucene-suggest-9.12.0.jar.sha1 b/server/licenses/lucene-suggest-9.12.0.jar.sha1
diff --git a/server/src/main/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumerWrapper.java b/server/src/main/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumerWrapper.java
@@ -27,12 +27,20 @@ public class Lucene90DocValuesConsumerWrapper implements Closeable {
 
     public Lucene90DocValuesConsumerWrapper(
         SegmentWriteState state,
+        int skipIndexIntervalSize,
         String dataCodec,
         String dataExtension,
         String metaCodec,
         String metaExtension
     ) throws IOException {
-        lucene90DocValuesConsumer = new Lucene90DocValuesConsumer(state, dataCodec, dataExtension, metaCodec, metaExtension);
+        lucene90DocValuesConsumer = new Lucene90DocValuesConsumer(
+            state,
+            skipIndexIntervalSize,
+            dataCodec,
+            dataExtension,
+            metaCodec,
+            metaExtension
+        );
     }
 
     public Lucene90DocValuesConsumer getLucene90DocValuesConsumer() {

diff --git a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java
@@ -42,6 +42,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.BytesRef;
@@ -84,7 +85,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
         return new ConstantScoreWeight(this, boost) {
 
             @Override
-            public Scorer scorer(LeafReaderContext context) throws IOException {
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
                 final BinaryDocValues values = context.reader().getBinaryDocValues(fieldName);
                 if (values == null) {
                     return null;
@@ -129,7 +130,9 @@ public float matchCost() {
                         return 4; // at most 4 comparisons
                     }
                 };
-                return new ConstantScoreScorer(this, score(), scoreMode, iterator);
+
+                final Scorer scorer = new ConstantScoreScorer(score(), scoreMode, iterator);
+                return new DefaultScorerSupplier(scorer);
             }
 
             @Override

diff --git a/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
@@ -47,6 +47,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.IOSupplier;
 import org.apache.lucene.util.InPlaceMergeSorter;
 
 import java.io.IOException;
@@ -208,7 +209,12 @@ private TermStates adjustTTF(IndexReaderContext readerContext, TermStates termCo
         int df = termContext.docFreq();
         long ttf = sumTTF;
         for (int i = 0; i < len; i++) {
-            TermState termState = termContext.get(leaves.get(i));
+            final IOSupplier<TermState> termStateSupplier = termContext.get(leaves.get(i));
+            if (termStateSupplier == null) {
+                continue;
+            }
+
+            final TermState termState = termStateSupplier.get();
             if (termState == null) {
                 continue;
             }
@@ -232,10 +238,16 @@ private static TermStates adjustDF(IndexReaderContext readerContext, TermStates
         }
         TermStates newCtx = new TermStates(readerContext);
         for (int i = 0; i < len; ++i) {
-            TermState termState = ctx.get(leaves.get(i));
+            final IOSupplier<TermState> termStateSupplier = ctx.get(leaves.get(i));
+            if (termStateSupplier == null) {
+                continue;
+            }
+
+            final TermState termState = termStateSupplier.get();
             if (termState == null) {
                 continue;
             }
+
             newCtx.register(termState, i, newDocFreq, newTTF);
             newDocFreq = 0;
             newTTF = 0;
@@ -385,7 +397,7 @@ protected Query topLevelQuery(Term[] terms, TermStates[] ctx, int[] docFreqs, in
                 if (low.clauses().isEmpty()) {
                     BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
                     for (BooleanClause booleanClause : high) {
-                        queryBuilder.add(booleanClause.getQuery(), Occur.MUST);
+                        queryBuilder.add(booleanClause.query(), Occur.MUST);
                     }
                     return queryBuilder.build();
                 } else if (high.clauses().isEmpty()) {

diff --git a/server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java b/server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java
@@ -192,10 +192,10 @@ public static CollapseTopFieldDocs merge(Sort sort, int start, int size, Collaps
             final CollapseTopFieldDocs shard = shardHits[shardIDX];
             // totalHits can be non-zero even if no hits were
             // collected, when searchAfter was used:
-            totalHitCount += shard.totalHits.value;
+            totalHitCount += shard.totalHits.value();
             // If any hit count is a lower bound then the merged
             // total hit count is a lower bound as well
-            if (shard.totalHits.relation == TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO) {
+            if (shard.totalHits.relation() == TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO) {
                 totalHitsRelation = TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO;
             }
             if (CollectionUtils.isEmpty(shard.scoreDocs) == false) {

diff --git a/server/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java b/server/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java
@@ -229,7 +229,7 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException {
                             public boolean advanceExact(int target) throws IOException {
                                 if (sorted.advanceExact(target)) {
                                     ord = (int) sorted.nextOrd();
-                                    if (sorted.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) {
+                                    if (sorted.nextOrd() != SortedSetDocValues.NO_MORE_DOCS) {
                                         throw new IllegalStateException(
                                             "failed to collapse " + target + ", the collapse field must be single valued"
                                         );