Merge branch 'opensearch-project:main' into main

.github/draft-release-notes-config.yml

@@ -37,4 +37,4 @@ categories:
       - 'Maintenance'
   - title: 'Refactoring'
     labels:
-      - 'Refactoring'
+      - 'Refactoring'

.github/workflows/add-untriaged.yml

@@ -1,19 +1,19 @@
-name: Apply 'untriaged' label during issue lifecycle
-
-on:
-  issues:
-    types: [opened, reopened, transferred]
-
-jobs:
-  apply-label:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/github-script@v6
-        with:
-          script: |
-            github.rest.issues.addLabels({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              labels: ['untriaged']
-            })
+name: Apply 'untriaged' label during issue lifecycle
+
+on:
+  issues:
+    types: [opened, reopened, transferred]
+
+jobs:
+  apply-label:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.issues.addLabels({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: ['untriaged']
+            })

.github/workflows/backwards_compatibility_tests_workflow.yml

@@ -0,0 +1,65 @@
+name: Backwards Compatibility Tests NeuralSearch
+on:
+  push:
+    branches:
+      - "*"
+      - "feature/**"
+  pull_request:
+    branches:
+      - "*"
+      - "feature/**"
+
+jobs:
+  Restart-Upgrade-BWCTests-NeuralSearch:
+    strategy:
+      matrix:
+        java: [ 11, 17, 21 ]
+        os: [ubuntu-latest,windows-latest]
+        bwc_version : ["2.9.0","2.10.0","2.11.0","2.12.0-SNAPSHOT"]
+        opensearch_version : [ "3.0.0-SNAPSHOT" ]
+
+    name: NeuralSearch Restart-Upgrade BWC Tests
+    runs-on: ${{ matrix.os }}
+    env:
+      BWC_VERSION_RESTART_UPGRADE: ${{ matrix.bwc_version }}
+
+    steps:
+      - name: Checkout NeuralSearch
+        uses: actions/checkout@v1
+
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}}
+        run: |
+          echo "Running restart-upgrade backwards compatibility tests ..."
+          ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=${{ matrix.bwc_version }}'
+
+  Rolling-Upgrade-BWCTests-NeuralSearch:
+    strategy:
+      matrix:
+        java: [ 11, 17, 21 ]
+        os: [ubuntu-latest,windows-latest]
+        bwc_version: [ "2.12.0-SNAPSHOT" ]
+        opensearch_version: [ "3.0.0-SNAPSHOT" ]
+
+    name: NeuralSearch Rolling-Upgrade BWC Tests
+    runs-on: ${{ matrix.os }}
+    env:
+      BWC_VERSION_ROLLING_UPGRADE: ${{ matrix.bwc_version }}
+
+    steps:
+      - name: Checkout NeuralSearch
+        uses: actions/checkout@v1
+
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}}
+        run: |
+          echo "Running rolling-upgrade backwards compatibility tests ..."
+          ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}'

.github/workflows/delete_backport_branch.yml

@@ -1,9 +1,9 @@
 name: Delete merged branch of the backport PRs
-on: 
+on:
   pull_request:
     types:
       - closed
-  
+
 jobs:
   delete-branch:
     runs-on: ubuntu-latest
@@ -12,4 +12,4 @@ jobs:
       - name: Delete merged branch
         uses: SvanBoxel/delete-merged-branch@main
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/links.yml

@@ -20,4 +20,4 @@ jobs:
         env:
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
       - name: Fail if there were link errors
-        run: exit ${{ steps.lychee.outputs.exit_code }}
+        run: exit ${{ steps.lychee.outputs.exit_code }}

CHANGELOG.md

@@ -20,7 +20,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Features
 ### Enhancements
 ### Bug Fixes
+- Fixing multiple issues reported in #497 ([#524](https://github.com/opensearch-project/neural-search/pull/524))
+- Fix Flaky test reported in #433 ([#533](https://github.com/opensearch-project/neural-search/pull/533))
 ### Infrastructure
+- BWC tests for Neural Search ([#515](https://github.com/opensearch-project/neural-search/pull/515))
 ### Documentation
 ### Maintenance
 ### Refactoring
+- Added spotless check in the build ([#515](https://github.com/opensearch-project/neural-search/pull/515))

DEVELOPER_GUIDE.md

@@ -11,6 +11,8 @@
     - [Run Single-node Cluster Locally](#run-single-node-cluster-locally)
     - [Run Multi-node Cluster Locally](#run-multi-node-cluster-locally)
   - [Debugging](#debugging)
+  - [Backwards Compatibility Testing](#backwards-compatibility-testing)
+    - [Adding new tests](#adding-new-tests)
   - [Supported configurations](#supported-configurations)
   - [Submitting Changes](#submitting-changes)
 
@@ -33,7 +35,7 @@ git clone https://github.com/[your username]/neural-search.git
 
 #### JDK 11
 
-OpenSearch builds using Java 11 at a minimum. This means you must have a JDK 11 installed with the environment variable 
+OpenSearch builds using Java 11 at a minimum. This means you must have a JDK 11 installed with the environment variable
 `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`.
 
 One easy way to get Java 11 on *nix is to use [sdkman](https://sdkman.io/).
@@ -83,10 +85,10 @@ Please follow these formatting guidelines:
 
 ## Build
 
-OpenSearch neural-search uses a [Gradle](https://docs.gradle.org/6.6.1/userguide/userguide.html) wrapper for its build. 
+OpenSearch neural-search uses a [Gradle](https://docs.gradle.org/6.6.1/userguide/userguide.html) wrapper for its build.
 Run `gradlew` on Unix systems.
 
-Build OpenSearch neural-search using `gradlew build` 
+Build OpenSearch neural-search using `gradlew build`
 
 ```
 ./gradlew build
@@ -184,6 +186,29 @@ Additionally, it is possible to attach one debugger to the cluster JVM and anoth
 ./gradlew :integTest -Dtest.debug=1 -Dcluster.debug=1
 ```
 
+## Backwards Compatibility Testing
+
+The purpose of Backwards Compatibility Testing and different types of BWC tests are explained [here](https://github.com/opensearch-project/opensearch-plugins/blob/main/TESTING.md#backwards-compatibility-testing). The BWC tests (i.e. Restart-Upgrade, Mixed-Cluster and Rolling-Upgrade scenarios) should be added with any new feature being added to Neural Search.
+The current design has mixed-cluster tests combined with rolling-upgrade tests in the same test class for [example](https://github.com/opensearch-project/neural-search/blob/main/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java).
+
+Use these commands to run BWC tests for neural search:
+1. Rolling upgrade tests: `./gradlew :qa:rolling-upgrade:testRollingUpgrade`
+2. Full restart upgrade tests: `./gradlew :qa:restart-upgrade:testAgainstNewCluster`
+3. `./gradlew :qa:bwcTestSuite` is used to run all the above bwc tests together.
+
+bwc.version stands for the older version of OpenSearch against which one needs to check the compatibility with the current version.
+The details regarding all bwc versions of OpenSearch can be found [here](https://github.com/opensearch-project/OpenSearch/blob/main/libs/core/src/main/java/org/opensearch/Version.java).
+Use this command to run BWC tests for a given Backwards Compatibility Version:
+```
+./gradlew :qa:bwcTestSuite -Dbwc.version=2.9.0
+```
+Here, we are testing BWC Tests with BWC version of plugin as 2.9.0.
+The tests will not run on MAC OS due to issues coming from the OS.
+
+### Adding new tests
+
+Before adding any new tests to Backward Compatibility Tests, we should be aware that the tests in BWC are not independent. While creating an index, a test cannot use the same index name if it is already used in other tests.
+
 ### Supported configurations
 
 By default, neural-search plugin supports `lucene` k-NN engine for local runs. Below is the sample request for creating of new index using this engine:
@@ -221,8 +246,8 @@ See [CONTRIBUTING](CONTRIBUTING.md).
 
 ## Backports
 
-The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the 
-original PR with an appropriate label `backport <backport-branch-name>` is merged to main with the backport workflow 
-run successfully on the PR. For example, if a PR on main needs to be backported to `2.x` branch, add a label 
-`backport 2.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is 
+The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the
+original PR with an appropriate label `backport <backport-branch-name>` is merged to main with the backport workflow
+run successfully on the PR. For example, if a PR on main needs to be backported to `2.x` branch, add a label
+`backport 2.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is
 merged to main, the workflow will create a backport PR to the `2.x` branch.

README.md

@@ -7,7 +7,7 @@
 ![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success)
 
 ## OpenSearch Neural Search
-**OpenSearch Neural Search** is an OpenSearch plugin that adds dense neural retrieval into the OpenSearch ecosystem. 
+**OpenSearch Neural Search** is an OpenSearch plugin that adds dense neural retrieval into the OpenSearch ecosystem.
 The plugin provides the capability for indexing documents and doing neural search on the indexed documents.
 
 ## Project Resources

build.gradle

@@ -11,12 +11,14 @@ import org.opensearch.gradle.test.RestIntegTestTask
 import java.util.concurrent.Callable
 
 apply plugin: 'java'
+apply plugin: 'java-test-fixtures'
 apply plugin: 'idea'
 apply plugin: 'opensearch.opensearchplugin'
 apply plugin: 'opensearch.pluginzip'
 apply plugin: 'jacoco'
 apply plugin: "com.diffplug.spotless"
 apply plugin: 'io.freefair.lombok'
+apply from: 'gradle/formatting.gradle'
 
 def pluginName = 'opensearch-neural-search'
 def pluginDescription = 'A plugin that adds dense neural retrieval into the OpenSearch ecosytem'
@@ -123,6 +125,25 @@ allprojects {
     version = "${opensearch_build}"
     targetCompatibility = JavaVersion.VERSION_11
     sourceCompatibility = JavaVersion.VERSION_11
+
+    apply from: rootProject.file('repositories.gradle').absoluteFile
+    plugins.withId('java') {
+        sourceCompatibility = targetCompatibility = "11"
+    }
+
+    afterEvaluate {
+        project.dependencyLicenses.enabled = false
+        project.thirdPartyAudit.enabled = false
+        project.loggerUsageCheck.enabled = false
+        project.forbiddenApis.ignoreFailures = false
+        project.forbiddenPatterns {
+            setEnabled(false)
+        }
+        project.testingConventions.enabled = false
+        project.validateNebulaPom.enabled = false
+        project.licenseFile = rootProject.file('LICENSE.txt')
+        project.noticeFile = rootProject.file('NOTICE.txt')
+    }
 }
 
 repositories {
@@ -144,6 +165,7 @@ dependencies {
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
     compileOnly fileTree(dir: knnJarDirectory, include: '*.jar')
     api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}"
+    testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}"
     implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
     // ml-common excluded reflection for runtime so we need to add it by ourselves.
     // https://github.com/opensearch-project/ml-commons/commit/464bfe34c66d7a729a00dd457f03587ea4e504d9
@@ -154,6 +176,10 @@ dependencies {
     runtimeOnly group: 'org.apache.commons', name: 'commons-text', version: '1.10.0'
     runtimeOnly group: 'com.google.code.gson', name: 'gson', version: '2.10.1'
     runtimeOnly group: 'org.json', name: 'json', version: '20231013'
+    testFixturesImplementation "org.opensearch:common-utils:${version}"
+    testFixturesImplementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
+    testFixturesCompileOnly group: 'com.google.guava', name: 'guava', version:'32.0.1-jre'
+    testFixturesCompileOnly fileTree(dir: knnJarDirectory, include: '*.jar')
 }
 
 // In order to add the jar to the classpath, we need to unzip the
@@ -168,11 +194,15 @@ project.tasks.delombok.dependsOn(extractKnnJar)
 
 compileJava {
     dependsOn extractKnnJar
+    dependsOn spotlessApply
     options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor'])
 }
 compileTestJava {
     options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor'])
 }
+compileTestFixturesJava {
+    options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor'])
+}
 
 def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
 opensearch_tmp_dir.mkdirs()
@@ -284,16 +314,6 @@ run {
     useCluster testClusters.integTest
 }
 
-spotless {
-    java {
-        removeUnusedImports()
-        importOrder 'java', 'javax', 'org', 'com'
-        eclipse().configFile rootProject.file('formatterConfig.xml')
-        trimTrailingWhitespace()
-        endWithNewline()
-    }
-}
-
 jacocoTestReport {
     dependsOn integTest, test
     reports {
@@ -302,6 +322,7 @@ jacocoTestReport {
     }
 }
 
+check.dependsOn spotlessCheck
 check.dependsOn jacocoTestCoverageVerification
 jacocoTestCoverageVerification.dependsOn jacocoTestReport
 

formatter/license-header.txt

@@ -0,0 +1,4 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */

gradle.properties

@@ -3,6 +3,12 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+# The BWC version here should always be the latest opensearch version set in
+# https://github.com/opensearch-project/OpenSearch/blob/main/libs/core/src/main/java/org/opensearch/Version.java .
+# Wired compatibility of OpenSearch works like 3.x version is compatible with 2.(latest-major) version.
+# Therefore, to run rolling-upgrade BWC Test on local machine the BWC version here should be set 2.(latest-major).
+systemProp.bwc.version=2.12.0-SNAPSHOT
+
 # For fixing Spotless check with Java 17
 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
   --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \

gradle/formatting.gradle

@@ -0,0 +1,32 @@
+allprojects {
+    project.apply plugin: "com.diffplug.spotless"
+    spotless {
+        java {
+            // Normally this isn't necessary, but we have Java sources in
+            // non-standard places
+            target '**/*.java'
+
+            removeUnusedImports()
+            eclipse().configFile rootProject.file('formatter/formatterConfig.xml')
+            trimTrailingWhitespace()
+            endWithNewline();
+
+            custom 'Refuse wildcard imports', {
+                // Wildcard imports can't be resolved; fail the build
+                if (it =~ /\s+import .*\*;/) {
+                    throw new AssertionError("Do not use wildcard imports.  'spotlessApply' cannot resolve this issue.")
+                }
+            }
+        }
+        format 'misc', {
+            target '*.md', '**/*.gradle', '**/*.json', '**/*.yaml', '**/*.yml', '**/*.svg'
+
+            trimTrailingWhitespace()
+            endWithNewline()
+        }
+        format("license", {
+            licenseHeaderFile("${rootProject.file("formatter/license-header.txt")}", "package ");
+            target("src/*/java/**/*.java","qa/*/java/**/*.java")
+        })
+    }
+}