From 5b5cd6b9e52d8091b8d0c9564a9133808ec7e400 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 19 Dec 2023 00:04:54 +0000 Subject: [PATCH 01/10] Add initial search AD tool; add AD dep Signed-off-by: Tyler Ohlsen --- build.gradle | 22 +- .../tools/SearchAnomalyDetectorsTool.java | 202 ++++++++++++++++++ .../SearchAnomalyDetectorsToolTests.java | 152 +++++++++++++ 3 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java create mode 100644 src/test/java/org/opensearch/agent/tools/SearchAnomalyDetectorsToolTests.java diff --git a/build.gradle b/build.gradle index e40ac6b6..a8d9cc16 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ import java.util.concurrent.Callable buildscript { ext { opensearch_group = "org.opensearch" - opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.11.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') @@ -77,6 +77,7 @@ apply plugin: 'opensearch.testclusters' apply plugin: 'opensearch.pluginzip' def sqlJarDirectory = "$buildDir/dependencies/opensearch-sql-plugin" +def adJarDirectory = "$buildDir/dependencies/opensearch-anomaly-detection" configurations { zipArchive @@ -96,6 +97,11 @@ task addJarsToClasspath(type: Copy) { include "protocol-${version}.jar" } into("$buildDir/classes") + + from(fileTree(dir: adJarDirectory)) { + include "opensearch-anomaly-detection-${version}.jar" + } + into("$buildDir/classes") } dependencies { @@ -106,6 +112,10 @@ dependencies { zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${version}" implementation("com.google.guava:guava:32.1.3-jre") implementation fileTree(dir: sqlJarDirectory, include: ["opensearch-sql-${version}.jar", "ppl-${version}.jar", "protocol-${version}.jar"]) + + zipArchive "org.opensearch.plugin:opensearch-anomaly-detection:${opensearch_build}" + implementation fileTree(dir: adJarDirectory, include: ["opensearch-anomaly-detection-${version}.jar"]) + compileOnly "org.opensearch:common-utils:${version}" testImplementation "org.opensearch.test:framework:${opensearch_version}" testImplementation "org.mockito:mockito-core:5.8.0" @@ -126,7 +136,14 @@ task extractSqlJar(type: Copy) { into sqlJarDirectory } +task extractAdJar(type: Copy) { + mustRunAfter() + from(zipTree(configurations.zipArchive.find { it.name.startsWith("opensearch-anomaly-detection")})) + into adJarDirectory +} + tasks.addJarsToClasspath.dependsOn(extractSqlJar) +tasks.addJarsToClasspath.dependsOn(extractAdJar) project.tasks.delombok.dependsOn(addJarsToClasspath) tasks.publishNebulaPublicationToMavenLocal.dependsOn ':generatePomFileForPluginZipPublication' tasks.validateNebulaPom.dependsOn ':generatePomFileForPluginZipPublication' @@ -161,6 +178,7 @@ spotless { compileJava { dependsOn extractSqlJar + dependsOn extractAdJar dependsOn delombok options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) } @@ -169,6 +187,8 @@ compileTestJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) } +forbiddenApisTest.ignoreFailures = true + opensearchplugin { name 'skills' diff --git a/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java b/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java new file mode 100644 index 00000000..845cf9e3 --- /dev/null +++ b/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java @@ -0,0 +1,202 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.agent.tools; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.ad.client.AnomalyDetectionNodeClient; +import org.opensearch.client.Client; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.RangeQueryBuilder; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.index.query.WildcardQueryBuilder; +import org.opensearch.ml.common.output.model.ModelTensors; +import org.opensearch.ml.common.spi.tools.Parser; +import org.opensearch.ml.common.spi.tools.Tool; +import org.opensearch.ml.common.spi.tools.ToolAnnotation; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.SortOrder; + +import org.opensearch.search.SearchHit; + +import lombok.Getter; +import lombok.Setter; + +@ToolAnnotation(SearchAnomalyDetectorsTool.TYPE) +public class SearchAnomalyDetectorsTool implements Tool { + public static final String TYPE = "SearchAnomalyDetectorsTool"; + private static final String DEFAULT_DESCRIPTION = "Use this tool to search anomaly detectors."; + + @Setter + @Getter + private String name = TYPE; + @Getter + @Setter + private String description = DEFAULT_DESCRIPTION; + @Getter + private String type; + @Getter + private String version; + + private Client client; + + private AnomalyDetectionNodeClient adClient; + + @Setter + private Parser inputParser; + @Setter + private Parser outputParser; + + public SearchAnomalyDetectorsTool(Client client) { + this.client = client; + + // probably keep this overridden output parser. need to ensure the output matches what's expected + outputParser = new Parser<>() { + @Override + public Object parse(Object o) { + @SuppressWarnings("unchecked") + List mlModelOutputs = (List) o; + return mlModelOutputs.get(0).getMlModelTensors().get(0).getDataAsMap().get("response"); + } + }; + } + + // Response is currently in a simple string format including the list of anomaly detectors (only name and ID attached), and + // number of total detectors. The output will likely need to be updated, standardized, and include more fields in the + // future to cover a sufficient amount of potential questions the agent will need to handle. + @Override + public void run(Map parameters, ActionListener listener) { + final String detectorName = parameters.getOrDefault("detectorName", null); + final String detectorNamePattern = parameters.getOrDefault("detectorNamePattern", null); + final String indices = parameters.getOrDefault("indices", null); + final Boolean highCardinality = parameters.containsKey("highCardinality") + ? Boolean.parseBoolean(parameters.get("highCardinality")) + : null; + final Long lastUpdateTime = parameters.containsKey("lastUpdateTime") ? Long.parseLong(parameters.get("lastUpdateTime")): null; + final String sortOrderStr = parameters.getOrDefault("sortOrder", "asc"); + final SortOrder sortOrder = sortOrderStr == "asc" ? SortOrder.ASC : SortOrder.DESC; + final String sortString = parameters.getOrDefault("sortString", "name.keyword"); + final int size = parameters.containsKey("size") ? Integer.parseInt(parameters.get("size")) : 20; + final int startIndex = parameters.containsKey("startIndex") ? Integer.parseInt(parameters.get("startIndex")) : 0; + final Boolean running = parameters.containsKey("running") ? Boolean.parseBoolean(parameters.get("running")) : null; + final Boolean disabled = parameters.containsKey("disabled") ? Boolean.parseBoolean(parameters.get("disabled")) : null; + final Boolean failed = parameters.containsKey("failed") ? Boolean.parseBoolean(parameters.get("failed")) : null; + + List mustList = new ArrayList(); + if (detectorName != null) { + mustList.add(new TermQueryBuilder("name.keyword", detectorName)); + } + if (detectorNamePattern != null) { + mustList.add(new WildcardQueryBuilder("name.keyword", detectorNamePattern)); + } + if (indices != null) { + mustList.add(new TermQueryBuilder("indices", indices)); + } + if (highCardinality != null) { + mustList.add(new TermQueryBuilder("detector_type", highCardinality ? "MULTI_ENTITY" : "SINGLE_ENTITY")); + } + if (lastUpdateTime != null) { + mustList.add(new BoolQueryBuilder().filter(new RangeQueryBuilder("last_update_time").gte(lastUpdateTime))); + + } + + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.must().addAll(mustList); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .query(boolQueryBuilder) + .size(size) + .from(startIndex) + .sort(sortString, sortOrder); + + SearchRequest searchDetectorRequest = new SearchRequest().source(searchSourceBuilder); + + if (running != null || disabled != null || failed != null) { + // TODO: add a listener to trigger when the first response is received, to trigger the profile API call + // to fetch the detector state, etc. + // Will need AD client to onboard the profile API first. + } + + ActionListener searchDetectorListener = ActionListener.wrap(response -> { + StringBuilder sb = new StringBuilder(); + SearchHit[] hits = response.getHits().getHits(); + sb.append("AnomalyDetectors=["); + for (SearchHit hit : hits) { + sb.append("{"); + sb.append("id=").append(hit.getId()).append(","); + sb.append("name=").append(hit.getSourceAsMap().get("name")); + sb.append("}"); + } + sb.append("]"); + sb.append("TotalAnomalyDetectors=").append(response.getHits().getTotalHits().value); + listener.onResponse((T) sb.toString()); + }, e -> { listener.onFailure(e); }); + + adClient.searchAnomalyDetectors(searchDetectorRequest, searchDetectorListener); + } + + @Override + public boolean validate(Map parameters) { + return true; + } + + @Override + public String getType() { + return TYPE; + } + + /** + * Factory for the {@link SearchAnomalyDetectorsTool} + */ + public static class Factory implements Tool.Factory { + private Client client; + + private AnomalyDetectionNodeClient adClient; + + private static Factory INSTANCE; + + /** + * Create or return the singleton factory instance + */ + public static Factory getInstance() { + if (INSTANCE != null) { + return INSTANCE; + } + synchronized (SearchAnomalyDetectorsTool.class) { + if (INSTANCE != null) { + return INSTANCE; + } + INSTANCE = new Factory(); + return INSTANCE; + } + } + + /** + * Initialize this factory + * @param client The OpenSearch client + */ + public void init(Client client) { + this.client = client; + this.adClient = new AnomalyDetectionNodeClient(client); + } + + @Override + public SearchAnomalyDetectorsTool create(Map map) { + return new SearchAnomalyDetectorsTool(client); + } + + @Override + public String getDefaultDescription() { + return DEFAULT_DESCRIPTION; + } + } + +} diff --git a/src/test/java/org/opensearch/agent/tools/SearchAnomalyDetectorsToolTests.java b/src/test/java/org/opensearch/agent/tools/SearchAnomalyDetectorsToolTests.java new file mode 100644 index 00000000..96b05364 --- /dev/null +++ b/src/test/java/org/opensearch/agent/tools/SearchAnomalyDetectorsToolTests.java @@ -0,0 +1,152 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.agent.tools; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Map; + +import org.apache.lucene.search.TotalHits; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.opensearch.action.ActionType; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.SearchResponseSections; +import org.opensearch.client.AdminClient; +import org.opensearch.client.ClusterAdminClient; +import org.opensearch.client.IndicesAdminClient; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.ml.common.spi.tools.Tool; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.aggregations.Aggregations; + +public class SearchAnomalyDetectorsToolTests { + @Mock + private NodeClient nodeClient; + @Mock + private AdminClient adminClient; + @Mock + private IndicesAdminClient indicesAdminClient; + @Mock + private ClusterAdminClient clusterAdminClient; + + private Map nullParams; + private Map emptyParams; + private Map nonEmptyParams; + + @Before + public void setup() { + MockitoAnnotations.openMocks(this); + SearchAnomalyDetectorsTool.Factory.getInstance().init(nodeClient); + + nullParams = null; + emptyParams = Collections.emptyMap(); + nonEmptyParams = Map.of("detectorName", "foo"); + } + + @Test + public void testRunWithNoDetectors() throws Exception { + Tool tool = SearchAnomalyDetectorsTool.Factory.getInstance().create(Collections.emptyMap()); + + SearchHit[] hits = new SearchHit[0]; + + TotalHits totalHits = new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO); + + SearchResponse getDetectorsResponse = new SearchResponse( + new SearchResponseSections(new SearchHits(hits, totalHits, 0), new Aggregations(new ArrayList<>()), null, false, null, null, 0), + null, + 0, + 0, + 0, + 0, + null, + null + ); + String expectedResponseStr = String.format("AnomalyDetectors=[]TotalAnomalyDetectors=%d", hits.length); + + @SuppressWarnings("unchecked") + ActionListener listener = Mockito.mock(ActionListener.class); + + doAnswer((invocation) -> { + ActionListener responseListener = invocation.getArgument(2); + responseListener.onResponse(getDetectorsResponse); + return null; + }).when(nodeClient).execute(any(ActionType.class), any(), any()); + + tool.run(emptyParams, listener); + ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(String.class); + verify(listener, times(1)).onResponse(responseCaptor.capture()); + assertEquals(expectedResponseStr, responseCaptor.getValue()); + } + + @Test + public void testRunWithSingleAnomalyDetector() throws Exception { + final String detectorName = "detector-1"; + final String detectorId = "detector-1-id"; + Tool tool = SearchAnomalyDetectorsTool.Factory.getInstance().create(Collections.emptyMap()); + + XContentBuilder content = XContentBuilder.builder(XContentType.JSON.xContent()); + content.startObject(); + content.field("name", detectorName); + content.endObject(); + SearchHit[] hits = new SearchHit[1]; + hits[0] = new SearchHit(0, detectorId, null, null).sourceRef(BytesReference.bytes(content)); + + TotalHits totalHits = new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO); + + SearchResponse getDetectorsResponse = new SearchResponse( + new SearchResponseSections(new SearchHits(hits, totalHits, 0), new Aggregations(new ArrayList<>()), null, false, null, null, 0), + null, + 0, + 0, + 0, + 0, + null, + null + ); + String expectedResponseStr = String + .format("AnomalyDetectors=[{id=%s,name=%s}]TotalAnomalyDetectors=%d", detectorId, detectorName, hits.length); + + @SuppressWarnings("unchecked") + ActionListener listener = Mockito.mock(ActionListener.class); + + doAnswer((invocation) -> { + ActionListener responseListener = invocation.getArgument(2); + responseListener.onResponse(getDetectorsResponse); + return null; + }).when(nodeClient).execute(any(ActionType.class), any(), any()); + + tool.run(emptyParams, listener); + ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(String.class); + verify(listener, times(1)).onResponse(responseCaptor.capture()); + assertEquals(expectedResponseStr, responseCaptor.getValue()); + } + + @Test + public void testValidate() { + Tool tool = SearchAnomalyDetectorsTool.Factory.getInstance().create(Collections.emptyMap()); + assertEquals(SearchAnomalyDetectorsTool.TYPE, tool.getType()); + assertTrue(tool.validate(emptyParams)); + assertTrue(tool.validate(nonEmptyParams)); + assertTrue(tool.validate(nullParams)); + } +} From 1a399fd660da565810b902ca8809db3d124d33d4 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 19 Dec 2023 00:07:39 +0000 Subject: [PATCH 02/10] Remove dup variable Signed-off-by: Tyler Ohlsen --- build.gradle | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/build.gradle b/build.gradle index a8d9cc16..91c63eb1 100644 --- a/build.gradle +++ b/build.gradle @@ -12,14 +12,6 @@ buildscript { opensearch_version = System.getProperty("opensearch.version", "2.11.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") - version_tokens = opensearch_version.tokenize('-') - opensearch_build = version_tokens[0] + '.0' - if (buildVersionQualifier) { - opensearch_build += "-${buildVersionQualifier}" - } - if (isSnapshot) { - opensearch_build += "-SNAPSHOT" - } } repositories { @@ -113,7 +105,7 @@ dependencies { implementation("com.google.guava:guava:32.1.3-jre") implementation fileTree(dir: sqlJarDirectory, include: ["opensearch-sql-${version}.jar", "ppl-${version}.jar", "protocol-${version}.jar"]) - zipArchive "org.opensearch.plugin:opensearch-anomaly-detection:${opensearch_build}" + zipArchive "org.opensearch.plugin:opensearch-anomaly-detection:${version}" implementation fileTree(dir: adJarDirectory, include: ["opensearch-anomaly-detection-${version}.jar"]) compileOnly "org.opensearch:common-utils:${version}" @@ -127,7 +119,7 @@ dependencies { testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.10.1' // ZipArchive dependencies used for integration tests - zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${version}" } task extractSqlJar(type: Copy) { From 10cbd38300edce5c9ac1f7507052ec3c59d731ba Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 19 Dec 2023 00:33:29 +0000 Subject: [PATCH 03/10] Add initial search AD tool with basic tests Signed-off-by: Tyler Ohlsen --- build.gradle | 10 ++++++++-- .../agent/tools/SearchAnomalyDetectorsTool.java | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index 91c63eb1..2e7677b9 100644 --- a/build.gradle +++ b/build.gradle @@ -109,8 +109,13 @@ dependencies { implementation fileTree(dir: adJarDirectory, include: ["opensearch-anomaly-detection-${version}.jar"]) compileOnly "org.opensearch:common-utils:${version}" + testImplementation "org.opensearch.test:framework:${opensearch_version}" - testImplementation "org.mockito:mockito-core:5.8.0" + testImplementation group: 'junit', name: 'junit', version: '4.13.2' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.8.0' + testImplementation group: 'org.mockito', name: 'mockito-inline', version: '5.2.0' + testImplementation("net.bytebuddy:byte-buddy:1.14.7") + testImplementation("net.bytebuddy:byte-buddy-agent:1.14.7") testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.1' testImplementation 'org.mockito:mockito-junit-jupiter:5.8.0' testImplementation "com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0" @@ -146,12 +151,13 @@ testingConventions.enabled = false thirdPartyAudit.enabled = false test { - useJUnitPlatform() testLogging { exceptionFormat "full" events "skipped", "passed", "failed" // "started" showStandardStreams true } + include '**/*Tests.class' + systemProperty 'tests.security.manager', 'false' } spotless { diff --git a/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java b/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java index 845cf9e3..0462644c 100644 --- a/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java +++ b/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java @@ -23,11 +23,10 @@ import org.opensearch.ml.common.spi.tools.Parser; import org.opensearch.ml.common.spi.tools.Tool; import org.opensearch.ml.common.spi.tools.ToolAnnotation; +import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.sort.SortOrder; -import org.opensearch.search.SearchHit; - import lombok.Getter; import lombok.Setter; @@ -58,6 +57,7 @@ public class SearchAnomalyDetectorsTool implements Tool { public SearchAnomalyDetectorsTool(Client client) { this.client = client; + this.adClient = new AnomalyDetectionNodeClient(client); // probably keep this overridden output parser. need to ensure the output matches what's expected outputParser = new Parser<>() { @@ -81,7 +81,7 @@ public void run(Map parameters, ActionListener listener) final Boolean highCardinality = parameters.containsKey("highCardinality") ? Boolean.parseBoolean(parameters.get("highCardinality")) : null; - final Long lastUpdateTime = parameters.containsKey("lastUpdateTime") ? Long.parseLong(parameters.get("lastUpdateTime")): null; + final Long lastUpdateTime = parameters.containsKey("lastUpdateTime") ? Long.parseLong(parameters.get("lastUpdateTime")) : null; final String sortOrderStr = parameters.getOrDefault("sortOrder", "asc"); final SortOrder sortOrder = sortOrderStr == "asc" ? SortOrder.ASC : SortOrder.DESC; final String sortString = parameters.getOrDefault("sortString", "name.keyword"); From 01d351c57908744982d1498a848de6e3cf37a294 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 19 Dec 2023 00:58:09 +0000 Subject: [PATCH 04/10] Build/test working; run errors with jarhell Signed-off-by: Tyler Ohlsen --- build.gradle | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index 2e7677b9..6a3488e6 100644 --- a/build.gradle +++ b/build.gradle @@ -69,6 +69,7 @@ apply plugin: 'opensearch.testclusters' apply plugin: 'opensearch.pluginzip' def sqlJarDirectory = "$buildDir/dependencies/opensearch-sql-plugin" +def jsJarDirectory = "$buildDir/dependencies/opensearch-job-scheduler" def adJarDirectory = "$buildDir/dependencies/opensearch-anomaly-detection" configurations { @@ -76,7 +77,7 @@ configurations { all { resolutionStrategy { force "org.mockito:mockito-core:5.8.0" - force "com.google.guava:guava:32.1.3-jre" // CVE for 31.1 + force "com.google.guava:guava:32.1.2-jre" // CVE for 31.1 force("org.eclipse.platform:org.eclipse.core.runtime:3.30.0") // CVE for < 3.29.0, forces JDK17 for spotless } } @@ -90,6 +91,11 @@ task addJarsToClasspath(type: Copy) { } into("$buildDir/classes") + from(fileTree(dir: jsJarDirectory)) { + include "opensearch-job-scheduler-${version}.jar" + } + into("$buildDir/classes") + from(fileTree(dir: adJarDirectory)) { include "opensearch-anomaly-detection-${version}.jar" } @@ -102,11 +108,8 @@ dependencies { compileOnly "org.apache.logging.log4j:log4j-slf4j-impl:2.22.0" compileOnly group: 'org.json', name: 'json', version: '20231013' zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${version}" - implementation("com.google.guava:guava:32.1.3-jre") + implementation("com.google.guava:guava:32.1.2-jre") implementation fileTree(dir: sqlJarDirectory, include: ["opensearch-sql-${version}.jar", "ppl-${version}.jar", "protocol-${version}.jar"]) - - zipArchive "org.opensearch.plugin:opensearch-anomaly-detection:${version}" - implementation fileTree(dir: adJarDirectory, include: ["opensearch-anomaly-detection-${version}.jar"]) compileOnly "org.opensearch:common-utils:${version}" @@ -125,6 +128,10 @@ dependencies { // ZipArchive dependencies used for integration tests zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${version}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${version}" + implementation fileTree(dir: jsJarDirectory, include: ["opensearch-job-scheduler-${version}.jar"]) + zipArchive "org.opensearch.plugin:opensearch-anomaly-detection:${version}" + implementation fileTree(dir: adJarDirectory, include: ["opensearch-anomaly-detection-${version}.jar"]) } task extractSqlJar(type: Copy) { @@ -133,6 +140,12 @@ task extractSqlJar(type: Copy) { into sqlJarDirectory } +task extractJsJar(type: Copy) { + mustRunAfter() + from(zipTree(configurations.zipArchive.find { it.name.startsWith("opensearch-job-scheduler")})) + into jsJarDirectory +} + task extractAdJar(type: Copy) { mustRunAfter() from(zipTree(configurations.zipArchive.find { it.name.startsWith("opensearch-anomaly-detection")})) @@ -140,6 +153,7 @@ task extractAdJar(type: Copy) { } tasks.addJarsToClasspath.dependsOn(extractSqlJar) +tasks.addJarsToClasspath.dependsOn(extractJsJar) tasks.addJarsToClasspath.dependsOn(extractAdJar) project.tasks.delombok.dependsOn(addJarsToClasspath) tasks.publishNebulaPublicationToMavenLocal.dependsOn ':generatePomFileForPluginZipPublication' @@ -176,6 +190,7 @@ spotless { compileJava { dependsOn extractSqlJar + dependsOn extractJsJar dependsOn extractAdJar dependsOn delombok options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) @@ -192,7 +207,7 @@ opensearchplugin { name 'skills' description 'OpenSearch Skills' classname 'org.opensearch.agent.ToolPlugin' - extendedPlugins = ['opensearch-ml'] + extendedPlugins = ['opensearch-ml', 'opensearch-job-scheduler'] licenseFile rootProject.file("LICENSE.txt") noticeFile rootProject.file("NOTICE") } @@ -316,6 +331,13 @@ testClusters.integTest { debugPort += 1 } } + + // nodes.each { node -> + // def plugins = node.plugins + // def firstPlugin = plugins.get(0) + // plugins.remove(0) + // plugins.add(firstPlugin) + // } } // Automatically sets up the integration test cluster locally From 75d483799d3c1725ad6123148d38d1eea44121e4 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 19 Dec 2023 19:18:10 +0000 Subject: [PATCH 05/10] Clean up build.gradle Signed-off-by: Tyler Ohlsen --- build.gradle | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/build.gradle b/build.gradle index 6a3488e6..c867dc62 100644 --- a/build.gradle +++ b/build.gradle @@ -103,16 +103,26 @@ task addJarsToClasspath(type: Copy) { } dependencies { - compileOnly group: 'org.opensearch', name:'opensearch-ml-client', version: "${version}" + // 3P dependencies compileOnly group: 'com.google.code.gson', name: 'gson', version: '2.10.1' compileOnly "org.apache.logging.log4j:log4j-slf4j-impl:2.22.0" compileOnly group: 'org.json', name: 'json', version: '20231013' - zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${version}" implementation("com.google.guava:guava:32.1.2-jre") - implementation fileTree(dir: sqlJarDirectory, include: ["opensearch-sql-${version}.jar", "ppl-${version}.jar", "protocol-${version}.jar"]) + // Plugin dependencies + compileOnly group: 'org.opensearch', name:'opensearch-ml-client', version: "${version}" + implementation fileTree(dir: jsJarDirectory, include: ["opensearch-job-scheduler-${version}.jar"]) + implementation fileTree(dir: adJarDirectory, include: ["opensearch-anomaly-detection-${version}.jar"]) + implementation fileTree(dir: sqlJarDirectory, include: ["opensearch-sql-${version}.jar", "ppl-${version}.jar", "protocol-${version}.jar"]) compileOnly "org.opensearch:common-utils:${version}" + // ZipArchive dependencies used for integration tests + zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${version}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${version}" + zipArchive "org.opensearch.plugin:opensearch-anomaly-detection:${version}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${version}" + + // Test dependencies testImplementation "org.opensearch.test:framework:${opensearch_version}" testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.8.0' @@ -125,13 +135,6 @@ dependencies { testImplementation "com.cronutils:cron-utils:9.2.1" testImplementation "commons-validator:commons-validator:1.8.0" testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.10.1' - - // ZipArchive dependencies used for integration tests - zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${version}" - zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${version}" - implementation fileTree(dir: jsJarDirectory, include: ["opensearch-job-scheduler-${version}.jar"]) - zipArchive "org.opensearch.plugin:opensearch-anomaly-detection:${version}" - implementation fileTree(dir: adJarDirectory, include: ["opensearch-anomaly-detection-${version}.jar"]) } task extractSqlJar(type: Copy) { @@ -207,7 +210,7 @@ opensearchplugin { name 'skills' description 'OpenSearch Skills' classname 'org.opensearch.agent.ToolPlugin' - extendedPlugins = ['opensearch-ml', 'opensearch-job-scheduler'] + extendedPlugins = ['opensearch-ml'] licenseFile rootProject.file("LICENSE.txt") noticeFile rootProject.file("NOTICE") } @@ -331,13 +334,6 @@ testClusters.integTest { debugPort += 1 } } - - // nodes.each { node -> - // def plugins = node.plugins - // def firstPlugin = plugins.get(0) - // plugins.remove(0) - // plugins.add(firstPlugin) - // } } // Automatically sets up the integration test cluster locally From 7ab84c6dbc667a3d2e7567f3874652e423006c61 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 19 Dec 2023 19:18:32 +0000 Subject: [PATCH 06/10] Revert to 3.0 Signed-off-by: Tyler Ohlsen --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index c867dc62..7b4f41a3 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ import java.util.concurrent.Callable buildscript { ext { opensearch_group = "org.opensearch" - opensearch_version = System.getProperty("opensearch.version", "2.11.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") } From 8c37fbab221a9a5d8f4f01d19cd08bfd21a37db4 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 20 Dec 2023 17:41:11 +0000 Subject: [PATCH 07/10] Address comments Signed-off-by: Tyler Ohlsen --- build.gradle | 2 +- .../opensearch/agent/tools/SearchAnomalyDetectorsTool.java | 5 ++--- .../agent/tools/SearchAnomalyDetectorsToolTests.java | 3 ++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index 7b4f41a3..42ef9594 100644 --- a/build.gradle +++ b/build.gradle @@ -77,7 +77,7 @@ configurations { all { resolutionStrategy { force "org.mockito:mockito-core:5.8.0" - force "com.google.guava:guava:32.1.2-jre" // CVE for 31.1 + force "com.google.guava:guava:32.1.3-jre" // CVE for 31.1 force("org.eclipse.platform:org.eclipse.core.runtime:3.30.0") // CVE for < 3.29.0, forces JDK17 for spotless } } diff --git a/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java b/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java index 0462644c..c5a52025 100644 --- a/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java +++ b/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java @@ -41,8 +41,7 @@ public class SearchAnomalyDetectorsTool implements Tool { @Getter @Setter private String description = DEFAULT_DESCRIPTION; - @Getter - private String type; + @Getter private String version; @@ -83,7 +82,7 @@ public void run(Map parameters, ActionListener listener) : null; final Long lastUpdateTime = parameters.containsKey("lastUpdateTime") ? Long.parseLong(parameters.get("lastUpdateTime")) : null; final String sortOrderStr = parameters.getOrDefault("sortOrder", "asc"); - final SortOrder sortOrder = sortOrderStr == "asc" ? SortOrder.ASC : SortOrder.DESC; + final SortOrder sortOrder = sortOrderStr.equalsIgnoreCase("asc") ? SortOrder.ASC : SortOrder.DESC; final String sortString = parameters.getOrDefault("sortString", "name.keyword"); final int size = parameters.containsKey("size") ? Integer.parseInt(parameters.get("size")) : 20; final int startIndex = parameters.containsKey("startIndex") ? Integer.parseInt(parameters.get("startIndex")) : 0; diff --git a/src/test/java/org/opensearch/agent/tools/SearchAnomalyDetectorsToolTests.java b/src/test/java/org/opensearch/agent/tools/SearchAnomalyDetectorsToolTests.java index 96b05364..37ff02a1 100644 --- a/src/test/java/org/opensearch/agent/tools/SearchAnomalyDetectorsToolTests.java +++ b/src/test/java/org/opensearch/agent/tools/SearchAnomalyDetectorsToolTests.java @@ -14,6 +14,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.Locale; import java.util.Map; import org.apache.lucene.search.TotalHits; @@ -81,7 +82,7 @@ public void testRunWithNoDetectors() throws Exception { null, null ); - String expectedResponseStr = String.format("AnomalyDetectors=[]TotalAnomalyDetectors=%d", hits.length); + String expectedResponseStr = String.format(Locale.getDefault(), "AnomalyDetectors=[]TotalAnomalyDetectors=%d", hits.length); @SuppressWarnings("unchecked") ActionListener listener = Mockito.mock(ActionListener.class); From eb9f128fce5e365bbd71ace8d41bfe8f2f8175eb Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 21 Dec 2023 18:17:07 +0000 Subject: [PATCH 08/10] Revert guava dep change Signed-off-by: Tyler Ohlsen --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 42ef9594..577e349d 100644 --- a/build.gradle +++ b/build.gradle @@ -107,7 +107,7 @@ dependencies { compileOnly group: 'com.google.code.gson', name: 'gson', version: '2.10.1' compileOnly "org.apache.logging.log4j:log4j-slf4j-impl:2.22.0" compileOnly group: 'org.json', name: 'json', version: '20231013' - implementation("com.google.guava:guava:32.1.2-jre") + implementation("com.google.guava:guava:32.1.3-jre") // Plugin dependencies compileOnly group: 'org.opensearch', name:'opensearch-ml-client', version: "${version}" From 27c638f359d3ff7b9f429e89806c0621f125975e Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 21 Dec 2023 21:16:57 +0000 Subject: [PATCH 09/10] Change to time-series-analytics for plugin dep Signed-off-by: Tyler Ohlsen --- build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 577e349d..9cf0adb5 100644 --- a/build.gradle +++ b/build.gradle @@ -70,7 +70,7 @@ apply plugin: 'opensearch.pluginzip' def sqlJarDirectory = "$buildDir/dependencies/opensearch-sql-plugin" def jsJarDirectory = "$buildDir/dependencies/opensearch-job-scheduler" -def adJarDirectory = "$buildDir/dependencies/opensearch-anomaly-detection" +def adJarDirectory = "$buildDir/dependencies/opensearch-time-series-analytics" configurations { zipArchive @@ -97,7 +97,7 @@ task addJarsToClasspath(type: Copy) { into("$buildDir/classes") from(fileTree(dir: adJarDirectory)) { - include "opensearch-anomaly-detection-${version}.jar" + include "opensearch-time-series-analytics-${version}.jar" } into("$buildDir/classes") } @@ -112,7 +112,7 @@ dependencies { // Plugin dependencies compileOnly group: 'org.opensearch', name:'opensearch-ml-client', version: "${version}" implementation fileTree(dir: jsJarDirectory, include: ["opensearch-job-scheduler-${version}.jar"]) - implementation fileTree(dir: adJarDirectory, include: ["opensearch-anomaly-detection-${version}.jar"]) + implementation fileTree(dir: adJarDirectory, include: ["opensearch-time-series-analytics-${version}.jar"]) implementation fileTree(dir: sqlJarDirectory, include: ["opensearch-sql-${version}.jar", "ppl-${version}.jar", "protocol-${version}.jar"]) compileOnly "org.opensearch:common-utils:${version}" From 00f35b066f5a3059b8b182ede78d40185a86779e Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Fri, 22 Dec 2023 03:03:18 +0000 Subject: [PATCH 10/10] Add null/invalid check on lastUpdateTime Signed-off-by: Tyler Ohlsen --- build.gradle | 1 + .../opensearch/agent/tools/SearchAnomalyDetectorsTool.java | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 9cf0adb5..826d94ea 100644 --- a/build.gradle +++ b/build.gradle @@ -108,6 +108,7 @@ dependencies { compileOnly "org.apache.logging.log4j:log4j-slf4j-impl:2.22.0" compileOnly group: 'org.json', name: 'json', version: '20231013' implementation("com.google.guava:guava:32.1.3-jre") + implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.13.0' // Plugin dependencies compileOnly group: 'org.opensearch', name:'opensearch-ml-client', version: "${version}" diff --git a/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java b/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java index c5a52025..f8f43bb0 100644 --- a/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java +++ b/src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java @@ -9,6 +9,7 @@ import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.ad.client.AnomalyDetectionNodeClient; @@ -80,7 +81,8 @@ public void run(Map parameters, ActionListener listener) final Boolean highCardinality = parameters.containsKey("highCardinality") ? Boolean.parseBoolean(parameters.get("highCardinality")) : null; - final Long lastUpdateTime = parameters.containsKey("lastUpdateTime") ? Long.parseLong(parameters.get("lastUpdateTime")) : null; + final Long lastUpdateTime = parameters.containsKey("lastUpdateTime") && StringUtils.isNumeric(parameters.get("lastUpdateTime")) + ? Long.parseLong(parameters.get("lastUpdateTime")) : null; final String sortOrderStr = parameters.getOrDefault("sortOrder", "asc"); final SortOrder sortOrder = sortOrderStr.equalsIgnoreCase("asc") ? SortOrder.ASC : SortOrder.DESC; final String sortString = parameters.getOrDefault("sortString", "name.keyword");