Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add queryFieldNames field in Doc Level Queries (#582) (#597) #614

Closed
wants to merge 2 commits into from
Closed

add queryFieldNames field in Doc Level Queries (#582) (#597) #614

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7beca26
add queryFieldNames field in Doc Level Queries (#582) (#597)
goyamegh Mar 15, 2024
1e7b37e
Fix merge conflicts
goyamegh Mar 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 61 additions & 4 deletions src/main/kotlin/org/opensearch/commons/alerting/model/DocLevelQuery.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@ import java.util.UUID
data class DocLevelQuery(
val id: String = UUID.randomUUID().toString(),
val name: String,
val fields: List<String>,
val query: String,
val tags: List<String> = mutableListOf()
val tags: List<String> = mutableListOf(),
val queryFieldNames: List<String> = mutableListOf()
) : BaseModel {

init {
Expand All @@ -30,42 +32,52 @@ data class DocLevelQuery(
constructor(sin: StreamInput) : this(
sin.readString(), // id
sin.readString(), // name
sin.readStringList(), // fields
sin.readString(), // query
sin.readStringList() // tags
sin.readStringList(), // tags,
sin.readStringList() // fieldsBeingQueried
)

fun asTemplateArg(): Map<String, Any> {
return mapOf(
QUERY_ID_FIELD to id,
NAME_FIELD to name,
FIELDS_FIELD to fields,
QUERY_FIELD to query,
TAGS_FIELD to tags
TAGS_FIELD to tags,
QUERY_FIELD_NAMES_FIELD to queryFieldNames
)
}

@Throws(IOException::class)
override fun writeTo(out: StreamOutput) {
out.writeString(id)
out.writeString(name)
out.writeStringCollection(fields)
out.writeString(query)
out.writeStringCollection(tags)
out.writeStringCollection(queryFieldNames)
}

override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder {
builder.startObject()
.field(QUERY_ID_FIELD, id)
.field(NAME_FIELD, name)
.field(FIELDS_FIELD, fields.toTypedArray())
.field(QUERY_FIELD, query)
.field(TAGS_FIELD, tags.toTypedArray())
.field(QUERY_FIELD_NAMES_FIELD, queryFieldNames.toTypedArray())
.endObject()
return builder
}

companion object {
const val QUERY_ID_FIELD = "id"
const val NAME_FIELD = "name"
const val FIELDS_FIELD = "fields"
const val QUERY_FIELD = "query"
const val TAGS_FIELD = "tags"
const val QUERY_FIELD_NAMES_FIELD = "query_field_names"
const val NO_ID = ""
val INVALID_CHARACTERS: List<String> = listOf(" ", "[", "]", "{", "}", "(", ")")

Expand All @@ -76,6 +88,8 @@ data class DocLevelQuery(
lateinit var query: String
lateinit var name: String
val tags: MutableList<String> = mutableListOf()
val fields: MutableList<String> = mutableListOf()
val queryFieldNames: MutableList<String> = mutableListOf()

XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp)
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
Expand All @@ -88,6 +102,7 @@ data class DocLevelQuery(
name = xcp.text()
validateQuery(name)
}

QUERY_FIELD -> query = xcp.text()
TAGS_FIELD -> {
XContentParserUtils.ensureExpectedToken(
Expand All @@ -101,14 +116,40 @@ data class DocLevelQuery(
tags.add(tag)
}
}

FIELDS_FIELD -> {
XContentParserUtils.ensureExpectedToken(
XContentParser.Token.START_ARRAY,
xcp.currentToken(),
xcp
)
while (xcp.nextToken() != XContentParser.Token.END_ARRAY) {
val field = xcp.text()
fields.add(field)
}
}

QUERY_FIELD_NAMES_FIELD -> {
XContentParserUtils.ensureExpectedToken(
XContentParser.Token.START_ARRAY,
xcp.currentToken(),
xcp
)
while (xcp.nextToken() != XContentParser.Token.END_ARRAY) {
val field = xcp.text()
queryFieldNames.add(field)
}
}
}
}

return DocLevelQuery(
id = id,
name = name,
fields = fields,
query = query,
tags = tags
tags = tags,
queryFieldNames = queryFieldNames
)
}

Expand All @@ -129,4 +170,20 @@ data class DocLevelQuery(
}
}
}

// constructor for java plugins' convenience to optionally avoid passing empty list for 'fieldsBeingQueried' field
constructor(
id: String,
name: String,
fields: MutableList<String>,
query: String,
tags: MutableList<String>
) : this(
id = id,
name = name,
fields = fields,
query = query,
tags = tags,
queryFieldNames = emptyList()
)
}
2 changes: 1 addition & 1 deletion src/test/kotlin/org/opensearch/commons/alerting/TestHelpers.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ fun randomDocLevelQuery(
name: String = "${RandomNumbers.randomIntBetween(Random(), 0, 5)}",
tags: List<String> = mutableListOf(0..RandomNumbers.randomIntBetween(Random(), 0, 10)).map { RandomStrings.randomAsciiLettersOfLength(Random(), 10) }
): DocLevelQuery {
return DocLevelQuery(id = id, query = query, name = name, tags = tags)
return DocLevelQuery(id = id, query = query, name = name, tags = tags, fields = listOf("*"))
}

fun randomDocLevelMonitorInput(
Expand Down
4 changes: 2 additions & 2 deletions src/test/kotlin/org/opensearch/commons/alerting/action/GetFindingsResponseTests.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ internal class GetFindingsResponseTests {
"monitor_id1",
"monitor_name1",
"test_index1",
listOf(DocLevelQuery("1", "myQuery", "fieldA:valABC", List.of())),
listOf(DocLevelQuery("1", "myQuery", listOf(), "fieldA:valABC", List.of(), List.of("*"))),
Instant.now()
)
val findingDocument1 = FindingDocument("test_index1", "doc1", true, "document 1 payload")
Expand All @@ -43,7 +43,7 @@ internal class GetFindingsResponseTests {
"monitor_id2",
"monitor_name2",
"test_index2",
listOf(DocLevelQuery("1", "myQuery", "fieldA:valABC", List.of())),
listOf(DocLevelQuery("1", "myQuery", listOf(), "fieldA:valABC", List.of())),
Instant.now()
)
val findingDocument21 = FindingDocument("test_index2", "doc21", true, "document 21 payload")
Expand Down
25 changes: 25 additions & 0 deletions src/test/kotlin/org/opensearch/commons/alerting/model/WriteableTests.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import org.opensearch.commons.alerting.model.action.Throttle
import org.opensearch.commons.alerting.randomAction
import org.opensearch.commons.alerting.randomActionExecutionPolicy
import org.opensearch.commons.alerting.randomBucketLevelTrigger
import org.opensearch.commons.alerting.randomDocLevelQuery
import org.opensearch.commons.alerting.randomDocumentLevelTrigger
import org.opensearch.commons.alerting.randomQueryLevelMonitor
import org.opensearch.commons.alerting.randomQueryLevelTrigger
Expand All @@ -18,6 +19,7 @@ import org.opensearch.commons.alerting.randomUser
import org.opensearch.commons.alerting.randomUserEmpty
import org.opensearch.commons.authuser.User
import org.opensearch.search.builder.SearchSourceBuilder
import kotlin.test.assertTrue

class WriteableTests {

Expand Down Expand Up @@ -111,6 +113,29 @@ class WriteableTests {
Assertions.assertEquals(trigger, newTrigger, "Round tripping DocumentLevelTrigger doesn't work")
}

@Test
fun `test doc-level query as stream`() {
val dlq = randomDocLevelQuery()
val out = BytesStreamOutput()
dlq.writeTo(out)
val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes)
val newDlq = DocLevelQuery.readFrom(sin)
Assertions.assertEquals(dlq, newDlq, "Round tripping DocLevelQuery doesn't work")
assertTrue(newDlq.queryFieldNames.isEmpty())
}

@Test
fun `test doc-level query with query Field Names as stream`() {
val dlq = randomDocLevelQuery().copy(queryFieldNames = listOf("f1", "f2"))
val out = BytesStreamOutput()
dlq.writeTo(out)
val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes)
val newDlq = DocLevelQuery.readFrom(sin)
assertTrue(newDlq.queryFieldNames.contains(dlq.queryFieldNames[0]))
assertTrue(newDlq.queryFieldNames.contains(dlq.queryFieldNames[1]))
Assertions.assertEquals(dlq, newDlq, "Round tripping DocLevelQuery doesn't work")
}

@Test
fun `test searchinput as stream`() {
val input = SearchInput(emptyList(), SearchSourceBuilder())
Expand Down
78 changes: 55 additions & 23 deletions src/test/kotlin/org/opensearch/commons/alerting/model/XContentTests.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,15 +95,27 @@ class XContentTests {
val throttleString = throttle.toXContent(builder(), ToXContent.EMPTY_PARAMS).string()
val wrongThrottleString = throttleString.replace("MINUTES", "wrongunit")

assertFailsWith<IllegalArgumentException>("Only support MINUTES throttle unit") { Throttle.parse(parser(wrongThrottleString)) }
assertFailsWith<IllegalArgumentException>("Only support MINUTES throttle unit") {
Throttle.parse(
parser(
wrongThrottleString
)
)
}
}

@Test
fun `test throttle parsing with negative value`() {
val throttle = randomThrottle().copy(value = -1)
val throttleString = throttle.toXContent(builder(), ToXContent.EMPTY_PARAMS).string()

assertFailsWith<IllegalArgumentException>("Can only set positive throttle period") { Throttle.parse(parser(throttleString)) }
assertFailsWith<IllegalArgumentException>("Can only set positive throttle period") {
Throttle.parse(
parser(
throttleString
)
)
}
}

fun `test query-level monitor parsing`() {
Expand Down Expand Up @@ -131,7 +143,13 @@ class XContentTests {
}
""".trimIndent()

assertFailsWith<IllegalArgumentException>("Monitor name is null") { Monitor.parse(parser(monitorStringWithoutName)) }
assertFailsWith<IllegalArgumentException>("Monitor name is null") {
Monitor.parse(
parser(
monitorStringWithoutName
)
)
}
}

@Test
Expand Down Expand Up @@ -358,6 +376,30 @@ class XContentTests {
)
}

@Test
fun `test doc level query toXcontent`() {
val dlq = DocLevelQuery("id", "name", listOf("f1", "f2"), "query", listOf("t1", "t2"))
val dlqString = dlq.toXContent(builder(), ToXContent.EMPTY_PARAMS).string()
val parsedDlq = DocLevelQuery.parse(parser(dlqString))
Assertions.assertEquals(
dlq,
parsedDlq,
"Round tripping Doc level query doesn't work"
)
}

@Test
fun `test doc level query toXcontent with query field names`() {
val dlq = DocLevelQuery("id", "name", listOf("f1", "f2"), "query", listOf("t1", "t2"), listOf("f1", "f2"))
val dlqString = dlq.toXContent(builder(), ToXContent.EMPTY_PARAMS).string()
val parsedDlq = DocLevelQuery.parse(parser(dlqString))
Assertions.assertEquals(
dlq,
parsedDlq,
"Round tripping Doc level query doesn't work"
)
}

@Test
fun `test alert parsing`() {
val alert = randomAlert()
Expand All @@ -368,39 +410,29 @@ class XContentTests {
assertEquals("Round tripping alert doesn't work", alert, parsedAlert)
}

@Test
fun `test alert parsing with noop trigger`() {
val monitor = randomQueryLevelMonitor()
val alert = Alert(
monitor = monitor,
trigger = NoOpTrigger(),
startTime = Instant.now().truncatedTo(ChronoUnit.MILLIS),
errorMessage = "some error",
lastNotificationTime = Instant.now()
)
assertEquals("Round tripping alert doesn't work", alert.triggerName, "NoOp trigger")
}

@Test
fun `test alert parsing without user`() {
val alertStr = "{\"id\":\"\",\"version\":-1,\"monitor_id\":\"\",\"schema_version\":0,\"monitor_version\":1," +
"\"monitor_name\":\"ARahqfRaJG\",\"trigger_id\":\"fhe1-XQBySl0wQKDBkOG\",\"trigger_name\":\"ffELMuhlro\"," +
"\"state\":\"ACTIVE\",\"error_message\":null,\"alert_history\":[],\"severity\":\"1\",\"action_execution_results\"" +
":[{\"action_id\":\"ghe1-XQBySl0wQKDBkOG\",\"last_execution_time\":1601917224583,\"throttled_count\":-1478015168}," +
"{\"action_id\":\"gxe1-XQBySl0wQKDBkOH\",\"last_execution_time\":1601917224583,\"throttled_count\":-768533744}]," +
"\"start_time\":1601917224599,\"last_notification_time\":null,\"end_time\":null,\"acknowledged_time\":null}"
"\"start_time\":1601917224599,\"last_notification_time\":null,\"end_time\":null,\"acknowledged_time\":null," +
"\"clusters\":[\"cluster-1\",\"cluster-2\"]}"
val parsedAlert = Alert.parse(parser(alertStr))
OpenSearchTestCase.assertNull(parsedAlert.monitorUser)
}

@Test
fun `test alert parsing with user as null`() {
val alertStr = "{\"id\":\"\",\"version\":-1,\"monitor_id\":\"\",\"schema_version\":0,\"monitor_version\":1,\"monitor_user\":null," +
"\"monitor_name\":\"ARahqfRaJG\",\"trigger_id\":\"fhe1-XQBySl0wQKDBkOG\",\"trigger_name\":\"ffELMuhlro\"," +
"\"state\":\"ACTIVE\",\"error_message\":null,\"alert_history\":[],\"severity\":\"1\",\"action_execution_results\"" +
":[{\"action_id\":\"ghe1-XQBySl0wQKDBkOG\",\"last_execution_time\":1601917224583,\"throttled_count\":-1478015168}," +
"{\"action_id\":\"gxe1-XQBySl0wQKDBkOH\",\"last_execution_time\":1601917224583,\"throttled_count\":-768533744}]," +
"\"start_time\":1601917224599,\"last_notification_time\":null,\"end_time\":null,\"acknowledged_time\":null}"
val alertStr =
"{\"id\":\"\",\"version\":-1,\"monitor_id\":\"\",\"schema_version\":0,\"monitor_version\":1,\"monitor_user\":null," +
"\"monitor_name\":\"ARahqfRaJG\",\"trigger_id\":\"fhe1-XQBySl0wQKDBkOG\",\"trigger_name\":\"ffELMuhlro\"," +
"\"state\":\"ACTIVE\",\"error_message\":null,\"alert_history\":[],\"severity\":\"1\",\"action_execution_results\"" +
":[{\"action_id\":\"ghe1-XQBySl0wQKDBkOG\",\"last_execution_time\":1601917224583,\"throttled_count\":-1478015168}," +
"{\"action_id\":\"gxe1-XQBySl0wQKDBkOH\",\"last_execution_time\":1601917224583,\"throttled_count\":-768533744}]," +
"\"start_time\":1601917224599,\"last_notification_time\":null,\"end_time\":null,\"acknowledged_time\":null," +
"\"clusters\":[\"cluster-1\",\"cluster-2\"]}"
val parsedAlert = Alert.parse(parser(alertStr))
OpenSearchTestCase.assertNull(parsedAlert.monitorUser)
}
Expand Down
Loading