From fecb842be194a16d1762a11822855be9c20ce744 Mon Sep 17 00:00:00 2001 From: David Venable Date: Tue, 22 Aug 2023 13:47:29 -0700 Subject: [PATCH] Normalize the include/exclude keys in the JacksonEvent implementation in order to fix a problem where the ndjson codec was not correctly including/excluding keys. (#3209) Signed-off-by: David Venable --- .../dataprepper/model/event/JacksonEvent.java | 3 +- .../model/event/JacksonEventTest.java | 28 +++++++++---------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java index d0ce16de14..389a73db31 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java @@ -607,11 +607,12 @@ String searchAndFilter(JsonNode node, String path, final List filterKeys List valueList = new ArrayList<>(); node.properties().forEach(entry -> { - String keyPath = path + SEPARATOR + entry.getKey(); + String keyPath = trimKey(path + SEPARATOR + entry.getKey()); // Track whether the key is found in the filter list. // Different behaviours between include and exclude action. boolean found = false; for (String key : filterKeys) { + key = trimKey(key); if (keyPath.equals(key)) { found = true; // To keep the order. diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java index 9d75a2253f..7b2dd728da 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java @@ -691,37 +691,37 @@ void testJsonStringBuilderWithIncludeKeys() { .build(); // Include Keys must start with / and also ordered, This is pre-processed in SinkModel - List includeKeys1 = Arrays.asList("/foo", "/info"); + List includeKeys1 = Arrays.asList("foo", "info"); final String expectedJsonString1 = "{\"foo\":\"bar\",\"info\":{\"name\":\"hello\",\"foo\":\"bar\"}}"; assertThat(event.jsonBuilder().rootKey(null).includeKeys(includeKeys1).toJsonString(), equalTo(expectedJsonString1)); // Test child node - List includeKeys2 = Arrays.asList("/foo", "/info/name"); + List includeKeys2 = Arrays.asList("foo", "info/name"); final String expectedJsonString2 = "{\"foo\":\"bar\",\"info\":{\"name\":\"hello\"}}"; assertThat(event.jsonBuilder().includeKeys(includeKeys2).toJsonString(), equalTo(expectedJsonString2)); // Test array node. - List includeKeys3 = Arrays.asList("/foo", "/tags/key"); + List includeKeys3 = Arrays.asList("foo", "tags/key"); final String expectedJsonString3 = "{\"foo\":\"bar\",\"tags\":[{\"key\":\"a\"},{\"key\":\"c\"}]}"; assertThat(event.jsonBuilder().includeKeys(includeKeys3).toJsonString(), equalTo(expectedJsonString3)); // Test some keys not found - List includeKeys4 = Arrays.asList("/foo", "/info/age"); + List includeKeys4 = Arrays.asList("foo", "info/age"); final String expectedJsonString4 = "{\"foo\":\"bar\",\"info\":{}}"; assertThat(event.jsonBuilder().includeKeys(includeKeys4).toJsonString(), equalTo(expectedJsonString4)); // Test all keys not found - List includeKeys5 = List.of("/hello"); + List includeKeys5 = List.of("/ello"); final String expectedJsonString5 = "{}"; assertThat(event.jsonBuilder().includeKeys(includeKeys5).toJsonString(), equalTo(expectedJsonString5)); // Test working with root node - List includeKeys6 = List.of("/name"); + List includeKeys6 = List.of("name"); final String expectedJsonString6 = "{\"name\":\"hello\"}"; assertThat(event.jsonBuilder().rootKey("info").includeKeys(includeKeys6).toJsonString(), equalTo(expectedJsonString6)); // Test working with unknown root node - List includeKeys7 = List.of("/name"); + List includeKeys7 = List.of("name"); final String expectedJsonString7 = "{}"; assertThat(event.jsonBuilder().rootKey("hello").includeKeys(includeKeys7).toJsonString(), equalTo(expectedJsonString7)); @@ -738,37 +738,37 @@ void testJsonStringBuilderWithExcludeKeys() { .build(); // Include Keys must start with / and also ordered, This is pre-processed in SinkModel - List excludeKeys1 = Arrays.asList("/foo", "/info"); + List excludeKeys1 = Arrays.asList("foo", "info"); final String expectedJsonString1 = "{\"id\":1,\"tags\":[{\"key\":\"a\",\"value\":\"b\"},{\"key\":\"c\",\"value\":\"d\"}]}"; assertThat(event.jsonBuilder().rootKey(null).excludeKeys(excludeKeys1).toJsonString(), equalTo(expectedJsonString1)); // Test child node - List excludeKeys2 = Arrays.asList("/foo", "/info/name"); + List excludeKeys2 = Arrays.asList("foo", "info/name"); final String expectedJsonString2 = "{\"id\":1,\"info\":{\"foo\":\"bar\"},\"tags\":[{\"key\":\"a\",\"value\":\"b\"},{\"key\":\"c\",\"value\":\"d\"}]}"; assertThat(event.jsonBuilder().excludeKeys(excludeKeys2).toJsonString(), equalTo(expectedJsonString2)); // Test array node. - List excludeKeys3 = Arrays.asList("/foo", "/tags/key"); + List excludeKeys3 = Arrays.asList("foo", "tags/key"); final String expectedJsonString3 = "{\"id\":1,\"info\":{\"name\":\"hello\",\"foo\":\"bar\"},\"tags\":[{\"value\":\"b\"},{\"value\":\"d\"}]}"; assertThat(event.jsonBuilder().excludeKeys(excludeKeys3).toJsonString(), equalTo(expectedJsonString3)); // Test some keys not found - List excludeKeys4 = Arrays.asList("/foo", "/info/age"); + List excludeKeys4 = Arrays.asList("foo", "info/age"); final String expectedJsonString4 = "{\"id\":1,\"info\":{\"name\":\"hello\",\"foo\":\"bar\"},\"tags\":[{\"key\":\"a\",\"value\":\"b\"},{\"key\":\"c\",\"value\":\"d\"}]}"; assertThat(event.jsonBuilder().excludeKeys(excludeKeys4).toJsonString(), equalTo(expectedJsonString4)); // Test all keys not found - List excludeKeys5 = List.of("/hello"); + List excludeKeys5 = List.of("hello"); final String expectedJsonString5 = "{\"id\":1,\"foo\":\"bar\",\"info\":{\"name\":\"hello\",\"foo\":\"bar\"},\"tags\":[{\"key\":\"a\",\"value\":\"b\"},{\"key\":\"c\",\"value\":\"d\"}]}"; assertThat(event.jsonBuilder().excludeKeys(excludeKeys5).toJsonString(), equalTo(expectedJsonString5)); // Test working with root node - List excludeKeys6 = List.of("/name"); + List excludeKeys6 = List.of("name"); final String expectedJsonString6 = "{\"foo\":\"bar\"}"; assertThat(event.jsonBuilder().rootKey("info").excludeKeys(excludeKeys6).toJsonString(), equalTo(expectedJsonString6)); // Test working with unknown root node - List includeKeys7 = List.of("/name"); + List includeKeys7 = List.of("name"); final String expectedJsonString7 = "{}"; assertThat(event.jsonBuilder().rootKey("hello").includeKeys(includeKeys7).toJsonString(), equalTo(expectedJsonString7));