Remove brackets feature option (#3035)

*add remove brackets feature option Signed-off-by: Kat Shen <[email protected]> --------- Signed-off-by: Kat Shen <[email protected]> Co-authored-by: Kat Shen <[email protected]>
opensearch-project · Jul 24, 2023 · 03c4819 · 03c4819
1 parent b003b08
commit 03c4819
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 3 deletions.
diff --git a/data-prepper-plugins/key-value-processor/README.md b/data-prepper-plugins/key-value-processor/README.md
@@ -74,6 +74,11 @@ When run, the processor will parse the message into the following output:
   * Default: `false`
   * Example: `skip_duplicate_values` is `false`. `{"key1=value1&key1=value1"}` will parse into `{"key1": ["value1", "value1"]}`
   * Example: `skip_duplicate_values` is `true`. `{"key1=value1&key1=value1"}` will parse into `{"key1": "value1"}`
+* `remove_brackets` - Specify whether to treat square brackets, angle brackets, and parentheses as value "wrappers" that should be removed from the value.
+  * Default: `false`
+  * Example: `remove_brackets` is `true`. `{"key1=(value1)"}` will parse into `{"key1": value1}`
+  * Example: `remove_brackets` is `false`. `{"key1=(value1)"}` will parse into `{"key1": "(value1)"}`
+  * In the case of a key-value pair with a brackets and a split character, the splitting will take priority over `remove_brackets=true`. `{key1=(value1&value2)}` will parse into `{"key1":"value1","value2)":null}`
 
 ## Developer Guide
 This plugin is compatible with Java 14. See

diff --git a/...rc/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java b/...rc/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java
@@ -111,10 +111,16 @@ public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProces
             throw new IllegalArgumentException(String.format("The whitespace value: %s is not a valid option", keyValueProcessorConfig.getWhitespace()));
         }
 
-        final Pattern duplicateValueBoolCheck = Pattern.compile("true|false", Pattern.CASE_INSENSITIVE);
-        final Matcher duplicateValueBoolMatch = duplicateValueBoolCheck.matcher(String.valueOf(keyValueProcessorConfig.getSkipDuplicateValues()));
+        final Pattern boolCheck = Pattern.compile("true|false", Pattern.CASE_INSENSITIVE);
+        final Matcher duplicateValueBoolMatch = boolCheck.matcher(String.valueOf(keyValueProcessorConfig.getSkipDuplicateValues()));
+        final Matcher removeBracketsBoolMatch = boolCheck.matcher(String.valueOf(keyValueProcessorConfig.getRemoveBrackets()));
+
         if (!duplicateValueBoolMatch.matches()) {
-            throw new IllegalArgumentException(String.format("The skip_duplicate_values value: %s is not a valid option", keyValueProcessorConfig.getSkipDuplicateValues()));
+            throw new IllegalArgumentException(String.format("The skip_duplicate_values value must be either true or false", keyValueProcessorConfig.getSkipDuplicateValues()));
+        }
+
+        if (!removeBracketsBoolMatch.matches()) {
+            throw new IllegalArgumentException(String.format("The remove_brackets value must be either true or false", keyValueProcessorConfig.getRemoveBrackets()));
         }
     }
 
@@ -195,6 +201,13 @@ public Collection<Record<Event>> doExecute(final Collection<Record<Event>> recor
                     key = transformKey(key);
                 }
 
+                if (keyValueProcessorConfig.getRemoveBrackets()) {
+                    final String bracketRegex = "[\\[\\]()<>]";
+                    if (value != null) {
+                        value = value.toString().replaceAll(bracketRegex,"");
+                    }
+                }
+
                 addKeyValueToMap(parsedMap, key, value);
             }
 

diff --git a/...n/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java b/...n/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java
@@ -25,6 +25,7 @@ public class KeyValueProcessorConfig {
     static final String DEFAULT_TRANSFORM_KEY = "";
     static final String DEFAULT_WHITESPACE = "lenient";
     static final boolean DEFAULT_SKIP_DUPLICATE_VALUES = false;
+    static final boolean DEFAULT_REMOVE_BRACKETS = false;
 
     @NotEmpty
     private String source = DEFAULT_SOURCE;
@@ -75,6 +76,10 @@ public class KeyValueProcessorConfig {
     @NotNull
     private boolean skipDuplicateValues = DEFAULT_SKIP_DUPLICATE_VALUES;
 
+    @JsonProperty("remove_brackets")
+    @NotNull
+    private boolean removeBrackets = DEFAULT_REMOVE_BRACKETS;
+
     public String getSource() {
         return source;
     }
@@ -130,4 +135,8 @@ public String getWhitespace() {
     public boolean getSkipDuplicateValues() {
         return skipDuplicateValues;
     }
+
+    public boolean getRemoveBrackets() {
+        return removeBrackets;
+    }
 }
diff --git a/...st/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java b/...st/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java
@@ -66,6 +66,7 @@ void setup() {
         lenient().when(mockConfig.getTransformKey()).thenReturn(defaultConfig.getTransformKey());
         lenient().when(mockConfig.getWhitespace()).thenReturn(defaultConfig.getWhitespace());
         lenient().when(mockConfig.getSkipDuplicateValues()).thenReturn(defaultConfig.getSkipDuplicateValues());
+        lenient().when(mockConfig.getRemoveBrackets()).thenReturn(defaultConfig.getRemoveBrackets());
 
         keyValueProcessor = new KeyValueProcessor(pluginMetrics, mockConfig);
     }
@@ -466,6 +467,33 @@ void testTrueThreeInputsDuplicateValuesKvProcessor() {
         assertThatKeyEquals(parsed_message, "key1", expectedValue);
     }
 
+    @Test
+    void testTrueRemoveBracketsKvProcessor() {
+        when(mockConfig.getRemoveBrackets()).thenReturn(true);
+
+        final Record<Event> record = getMessage("key1=(value1)&key2=[value2]&key3=<value3>");
+        final List<Record<Event>> editedRecords = (List<Record<Event>>) keyValueProcessor.doExecute(Collections.singletonList(record));
+        final LinkedHashMap<String, Object> parsed_message = getLinkedHashMap(editedRecords);
+
+        assertThat(parsed_message.size(), equalTo(3));
+        assertThatKeyEquals(parsed_message, "key1", "value1");
+        assertThatKeyEquals(parsed_message, "key2", "value2");
+        assertThatKeyEquals(parsed_message, "key3", "value3");
+    }
+
+    @Test
+    void testTrueRemoveMultipleBracketsKvProcessor() {
+        when(mockConfig.getRemoveBrackets()).thenReturn(true);
+
+        final Record<Event> record = getMessage("key1=((value1)&key2=[value1][value2]");
+        final List<Record<Event>> editedRecords = (List<Record<Event>>) keyValueProcessor.doExecute(Collections.singletonList(record));
+        final LinkedHashMap<String, Object> parsed_message = getLinkedHashMap(editedRecords);
+
+        assertThat(parsed_message.size(), equalTo(2));
+        assertThatKeyEquals(parsed_message, "key1", "value1");
+        assertThatKeyEquals(parsed_message, "key2", "value1value2");
+    }
+
     @Test
     void testShutdownIsReady() {
         assertThat(keyValueProcessor.isReadyForShutdown(), is(true));