Whitespace (#3004)

implement transform_key feature Signed-off-by: Kat Shen <[email protected]> --------- Signed-off-by: Kat Shen <[email protected]> Co-authored-by: Kat Shen <[email protected]>
opensearch-project · Jul 12, 2023 · 2ed818f · 2ed818f
1 parent 276268b
commit 2ed818f
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 13 deletions.
diff --git a/data-prepper-plugins/key-value-processor/README.md b/data-prepper-plugins/key-value-processor/README.md
@@ -66,6 +66,10 @@ When run, the processor will parse the message into the following output:
   * Example: `transform_key` is `lowercase`. `{"Key1=value1"}` will parse into `{"key1": "value1"}`
   * Example: `transform_key` is `uppercase`. `{"key1=value1"}` will parse into `{"Key1": "value1"}`
   * Example: `transform_key` is `capitalize`. `{"key1=value1"}` will parse into `{"KEY1": "value1"}`
+* `whitespace` - Specify whether to be lenient or strict with the acceptance of unnecessary whitespace surrounding the configured value-split sequence.
+  * Default: `lenient`
+  * Example: `whitespace` is `"lenient"`. `{"key1  =  value1"}` will parse into `{"key1  ": "  value1"}`
+  * Example: `whitespace` is `"strict"`. `{"key1  =  value1"}` will parse into `{"key1": "value1"}`
 
 ## Developer Guide
 This plugin is compatible with Java 14. See

diff --git a/...rc/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java b/...rc/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java
@@ -35,10 +35,13 @@ public class KeyValueProcessor extends AbstractProcessor<Record<Event>, Record<E
     private final Pattern fieldDelimiterPattern;
     private final Pattern keyValueDelimiterPattern;
     private final Set<String> includeKeysSet = new HashSet<String>();
-    private final String LOWERCASE_KEY = "lowercase";
-    private final String UPPERCASE_KEY = "uppercase";
-    private final String CAPITALIZE_KEY = "capitalize";
-    private final Set<String> validTransformOptionSet = Set.of("", LOWERCASE_KEY, UPPERCASE_KEY, CAPITALIZE_KEY);
+    private final String lowercaseKey = "lowercase";
+    private final String uppercaseKey = "uppercase";
+    private final String capitalizeKey = "capitalize";
+    private final Set<String> validTransformOptionSet = Set.of("", lowercaseKey, uppercaseKey, capitalizeKey);
+    private final String whitespaceStrict = "strict";
+    private final String whitespaceLenient = "lenient";
+    private final Set<String> validWhitespaceSet = Set.of(whitespaceLenient, whitespaceStrict);
 
     @DataPrepperPluginConstructor
     public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProcessorConfig keyValueProcessorConfig) {
@@ -87,21 +90,25 @@ public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProces
             keyValueDelimiterPattern = Pattern.compile(regex);
         }
 
-        if(!validateRegex(keyValueProcessorConfig.getDeleteKeyRegex())) {
+        if (!validateRegex(keyValueProcessorConfig.getDeleteKeyRegex())) {
             throw new PatternSyntaxException("delete_key_regex is not a valid regex string", keyValueProcessorConfig.getDeleteKeyRegex(), -1);
         }
 
-        if(!validateRegex(keyValueProcessorConfig.getDeleteValueRegex())) {
+        if (!validateRegex(keyValueProcessorConfig.getDeleteValueRegex())) {
             throw new PatternSyntaxException("delete_value_regex is not a valid regex string", keyValueProcessorConfig.getDeleteValueRegex(), -1);
         }
 
-        if(keyValueProcessorConfig.getIncludeKeys() != null) {
+        if (keyValueProcessorConfig.getIncludeKeys() != null) {
             includeKeysSet.addAll(keyValueProcessorConfig.getIncludeKeys());
         }
 
-        if(!validTransformOptionSet.contains(keyValueProcessorConfig.getTransformKey())) {
+        if (!validTransformOptionSet.contains(keyValueProcessorConfig.getTransformKey())) {
             throw new IllegalArgumentException(String.format("The transform_key value: %s is not a valid option", keyValueProcessorConfig.getTransformKey()));
         }
+
+        if (!(validWhitespaceSet.contains(keyValueProcessorConfig.getWhitespace()))) {
+            throw new IllegalArgumentException(String.format("The whitespace value: %s is not a valid option", keyValueProcessorConfig.getWhitespace()));
+        }
     }
 
     private String buildRegexFromCharacters(String s) {
@@ -170,7 +177,13 @@ public Collection<Record<Event>> doExecute(final Collection<Record<Event>> recor
                     value = ((String)value).replaceAll(keyValueProcessorConfig.getDeleteValueRegex(), "");
                 }
 
-                if(keyValueProcessorConfig.getTransformKey() != null
+                if (keyValueProcessorConfig.getWhitespace().equals(whitespaceStrict)) {
+                    String[] whitespace_arr = trimWhitespace(key, value);
+                    key = whitespace_arr[0];
+                    value = whitespace_arr[1];
+                }
+
+                if (keyValueProcessorConfig.getTransformKey() != null
                         && !keyValueProcessorConfig.getTransformKey().isEmpty()) {
                     key = transformKey(key);
                 }
@@ -184,12 +197,17 @@ public Collection<Record<Event>> doExecute(final Collection<Record<Event>> recor
         return records;
     }
 
+    private String[] trimWhitespace(String key, Object value) {
+        String[] arr = {key.stripTrailing(), value.toString().stripLeading()};
+        return arr;
+    }
+
     private String transformKey(String key) {
-        if(keyValueProcessorConfig.getTransformKey().equals(LOWERCASE_KEY)) {
+        if (keyValueProcessorConfig.getTransformKey().equals(lowercaseKey)) {
             key = key.toLowerCase();
-        } else if(keyValueProcessorConfig.getTransformKey().equals(UPPERCASE_KEY)) {
+        } else if (keyValueProcessorConfig.getTransformKey().equals(uppercaseKey)) {
             key = key.substring(0, 1).toUpperCase() + key.substring(1);
-        } else if(keyValueProcessorConfig.getTransformKey().equals(CAPITALIZE_KEY)) {
+        } else if (keyValueProcessorConfig.getTransformKey().equals(capitalizeKey)) {
             key = key.toUpperCase();
         }
         return key;

diff --git a/...n/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java b/...n/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java
@@ -23,6 +23,7 @@ public class KeyValueProcessorConfig {
     static final String DEFAULT_DELETE_KEY_REGEX = "";
     static final String DEFAULT_DELETE_VALUE_REGEX = "";
     static final String DEFAULT_TRANSFORM_KEY = "";
+    static final String DEFAULT_WHITESPACE = "lenient";
 
     @NotEmpty
     private String source = DEFAULT_SOURCE;
@@ -65,6 +66,10 @@ public class KeyValueProcessorConfig {
     @NotNull
     private String transformKey = DEFAULT_TRANSFORM_KEY;
 
+    @JsonProperty("whitespace")
+    @NotNull
+    private String whitespace = DEFAULT_WHITESPACE;
+
     public String getSource() {
         return source;
     }
@@ -112,4 +117,8 @@ public String getDeleteValueRegex() {
     public String getTransformKey() {
         return transformKey;
     }
+
+    public String getWhitespace() {
+        return whitespace;
+    }
 }
diff --git a/...st/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java b/...st/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java
@@ -64,6 +64,7 @@ void setup() {
         lenient().when(mockConfig.getDeleteKeyRegex()).thenReturn(defaultConfig.getDeleteKeyRegex());
         lenient().when(mockConfig.getDeleteValueRegex()).thenReturn(defaultConfig.getDeleteValueRegex());
         lenient().when(mockConfig.getTransformKey()).thenReturn(defaultConfig.getTransformKey());
+        lenient().when(mockConfig.getWhitespace()).thenReturn(defaultConfig.getWhitespace());
 
         keyValueProcessor = new KeyValueProcessor(pluginMetrics, mockConfig);
     }
@@ -408,6 +409,18 @@ void testCapitalizeTransformKvProcessor() {
         assertThatKeyEquals(parsed_message, "KEY1", "value1");
     }
 
+    @Test
+    void testStrictWhitespaceKvProcessor() {
+        when(mockConfig.getWhitespace()).thenReturn("strict");
+
+        final Record<Event> record = getMessage("key1  =  value1");
+        final List<Record<Event>> editedRecords = (List<Record<Event>>) keyValueProcessor.doExecute(Collections.singletonList(record));
+        final LinkedHashMap<String, Object> parsed_message = getLinkedHashMap(editedRecords);
+
+        assertThat(parsed_message.size(), equalTo(1));
+        assertThatKeyEquals(parsed_message, "key1", "value1");
+    }
+
     @Test
     void testShutdownIsReady() {
         assertThat(keyValueProcessor.isReadyForShutdown(), is(true));
@@ -431,4 +444,4 @@ private void assertThatKeyEquals(final LinkedHashMap<String, Object> parsed_mess
         assertThat(parsed_message.containsKey(key), is(true));
         assertThat(parsed_message.get(key), equalTo(value));
     }
-}
+}