diff --git a/data-prepper-plugins/key-value-processor/README.md b/data-prepper-plugins/key-value-processor/README.md index 55dda2dfc2..0d7ff129fe 100644 --- a/data-prepper-plugins/key-value-processor/README.md +++ b/data-prepper-plugins/key-value-processor/README.md @@ -66,6 +66,10 @@ When run, the processor will parse the message into the following output: * Example: `transform_key` is `lowercase`. `{"Key1=value1"}` will parse into `{"key1": "value1"}` * Example: `transform_key` is `uppercase`. `{"key1=value1"}` will parse into `{"Key1": "value1"}` * Example: `transform_key` is `capitalize`. `{"key1=value1"}` will parse into `{"KEY1": "value1"}` +* `whitespace` - Specify whether to be lenient or strict with the acceptance of unnecessary whitespace surrounding the configured value-split sequence. + * Default: `lenient` + * Example: `whitespace` is `"lenient"`. `{"key1 = value1"}` will parse into `{"key1 ": " value1"}` + * Example: `whitespace` is `"strict"`. `{"key1 = value1"}` will parse into `{"key1": "value1"}` ## Developer Guide This plugin is compatible with Java 14. See diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java index d531c0b57a..d0549dc096 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java +++ b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java @@ -35,10 +35,13 @@ public class KeyValueProcessor extends AbstractProcessor, Record includeKeysSet = new HashSet(); - private final String LOWERCASE_KEY = "lowercase"; - private final String UPPERCASE_KEY = "uppercase"; - private final String CAPITALIZE_KEY = "capitalize"; - private final Set validTransformOptionSet = Set.of("", LOWERCASE_KEY, UPPERCASE_KEY, CAPITALIZE_KEY); + private final String lowercaseKey = "lowercase"; + private final String uppercaseKey = "uppercase"; + private final String capitalizeKey = "capitalize"; + private final Set validTransformOptionSet = Set.of("", lowercaseKey, uppercaseKey, capitalizeKey); + private final String whitespaceStrict = "strict"; + private final String whitespaceLenient = "lenient"; + private final Set validWhitespaceSet = Set.of(whitespaceLenient, whitespaceStrict); @DataPrepperPluginConstructor public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProcessorConfig keyValueProcessorConfig) { @@ -87,21 +90,25 @@ public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProces keyValueDelimiterPattern = Pattern.compile(regex); } - if(!validateRegex(keyValueProcessorConfig.getDeleteKeyRegex())) { + if (!validateRegex(keyValueProcessorConfig.getDeleteKeyRegex())) { throw new PatternSyntaxException("delete_key_regex is not a valid regex string", keyValueProcessorConfig.getDeleteKeyRegex(), -1); } - if(!validateRegex(keyValueProcessorConfig.getDeleteValueRegex())) { + if (!validateRegex(keyValueProcessorConfig.getDeleteValueRegex())) { throw new PatternSyntaxException("delete_value_regex is not a valid regex string", keyValueProcessorConfig.getDeleteValueRegex(), -1); } - if(keyValueProcessorConfig.getIncludeKeys() != null) { + if (keyValueProcessorConfig.getIncludeKeys() != null) { includeKeysSet.addAll(keyValueProcessorConfig.getIncludeKeys()); } - if(!validTransformOptionSet.contains(keyValueProcessorConfig.getTransformKey())) { + if (!validTransformOptionSet.contains(keyValueProcessorConfig.getTransformKey())) { throw new IllegalArgumentException(String.format("The transform_key value: %s is not a valid option", keyValueProcessorConfig.getTransformKey())); } + + if (!(validWhitespaceSet.contains(keyValueProcessorConfig.getWhitespace()))) { + throw new IllegalArgumentException(String.format("The whitespace value: %s is not a valid option", keyValueProcessorConfig.getWhitespace())); + } } private String buildRegexFromCharacters(String s) { @@ -170,7 +177,13 @@ public Collection> doExecute(final Collection> recor value = ((String)value).replaceAll(keyValueProcessorConfig.getDeleteValueRegex(), ""); } - if(keyValueProcessorConfig.getTransformKey() != null + if (keyValueProcessorConfig.getWhitespace().equals(whitespaceStrict)) { + String[] whitespace_arr = trimWhitespace(key, value); + key = whitespace_arr[0]; + value = whitespace_arr[1]; + } + + if (keyValueProcessorConfig.getTransformKey() != null && !keyValueProcessorConfig.getTransformKey().isEmpty()) { key = transformKey(key); } @@ -184,12 +197,17 @@ public Collection> doExecute(final Collection> recor return records; } + private String[] trimWhitespace(String key, Object value) { + String[] arr = {key.stripTrailing(), value.toString().stripLeading()}; + return arr; + } + private String transformKey(String key) { - if(keyValueProcessorConfig.getTransformKey().equals(LOWERCASE_KEY)) { + if (keyValueProcessorConfig.getTransformKey().equals(lowercaseKey)) { key = key.toLowerCase(); - } else if(keyValueProcessorConfig.getTransformKey().equals(UPPERCASE_KEY)) { + } else if (keyValueProcessorConfig.getTransformKey().equals(uppercaseKey)) { key = key.substring(0, 1).toUpperCase() + key.substring(1); - } else if(keyValueProcessorConfig.getTransformKey().equals(CAPITALIZE_KEY)) { + } else if (keyValueProcessorConfig.getTransformKey().equals(capitalizeKey)) { key = key.toUpperCase(); } return key; diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java index 4e696696f1..1b68d27e3a 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java +++ b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java @@ -23,6 +23,7 @@ public class KeyValueProcessorConfig { static final String DEFAULT_DELETE_KEY_REGEX = ""; static final String DEFAULT_DELETE_VALUE_REGEX = ""; static final String DEFAULT_TRANSFORM_KEY = ""; + static final String DEFAULT_WHITESPACE = "lenient"; @NotEmpty private String source = DEFAULT_SOURCE; @@ -65,6 +66,10 @@ public class KeyValueProcessorConfig { @NotNull private String transformKey = DEFAULT_TRANSFORM_KEY; + @JsonProperty("whitespace") + @NotNull + private String whitespace = DEFAULT_WHITESPACE; + public String getSource() { return source; } @@ -112,4 +117,8 @@ public String getDeleteValueRegex() { public String getTransformKey() { return transformKey; } + + public String getWhitespace() { + return whitespace; + } } diff --git a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java b/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java index 35af3e0108..90a0de9e56 100644 --- a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java +++ b/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java @@ -64,6 +64,7 @@ void setup() { lenient().when(mockConfig.getDeleteKeyRegex()).thenReturn(defaultConfig.getDeleteKeyRegex()); lenient().when(mockConfig.getDeleteValueRegex()).thenReturn(defaultConfig.getDeleteValueRegex()); lenient().when(mockConfig.getTransformKey()).thenReturn(defaultConfig.getTransformKey()); + lenient().when(mockConfig.getWhitespace()).thenReturn(defaultConfig.getWhitespace()); keyValueProcessor = new KeyValueProcessor(pluginMetrics, mockConfig); } @@ -408,6 +409,18 @@ void testCapitalizeTransformKvProcessor() { assertThatKeyEquals(parsed_message, "KEY1", "value1"); } + @Test + void testStrictWhitespaceKvProcessor() { + when(mockConfig.getWhitespace()).thenReturn("strict"); + + final Record record = getMessage("key1 = value1"); + final List> editedRecords = (List>) keyValueProcessor.doExecute(Collections.singletonList(record)); + final LinkedHashMap parsed_message = getLinkedHashMap(editedRecords); + + assertThat(parsed_message.size(), equalTo(1)); + assertThatKeyEquals(parsed_message, "key1", "value1"); + } + @Test void testShutdownIsReady() { assertThat(keyValueProcessor.isReadyForShutdown(), is(true)); @@ -431,4 +444,4 @@ private void assertThatKeyEquals(final LinkedHashMap parsed_mess assertThat(parsed_message.containsKey(key), is(true)); assertThat(parsed_message.get(key), equalTo(value)); } -} +} \ No newline at end of file