From 03c48198cac692cade3875301e96a43bd6a786b0 Mon Sep 17 00:00:00 2001 From: Katherine Shen <40495707+shenkw1@users.noreply.github.com> Date: Mon, 24 Jul 2023 11:17:14 -0700 Subject: [PATCH] Remove brackets feature option (#3035) *add remove brackets feature option Signed-off-by: Kat Shen --------- Signed-off-by: Kat Shen Co-authored-by: Kat Shen --- .../key-value-processor/README.md | 5 ++++ .../processor/keyvalue/KeyValueProcessor.java | 19 +++++++++++-- .../keyvalue/KeyValueProcessorConfig.java | 9 ++++++ .../keyvalue/KeyValueProcessorTests.java | 28 +++++++++++++++++++ 4 files changed, 58 insertions(+), 3 deletions(-) diff --git a/data-prepper-plugins/key-value-processor/README.md b/data-prepper-plugins/key-value-processor/README.md index 505d0dbf91..91c34f7ea4 100644 --- a/data-prepper-plugins/key-value-processor/README.md +++ b/data-prepper-plugins/key-value-processor/README.md @@ -74,6 +74,11 @@ When run, the processor will parse the message into the following output: * Default: `false` * Example: `skip_duplicate_values` is `false`. `{"key1=value1&key1=value1"}` will parse into `{"key1": ["value1", "value1"]}` * Example: `skip_duplicate_values` is `true`. `{"key1=value1&key1=value1"}` will parse into `{"key1": "value1"}` +* `remove_brackets` - Specify whether to treat square brackets, angle brackets, and parentheses as value "wrappers" that should be removed from the value. + * Default: `false` + * Example: `remove_brackets` is `true`. `{"key1=(value1)"}` will parse into `{"key1": value1}` + * Example: `remove_brackets` is `false`. `{"key1=(value1)"}` will parse into `{"key1": "(value1)"}` + * In the case of a key-value pair with a brackets and a split character, the splitting will take priority over `remove_brackets=true`. `{key1=(value1&value2)}` will parse into `{"key1":"value1","value2)":null}` ## Developer Guide This plugin is compatible with Java 14. See diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java index 035c56f89c..b5c6671b0d 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java +++ b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java @@ -111,10 +111,16 @@ public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProces throw new IllegalArgumentException(String.format("The whitespace value: %s is not a valid option", keyValueProcessorConfig.getWhitespace())); } - final Pattern duplicateValueBoolCheck = Pattern.compile("true|false", Pattern.CASE_INSENSITIVE); - final Matcher duplicateValueBoolMatch = duplicateValueBoolCheck.matcher(String.valueOf(keyValueProcessorConfig.getSkipDuplicateValues())); + final Pattern boolCheck = Pattern.compile("true|false", Pattern.CASE_INSENSITIVE); + final Matcher duplicateValueBoolMatch = boolCheck.matcher(String.valueOf(keyValueProcessorConfig.getSkipDuplicateValues())); + final Matcher removeBracketsBoolMatch = boolCheck.matcher(String.valueOf(keyValueProcessorConfig.getRemoveBrackets())); + if (!duplicateValueBoolMatch.matches()) { - throw new IllegalArgumentException(String.format("The skip_duplicate_values value: %s is not a valid option", keyValueProcessorConfig.getSkipDuplicateValues())); + throw new IllegalArgumentException(String.format("The skip_duplicate_values value must be either true or false", keyValueProcessorConfig.getSkipDuplicateValues())); + } + + if (!removeBracketsBoolMatch.matches()) { + throw new IllegalArgumentException(String.format("The remove_brackets value must be either true or false", keyValueProcessorConfig.getRemoveBrackets())); } } @@ -195,6 +201,13 @@ public Collection> doExecute(final Collection> recor key = transformKey(key); } + if (keyValueProcessorConfig.getRemoveBrackets()) { + final String bracketRegex = "[\\[\\]()<>]"; + if (value != null) { + value = value.toString().replaceAll(bracketRegex,""); + } + } + addKeyValueToMap(parsedMap, key, value); } diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java index fa0cb6eac4..daa36bf5e3 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java +++ b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java @@ -25,6 +25,7 @@ public class KeyValueProcessorConfig { static final String DEFAULT_TRANSFORM_KEY = ""; static final String DEFAULT_WHITESPACE = "lenient"; static final boolean DEFAULT_SKIP_DUPLICATE_VALUES = false; + static final boolean DEFAULT_REMOVE_BRACKETS = false; @NotEmpty private String source = DEFAULT_SOURCE; @@ -75,6 +76,10 @@ public class KeyValueProcessorConfig { @NotNull private boolean skipDuplicateValues = DEFAULT_SKIP_DUPLICATE_VALUES; + @JsonProperty("remove_brackets") + @NotNull + private boolean removeBrackets = DEFAULT_REMOVE_BRACKETS; + public String getSource() { return source; } @@ -130,4 +135,8 @@ public String getWhitespace() { public boolean getSkipDuplicateValues() { return skipDuplicateValues; } + + public boolean getRemoveBrackets() { + return removeBrackets; + } } diff --git a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java b/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java index 743cd29f76..f54edea0c4 100644 --- a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java +++ b/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java @@ -66,6 +66,7 @@ void setup() { lenient().when(mockConfig.getTransformKey()).thenReturn(defaultConfig.getTransformKey()); lenient().when(mockConfig.getWhitespace()).thenReturn(defaultConfig.getWhitespace()); lenient().when(mockConfig.getSkipDuplicateValues()).thenReturn(defaultConfig.getSkipDuplicateValues()); + lenient().when(mockConfig.getRemoveBrackets()).thenReturn(defaultConfig.getRemoveBrackets()); keyValueProcessor = new KeyValueProcessor(pluginMetrics, mockConfig); } @@ -466,6 +467,33 @@ void testTrueThreeInputsDuplicateValuesKvProcessor() { assertThatKeyEquals(parsed_message, "key1", expectedValue); } + @Test + void testTrueRemoveBracketsKvProcessor() { + when(mockConfig.getRemoveBrackets()).thenReturn(true); + + final Record record = getMessage("key1=(value1)&key2=[value2]&key3="); + final List> editedRecords = (List>) keyValueProcessor.doExecute(Collections.singletonList(record)); + final LinkedHashMap parsed_message = getLinkedHashMap(editedRecords); + + assertThat(parsed_message.size(), equalTo(3)); + assertThatKeyEquals(parsed_message, "key1", "value1"); + assertThatKeyEquals(parsed_message, "key2", "value2"); + assertThatKeyEquals(parsed_message, "key3", "value3"); + } + + @Test + void testTrueRemoveMultipleBracketsKvProcessor() { + when(mockConfig.getRemoveBrackets()).thenReturn(true); + + final Record record = getMessage("key1=((value1)&key2=[value1][value2]"); + final List> editedRecords = (List>) keyValueProcessor.doExecute(Collections.singletonList(record)); + final LinkedHashMap parsed_message = getLinkedHashMap(editedRecords); + + assertThat(parsed_message.size(), equalTo(2)); + assertThatKeyEquals(parsed_message, "key1", "value1"); + assertThatKeyEquals(parsed_message, "key2", "value1value2"); + } + @Test void testShutdownIsReady() { assertThat(keyValueProcessor.isReadyForShutdown(), is(true));