From 664674b189dea0a62b116749bf4f5e6322b5d445 Mon Sep 17 00:00:00 2001 From: Katherine Shen <40495707+shenkw1@users.noreply.github.com> Date: Thu, 6 Jul 2023 09:37:55 -0700 Subject: [PATCH] Implement transform_key feature (#2977) implement transform_key feature Signed-off-by: Kat Shen Signed-off-by: Marcos Gonzalez Mayedo --- .../key-value-processor/README.md | 5 +++ .../processor/keyvalue/KeyValueProcessor.java | 24 ++++++++++++ .../keyvalue/KeyValueProcessorConfig.java | 9 +++++ .../keyvalue/KeyValueProcessorTests.java | 37 +++++++++++++++++++ 4 files changed, 75 insertions(+) diff --git a/data-prepper-plugins/key-value-processor/README.md b/data-prepper-plugins/key-value-processor/README.md index a53a818ba7..55dda2dfc2 100644 --- a/data-prepper-plugins/key-value-processor/README.md +++ b/data-prepper-plugins/key-value-processor/README.md @@ -61,6 +61,11 @@ When run, the processor will parse the message into the following output: * There is no default * Cannot be an empty string * Example: `delete_value_regex` is `"\s"`. `{"key1=value1 "}` will parse into `{"key1": "value1"}` +* `transform_key` - Change keys to lowercase, uppercase, or all capitals. + * Default is an empty string (no transformation) + * Example: `transform_key` is `lowercase`. `{"Key1=value1"}` will parse into `{"key1": "value1"}` + * Example: `transform_key` is `uppercase`. `{"key1=value1"}` will parse into `{"Key1": "value1"}` + * Example: `transform_key` is `capitalize`. `{"key1=value1"}` will parse into `{"KEY1": "value1"}` ## Developer Guide This plugin is compatible with Java 14. See diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java index 49cfc323cd..d531c0b57a 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java +++ b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java @@ -35,6 +35,10 @@ public class KeyValueProcessor extends AbstractProcessor, Record includeKeysSet = new HashSet(); + private final String LOWERCASE_KEY = "lowercase"; + private final String UPPERCASE_KEY = "uppercase"; + private final String CAPITALIZE_KEY = "capitalize"; + private final Set validTransformOptionSet = Set.of("", LOWERCASE_KEY, UPPERCASE_KEY, CAPITALIZE_KEY); @DataPrepperPluginConstructor public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProcessorConfig keyValueProcessorConfig) { @@ -94,6 +98,10 @@ public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProces if(keyValueProcessorConfig.getIncludeKeys() != null) { includeKeysSet.addAll(keyValueProcessorConfig.getIncludeKeys()); } + + if(!validTransformOptionSet.contains(keyValueProcessorConfig.getTransformKey())) { + throw new IllegalArgumentException(String.format("The transform_key value: %s is not a valid option", keyValueProcessorConfig.getTransformKey())); + } } private String buildRegexFromCharacters(String s) { @@ -162,6 +170,11 @@ public Collection> doExecute(final Collection> recor value = ((String)value).replaceAll(keyValueProcessorConfig.getDeleteValueRegex(), ""); } + if(keyValueProcessorConfig.getTransformKey() != null + && !keyValueProcessorConfig.getTransformKey().isEmpty()) { + key = transformKey(key); + } + addKeyValueToMap(parsedMap, key, value); } @@ -171,6 +184,17 @@ public Collection> doExecute(final Collection> recor return records; } + private String transformKey(String key) { + if(keyValueProcessorConfig.getTransformKey().equals(LOWERCASE_KEY)) { + key = key.toLowerCase(); + } else if(keyValueProcessorConfig.getTransformKey().equals(UPPERCASE_KEY)) { + key = key.substring(0, 1).toUpperCase() + key.substring(1); + } else if(keyValueProcessorConfig.getTransformKey().equals(CAPITALIZE_KEY)) { + key = key.toUpperCase(); + } + return key; + } + private void addKeyValueToMap(final Map parsedMap, final String key, final Object value) { if(!parsedMap.containsKey(key)) { parsedMap.put(key, value); diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java index 42d75d7abe..4e696696f1 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java +++ b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java @@ -22,6 +22,7 @@ public class KeyValueProcessorConfig { static final String DEFAULT_PREFIX = ""; static final String DEFAULT_DELETE_KEY_REGEX = ""; static final String DEFAULT_DELETE_VALUE_REGEX = ""; + static final String DEFAULT_TRANSFORM_KEY = ""; @NotEmpty private String source = DEFAULT_SOURCE; @@ -60,6 +61,10 @@ public class KeyValueProcessorConfig { @NotNull private String deleteValueRegex = DEFAULT_DELETE_VALUE_REGEX; + @JsonProperty("transform_key") + @NotNull + private String transformKey = DEFAULT_TRANSFORM_KEY; + public String getSource() { return source; } @@ -103,4 +108,8 @@ public String getDeleteKeyRegex() { public String getDeleteValueRegex() { return deleteValueRegex; } + + public String getTransformKey() { + return transformKey; + } } diff --git a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java b/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java index 1b2ef07971..35af3e0108 100644 --- a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java +++ b/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java @@ -63,6 +63,7 @@ void setup() { lenient().when(mockConfig.getPrefix()).thenReturn(defaultConfig.getPrefix()); lenient().when(mockConfig.getDeleteKeyRegex()).thenReturn(defaultConfig.getDeleteKeyRegex()); lenient().when(mockConfig.getDeleteValueRegex()).thenReturn(defaultConfig.getDeleteValueRegex()); + lenient().when(mockConfig.getTransformKey()).thenReturn(defaultConfig.getTransformKey()); keyValueProcessor = new KeyValueProcessor(pluginMetrics, mockConfig); } @@ -371,6 +372,42 @@ void testDeleteValueAndKeyRegexKvProcessor() { assertThatKeyEquals(parsed_message, "key2", "value2"); } + @Test + void testLowercaseTransformKvProcessor() { + when(mockConfig.getTransformKey()).thenReturn("lowercase"); + + final Record record = getMessage("Key1=value1"); + final List> editedRecords = (List>) keyValueProcessor.doExecute(Collections.singletonList(record)); + final LinkedHashMap parsed_message = getLinkedHashMap(editedRecords); + + assertThat(parsed_message.size(), equalTo(1)); + assertThatKeyEquals(parsed_message, "key1", "value1"); + } + + @Test + void testUppercaseTransformKvProcessor() { + when(mockConfig.getTransformKey()).thenReturn("uppercase"); + + final Record record = getMessage("key1=value1"); + final List> editedRecords = (List>) keyValueProcessor.doExecute(Collections.singletonList(record)); + final LinkedHashMap parsed_message = getLinkedHashMap(editedRecords); + + assertThat(parsed_message.size(), equalTo(1)); + assertThatKeyEquals(parsed_message, "Key1", "value1"); + } + + @Test + void testCapitalizeTransformKvProcessor() { + when(mockConfig.getTransformKey()).thenReturn("capitalize"); + + final Record record = getMessage("key1=value1"); + final List> editedRecords = (List>) keyValueProcessor.doExecute(Collections.singletonList(record)); + final LinkedHashMap parsed_message = getLinkedHashMap(editedRecords); + + assertThat(parsed_message.size(), equalTo(1)); + assertThatKeyEquals(parsed_message, "KEY1", "value1"); + } + @Test void testShutdownIsReady() { assertThat(keyValueProcessor.isReadyForShutdown(), is(true));