diff --git a/data-prepper-plugins/key-value-processor/README.md b/data-prepper-plugins/key-value-processor/README.md index a53a818ba7..4bdd336cb5 100644 --- a/data-prepper-plugins/key-value-processor/README.md +++ b/data-prepper-plugins/key-value-processor/README.md @@ -61,6 +61,11 @@ When run, the processor will parse the message into the following output: * There is no default * Cannot be an empty string * Example: `delete_value_regex` is `"\s"`. `{"key1=value1 "}` will parse into `{"key1": "value1"}` +* `transform_key` - Change keys to lowercase, uppercase, or all capitals. + * Default is an empty string + * Example: `transform_key` is `lowercase`. `{"Key1=value1"}` will parse into `{"key1": "value1"}` + * Example: `transform_key` is `uppercase`. `{"key1=value1"}` will parse into `{"Key1": "value1"}` + * Example: `transform_key` is `capitalize`. `{"key1=value1"}` will parse into `{"KEY1": "value1"}` ## Developer Guide This plugin is compatible with Java 14. See diff --git a/data-prepper-plugins/key-value-processor/bin/main/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.class b/data-prepper-plugins/key-value-processor/bin/main/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.class new file mode 100644 index 0000000000..10c18f7784 Binary files /dev/null and b/data-prepper-plugins/key-value-processor/bin/main/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.class differ diff --git a/data-prepper-plugins/key-value-processor/bin/main/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.class b/data-prepper-plugins/key-value-processor/bin/main/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.class new file mode 100644 index 0000000000..770eed19e2 Binary files /dev/null and b/data-prepper-plugins/key-value-processor/bin/main/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.class differ diff --git a/data-prepper-plugins/key-value-processor/bin/test/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.class b/data-prepper-plugins/key-value-processor/bin/test/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.class new file mode 100644 index 0000000000..75aee46f15 Binary files /dev/null and b/data-prepper-plugins/key-value-processor/bin/test/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.class differ diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java index 49cfc323cd..d24f2499c0 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java +++ b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java @@ -15,6 +15,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -35,6 +36,7 @@ public class KeyValueProcessor extends AbstractProcessor, Record includeKeysSet = new HashSet(); + private final Set validTransformOptionSet = new HashSet<>(Arrays.asList("lowercase", "uppercase", "capitalize")); @DataPrepperPluginConstructor public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProcessorConfig keyValueProcessorConfig) { @@ -94,6 +96,13 @@ public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProces if(keyValueProcessorConfig.getIncludeKeys() != null) { includeKeysSet.addAll(keyValueProcessorConfig.getIncludeKeys()); } + + if(keyValueProcessorConfig.getTransformKey() != null + && !keyValueProcessorConfig.getTransformKey().isEmpty()) { + if(!validTransformOptionSet.contains(keyValueProcessorConfig.getTransformKey())) { + throw new IllegalArgumentException("transform_key is not a valid option"); + } + } } private String buildRegexFromCharacters(String s) { @@ -162,6 +171,11 @@ public Collection> doExecute(final Collection> recor value = ((String)value).replaceAll(keyValueProcessorConfig.getDeleteValueRegex(), ""); } + if(keyValueProcessorConfig.getTransformKey() != null + && !keyValueProcessorConfig.getTransformKey().isEmpty()) { + key = transformKey(key); + } + addKeyValueToMap(parsedMap, key, value); } @@ -171,6 +185,17 @@ public Collection> doExecute(final Collection> recor return records; } + private String transformKey(String key) { + if(keyValueProcessorConfig.getTransformKey().equals("lowercase")) { + key = key.toLowerCase(); + } else if(keyValueProcessorConfig.getTransformKey().equals("uppercase")) { + key = key.substring(0, 1).toUpperCase() + key.substring(1); + } else if(keyValueProcessorConfig.getTransformKey().equals("capitalize")) { + key = key.toUpperCase(); + } + return key; + } + private void addKeyValueToMap(final Map parsedMap, final String key, final Object value) { if(!parsedMap.containsKey(key)) { parsedMap.put(key, value); diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java index 42d75d7abe..4e696696f1 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java +++ b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java @@ -22,6 +22,7 @@ public class KeyValueProcessorConfig { static final String DEFAULT_PREFIX = ""; static final String DEFAULT_DELETE_KEY_REGEX = ""; static final String DEFAULT_DELETE_VALUE_REGEX = ""; + static final String DEFAULT_TRANSFORM_KEY = ""; @NotEmpty private String source = DEFAULT_SOURCE; @@ -60,6 +61,10 @@ public class KeyValueProcessorConfig { @NotNull private String deleteValueRegex = DEFAULT_DELETE_VALUE_REGEX; + @JsonProperty("transform_key") + @NotNull + private String transformKey = DEFAULT_TRANSFORM_KEY; + public String getSource() { return source; } @@ -103,4 +108,8 @@ public String getDeleteKeyRegex() { public String getDeleteValueRegex() { return deleteValueRegex; } + + public String getTransformKey() { + return transformKey; + } } diff --git a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java b/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java index 1b2ef07971..8379a8c877 100644 --- a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java +++ b/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java @@ -63,6 +63,7 @@ void setup() { lenient().when(mockConfig.getPrefix()).thenReturn(defaultConfig.getPrefix()); lenient().when(mockConfig.getDeleteKeyRegex()).thenReturn(defaultConfig.getDeleteKeyRegex()); lenient().when(mockConfig.getDeleteValueRegex()).thenReturn(defaultConfig.getDeleteValueRegex()); + lenient().when(mockConfig.getTransformKey()).thenReturn(defaultConfig.getTransformKey()); keyValueProcessor = new KeyValueProcessor(pluginMetrics, mockConfig); } @@ -371,6 +372,36 @@ void testDeleteValueAndKeyRegexKvProcessor() { assertThatKeyEquals(parsed_message, "key2", "value2"); } + @Test + void testLowercaseTransformKvProcessor() { + final Record record = getMessage("Key1=value1"); + final List> editedRecords = (List>) keyValueProcessor.doExecute(Collections.singletonList(record)); + final LinkedHashMap parsed_message = getLinkedHashMap(editedRecords); + + assertThat(parsed_message.size(), equalTo(2)); + assertThatKeyEquals(parsed_message, "key1", "value1"); + } + + @Test + void testUppercaseTransformKvProcessor() { + final Record record = getMessage("key1=value1"); + final List> editedRecords = (List>) keyValueProcessor.doExecute(Collections.singletonList(record)); + final LinkedHashMap parsed_message = getLinkedHashMap(editedRecords); + + assertThat(parsed_message.size(), equalTo(2)); + assertThatKeyEquals(parsed_message, "Key1", "value1"); + } + + @Test + void testCapitalizeTransformKvProcessor() { + final Record record = getMessage("key1=value1"); + final List> editedRecords = (List>) keyValueProcessor.doExecute(Collections.singletonList(record)); + final LinkedHashMap parsed_message = getLinkedHashMap(editedRecords); + + assertThat(parsed_message.size(), equalTo(2)); + assertThatKeyEquals(parsed_message, "KEY1", "value1"); + } + @Test void testShutdownIsReady() { assertThat(keyValueProcessor.isReadyForShutdown(), is(true));