Implement transform_key feature (opensearch-project#2977)

implement transform_key feature Signed-off-by: Kat Shen <[email protected]> Signed-off-by: Marcos Gonzalez Mayedo <[email protected]>
MaGonzalMayedo · Jul 25, 2023 · 664674b · 664674b
1 parent 7538414
commit 664674b
Show file tree

Hide file tree

Showing 4 changed files with 75 additions and 0 deletions.
diff --git a/data-prepper-plugins/key-value-processor/README.md b/data-prepper-plugins/key-value-processor/README.md
@@ -61,6 +61,11 @@ When run, the processor will parse the message into the following output:
   * There is no default
   * Cannot be an empty string
   * Example: `delete_value_regex` is `"\s"`. `{"key1=value1 "}` will parse into `{"key1": "value1"}`
+* `transform_key` - Change keys to lowercase, uppercase, or all capitals.
+  * Default is an empty string (no transformation)
+  * Example: `transform_key` is `lowercase`. `{"Key1=value1"}` will parse into `{"key1": "value1"}`
+  * Example: `transform_key` is `uppercase`. `{"key1=value1"}` will parse into `{"Key1": "value1"}`
+  * Example: `transform_key` is `capitalize`. `{"key1=value1"}` will parse into `{"KEY1": "value1"}`
 
 ## Developer Guide
 This plugin is compatible with Java 14. See

diff --git a/...rc/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java b/...rc/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java
@@ -35,6 +35,10 @@ public class KeyValueProcessor extends AbstractProcessor<Record<Event>, Record<E
     private final Pattern fieldDelimiterPattern;
     private final Pattern keyValueDelimiterPattern;
     private final Set<String> includeKeysSet = new HashSet<String>();
+    private final String LOWERCASE_KEY = "lowercase";
+    private final String UPPERCASE_KEY = "uppercase";
+    private final String CAPITALIZE_KEY = "capitalize";
+    private final Set<String> validTransformOptionSet = Set.of("", LOWERCASE_KEY, UPPERCASE_KEY, CAPITALIZE_KEY);
 
     @DataPrepperPluginConstructor
     public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProcessorConfig keyValueProcessorConfig) {
@@ -94,6 +98,10 @@ public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProces
         if(keyValueProcessorConfig.getIncludeKeys() != null) {
             includeKeysSet.addAll(keyValueProcessorConfig.getIncludeKeys());
         }
+
+        if(!validTransformOptionSet.contains(keyValueProcessorConfig.getTransformKey())) {
+            throw new IllegalArgumentException(String.format("The transform_key value: %s is not a valid option", keyValueProcessorConfig.getTransformKey()));
+        }
     }
 
     private String buildRegexFromCharacters(String s) {
@@ -162,6 +170,11 @@ public Collection<Record<Event>> doExecute(final Collection<Record<Event>> recor
                     value = ((String)value).replaceAll(keyValueProcessorConfig.getDeleteValueRegex(), "");
                 }
 
+                if(keyValueProcessorConfig.getTransformKey() != null
+                        && !keyValueProcessorConfig.getTransformKey().isEmpty()) {
+                    key = transformKey(key);
+                }
+
                 addKeyValueToMap(parsedMap, key, value);
             }
 
@@ -171,6 +184,17 @@ public Collection<Record<Event>> doExecute(final Collection<Record<Event>> recor
         return records;
     }
 
+    private String transformKey(String key) {
+        if(keyValueProcessorConfig.getTransformKey().equals(LOWERCASE_KEY)) {
+            key = key.toLowerCase();
+        } else if(keyValueProcessorConfig.getTransformKey().equals(UPPERCASE_KEY)) {
+            key = key.substring(0, 1).toUpperCase() + key.substring(1);
+        } else if(keyValueProcessorConfig.getTransformKey().equals(CAPITALIZE_KEY)) {
+            key = key.toUpperCase();
+        }
+        return key;
+    }
+
     private void addKeyValueToMap(final Map<String, Object> parsedMap, final String key, final Object value) {
         if(!parsedMap.containsKey(key)) {
             parsedMap.put(key, value);

diff --git a/...n/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java b/...n/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java
@@ -22,6 +22,7 @@ public class KeyValueProcessorConfig {
     static final String DEFAULT_PREFIX = "";
     static final String DEFAULT_DELETE_KEY_REGEX = "";
     static final String DEFAULT_DELETE_VALUE_REGEX = "";
+    static final String DEFAULT_TRANSFORM_KEY = "";
 
     @NotEmpty
     private String source = DEFAULT_SOURCE;
@@ -60,6 +61,10 @@ public class KeyValueProcessorConfig {
     @NotNull
     private String deleteValueRegex = DEFAULT_DELETE_VALUE_REGEX;
 
+    @JsonProperty("transform_key")
+    @NotNull
+    private String transformKey = DEFAULT_TRANSFORM_KEY;
+
     public String getSource() {
         return source;
     }
@@ -103,4 +108,8 @@ public String getDeleteKeyRegex() {
     public String getDeleteValueRegex() {
         return deleteValueRegex;
     }
+
+    public String getTransformKey() {
+        return transformKey;
+    }
 }
diff --git a/...st/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java b/...st/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java
@@ -63,6 +63,7 @@ void setup() {
         lenient().when(mockConfig.getPrefix()).thenReturn(defaultConfig.getPrefix());
         lenient().when(mockConfig.getDeleteKeyRegex()).thenReturn(defaultConfig.getDeleteKeyRegex());
         lenient().when(mockConfig.getDeleteValueRegex()).thenReturn(defaultConfig.getDeleteValueRegex());
+        lenient().when(mockConfig.getTransformKey()).thenReturn(defaultConfig.getTransformKey());
 
         keyValueProcessor = new KeyValueProcessor(pluginMetrics, mockConfig);
     }
@@ -371,6 +372,42 @@ void testDeleteValueAndKeyRegexKvProcessor() {
         assertThatKeyEquals(parsed_message, "key2", "value2");
     }
 
+    @Test
+    void testLowercaseTransformKvProcessor() {
+        when(mockConfig.getTransformKey()).thenReturn("lowercase");
+
+        final Record<Event> record = getMessage("Key1=value1");
+        final List<Record<Event>> editedRecords = (List<Record<Event>>) keyValueProcessor.doExecute(Collections.singletonList(record));
+        final LinkedHashMap<String, Object> parsed_message = getLinkedHashMap(editedRecords);
+
+        assertThat(parsed_message.size(), equalTo(1));
+        assertThatKeyEquals(parsed_message, "key1", "value1");
+    }
+
+    @Test
+    void testUppercaseTransformKvProcessor() {
+        when(mockConfig.getTransformKey()).thenReturn("uppercase");
+
+        final Record<Event> record = getMessage("key1=value1");
+        final List<Record<Event>> editedRecords = (List<Record<Event>>) keyValueProcessor.doExecute(Collections.singletonList(record));
+        final LinkedHashMap<String, Object> parsed_message = getLinkedHashMap(editedRecords);
+
+        assertThat(parsed_message.size(), equalTo(1));
+        assertThatKeyEquals(parsed_message, "Key1", "value1");
+    }
+
+    @Test
+    void testCapitalizeTransformKvProcessor() {
+        when(mockConfig.getTransformKey()).thenReturn("capitalize");
+
+        final Record<Event> record = getMessage("key1=value1");
+        final List<Record<Event>> editedRecords = (List<Record<Event>>) keyValueProcessor.doExecute(Collections.singletonList(record));
+        final LinkedHashMap<String, Object> parsed_message = getLinkedHashMap(editedRecords);
+
+        assertThat(parsed_message.size(), equalTo(1));
+        assertThatKeyEquals(parsed_message, "KEY1", "value1");
+    }
+
     @Test
     void testShutdownIsReady() {
         assertThat(keyValueProcessor.isReadyForShutdown(), is(true));