implement transform_key feature

opensearch-project · Jul 3, 2023 · 78f1667 · 78f1667
1 parent 4967df2
commit 78f1667
Show file tree

Hide file tree

Showing 7 changed files with 70 additions and 0 deletions.
diff --git a/data-prepper-plugins/key-value-processor/README.md b/data-prepper-plugins/key-value-processor/README.md
@@ -61,6 +61,11 @@ When run, the processor will parse the message into the following output:
   * There is no default
   * Cannot be an empty string
   * Example: `delete_value_regex` is `"\s"`. `{"key1=value1 "}` will parse into `{"key1": "value1"}`
+* `transform_key` - Change keys to lowercase, uppercase, or all capitals.
+  * Default is an empty string
+  * Example: `transform_key` is `lowercase`. `{"Key1=value1"}` will parse into `{"key1": "value1"}`
+  * Example: `transform_key` is `uppercase`. `{"key1=value1"}` will parse into `{"Key1": "value1"}`
+  * Example: `transform_key` is `capitalize`. `{"key1=value1"}` will parse into `{"KEY1": "value1"}`
 
 ## Developer Guide
 This plugin is compatible with Java 14. See

diff --git a/...or/bin/main/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.class b/...or/bin/main/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.class
diff --git a/.../main/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.class b/.../main/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.class
diff --git a/...n/test/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.class b/...n/test/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.class
diff --git a/...rc/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java b/...rc/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java
@@ -15,6 +15,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -35,6 +36,7 @@ public class KeyValueProcessor extends AbstractProcessor<Record<Event>, Record<E
     private final Pattern fieldDelimiterPattern;
     private final Pattern keyValueDelimiterPattern;
     private final Set<String> includeKeysSet = new HashSet<String>();
+    private final Set<String> validTransformOptionSet = new HashSet<>(Arrays.asList("lowercase", "uppercase", "capitalize"));
 
     @DataPrepperPluginConstructor
     public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProcessorConfig keyValueProcessorConfig) {
@@ -94,6 +96,13 @@ public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProces
         if(keyValueProcessorConfig.getIncludeKeys() != null) {
             includeKeysSet.addAll(keyValueProcessorConfig.getIncludeKeys());
         }
+
+        if(keyValueProcessorConfig.getTransformKey() != null
+                && !keyValueProcessorConfig.getTransformKey().isEmpty()) {
+            if(!validTransformOptionSet.contains(keyValueProcessorConfig.getTransformKey())) {
+                throw new IllegalArgumentException("transform_key is not a valid option");
+            }
+        }
     }
 
     private String buildRegexFromCharacters(String s) {
@@ -162,6 +171,11 @@ public Collection<Record<Event>> doExecute(final Collection<Record<Event>> recor
                     value = ((String)value).replaceAll(keyValueProcessorConfig.getDeleteValueRegex(), "");
                 }
 
+                if(keyValueProcessorConfig.getTransformKey() != null
+                        && !keyValueProcessorConfig.getTransformKey().isEmpty()) {
+                    key = transformKey(key);
+                }
+
                 addKeyValueToMap(parsedMap, key, value);
             }
 
@@ -171,6 +185,17 @@ public Collection<Record<Event>> doExecute(final Collection<Record<Event>> recor
         return records;
     }
 
+    private String transformKey(String key) {
+        if(keyValueProcessorConfig.getTransformKey().equals("lowercase")) {
+            key = key.toLowerCase();
+        } else if(keyValueProcessorConfig.getTransformKey().equals("uppercase")) {
+            key = key.substring(0, 1).toUpperCase() + key.substring(1);
+        } else if(keyValueProcessorConfig.getTransformKey().equals("capitalize")) {
+            key = key.toUpperCase();
+        }
+        return key;
+    }
+
     private void addKeyValueToMap(final Map<String, Object> parsedMap, final String key, final Object value) {
         if(!parsedMap.containsKey(key)) {
             parsedMap.put(key, value);

diff --git a/...n/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java b/...n/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java
@@ -22,6 +22,7 @@ public class KeyValueProcessorConfig {
     static final String DEFAULT_PREFIX = "";
     static final String DEFAULT_DELETE_KEY_REGEX = "";
     static final String DEFAULT_DELETE_VALUE_REGEX = "";
+    static final String DEFAULT_TRANSFORM_KEY = "";
 
     @NotEmpty
     private String source = DEFAULT_SOURCE;
@@ -60,6 +61,10 @@ public class KeyValueProcessorConfig {
     @NotNull
     private String deleteValueRegex = DEFAULT_DELETE_VALUE_REGEX;
 
+    @JsonProperty("transform_key")
+    @NotNull
+    private String transformKey = DEFAULT_TRANSFORM_KEY;
+
     public String getSource() {
         return source;
     }
@@ -103,4 +108,8 @@ public String getDeleteKeyRegex() {
     public String getDeleteValueRegex() {
         return deleteValueRegex;
     }
+
+    public String getTransformKey() {
+        return transformKey;
+    }
 }
diff --git a/...st/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java b/...st/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java
@@ -63,6 +63,7 @@ void setup() {
         lenient().when(mockConfig.getPrefix()).thenReturn(defaultConfig.getPrefix());
         lenient().when(mockConfig.getDeleteKeyRegex()).thenReturn(defaultConfig.getDeleteKeyRegex());
         lenient().when(mockConfig.getDeleteValueRegex()).thenReturn(defaultConfig.getDeleteValueRegex());
+        lenient().when(mockConfig.getTransformKey()).thenReturn(defaultConfig.getTransformKey());
 
         keyValueProcessor = new KeyValueProcessor(pluginMetrics, mockConfig);
     }
@@ -371,6 +372,36 @@ void testDeleteValueAndKeyRegexKvProcessor() {
         assertThatKeyEquals(parsed_message, "key2", "value2");
     }
 
+    @Test
+    void testLowercaseTransformKvProcessor() {
+        final Record<Event> record = getMessage("Key1=value1");
+        final List<Record<Event>> editedRecords = (List<Record<Event>>) keyValueProcessor.doExecute(Collections.singletonList(record));
+        final LinkedHashMap<String, Object> parsed_message = getLinkedHashMap(editedRecords);
+
+        assertThat(parsed_message.size(), equalTo(2));
+        assertThatKeyEquals(parsed_message, "key1", "value1");
+    }
+
+    @Test
+    void testUppercaseTransformKvProcessor() {
+        final Record<Event> record = getMessage("key1=value1");
+        final List<Record<Event>> editedRecords = (List<Record<Event>>) keyValueProcessor.doExecute(Collections.singletonList(record));
+        final LinkedHashMap<String, Object> parsed_message = getLinkedHashMap(editedRecords);
+
+        assertThat(parsed_message.size(), equalTo(2));
+        assertThatKeyEquals(parsed_message, "Key1", "value1");
+    }
+
+    @Test
+    void testCapitalizeTransformKvProcessor() {
+        final Record<Event> record = getMessage("key1=value1");
+        final List<Record<Event>> editedRecords = (List<Record<Event>>) keyValueProcessor.doExecute(Collections.singletonList(record));
+        final LinkedHashMap<String, Object> parsed_message = getLinkedHashMap(editedRecords);
+
+        assertThat(parsed_message.size(), equalTo(2));
+        assertThatKeyEquals(parsed_message, "KEY1", "value1");
+    }
+
     @Test
     void testShutdownIsReady() {
         assertThat(keyValueProcessor.isReadyForShutdown(), is(true));