Merge pull request #1190 from engix-ltd/master

Fixed and improved default JSON MQTT uplink converter.
thingsboard · Nov 14, 2023 · ae1e305 · ae1e305
2 parents 6eada6a + 7e7a8ea
commit ae1e305
Show file tree

Hide file tree

Showing 3 changed files with 88 additions and 12 deletions.
diff --git a/tests/converters/test_mqtt_json_uplink_converter.py b/tests/converters/test_mqtt_json_uplink_converter.py
@@ -78,6 +78,39 @@ def test_with_send_on_change_option_enabled(self):
         converted_array_data = converter.convert(topic, data)
         self.assertTrue(converted_array_data.get(SEND_ON_CHANGE_PARAMETER))
 
+    def test_parse_device_name_from_spaced_key_name(self):
+        device_key_name = "device name"
+
+        topic, config, data = self._get_device_test_data_with_spaced_key_and_different_out_type(device_key_name)
+        converter = JsonMqttUplinkConverter(config, logger=logging.getLogger('converter'))
+        converted_data = converter.convert(topic, data)
+
+        self.assertEqual(data[device_key_name], converted_data["deviceName"])
+
+    def test_convert_data_from_string_to_int_without_eval(self):
+        use_eval = False
+        device_key_name = "device name"
+        attr_key_name = "test_key"
+
+        topic, config, data = self._get_device_test_data_with_spaced_key_and_different_out_type(
+                                                                device_key_name, attr_key_name, use_eval)
+        converter = JsonMqttUplinkConverter(config, logger=logging.getLogger('converter'))
+        converted_data = converter.convert(topic, data)
+
+        self.assertEqual(converted_data[ATTRIBUTES_PARAMETER][0][attr_key_name], int(float(data[attr_key_name])))
+
+    def test_convert_data_from_string_to_int_with_eval(self):
+        use_eval = True
+        device_key_name = "device name"
+        attr_key_name = "test_key"
+
+        topic, config, data = self._get_device_test_data_with_spaced_key_and_different_out_type(
+                                                                device_key_name, attr_key_name, use_eval)
+        converter = JsonMqttUplinkConverter(config, logger=logging.getLogger('converter'))
+        converted_data = converter.convert(topic, data)
+
+        self.assertEqual(converted_data[ATTRIBUTES_PARAMETER][0][attr_key_name], 2 * int(float(data[attr_key_name])))
+
     @staticmethod
     def _convert_to_dict(data_array):
         data_dict = {}
@@ -128,6 +161,32 @@ def _get_device_2_test_data(self):
         }
         return topic, config, data
 
+    def _get_device_test_data_with_spaced_key_and_different_out_type(self, device_name_key, attr_key_name="test_key", use_eval=False):
+        topic = "topic"
+        value_expression = "${" + attr_key_name + "}"
+        config = {
+            "topicFilter": topic,
+            "converter": {
+                "type": "json",
+                "useEval": use_eval,
+                "deviceNameJsonExpression": "${" + device_name_key + "}",
+                "deviceTypeJsonExpression": self.DEVICE_TYPE,
+                "attributes": [
+                    {
+                        "type": "int",
+                        "key": attr_key_name,
+                        "value": f"{value_expression} + {value_expression}" if use_eval else value_expression
+                    }
+                ],
+                "timeseries": []
+            }
+        }
+        data = {
+          device_name_key: self.DEVICE_NAME,
+          attr_key_name: "21.420000"
+        }
+        return topic, config, data
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/thingsboard_gateway/connectors/mqtt/json_mqtt_uplink_converter.py b/thingsboard_gateway/connectors/mqtt/json_mqtt_uplink_converter.py
@@ -23,10 +23,13 @@
 
 
 class JsonMqttUplinkConverter(MqttUplinkConverter):
+    CONFIGURATION_OPTION_USE_EVAL = "useEval"
+
     def __init__(self, config, logger):
         self._log = logger
         self.__config = config.get('converter')
         self.__send_data_on_change = self.__config.get(SEND_ON_CHANGE_PARAMETER)
+        self.__use_eval = self.__config.get(self.CONFIGURATION_OPTION_USE_EVAL, False)
 
     @property
     def config(self):
@@ -82,22 +85,18 @@ def _convert_single_item(self, topic, data):
 
                         full_key = datatype_config["key"]
                         for (key, key_tag) in zip(keys, keys_tags):
-                            is_valid_key = "${" in datatype_config["key"] and "}" in \
-                                           datatype_config["key"]
-                            full_key = full_key.replace('${' + str(key_tag) + '}',
-                                                        str(key)) if is_valid_key else key_tag
+                            is_valid_key = "${" in datatype_config["key"] and "}" in datatype_config["key"]
+                            full_key = full_key.replace('${' + str(key_tag) + '}', str(key)) if is_valid_key else key_tag
 
                         full_value = datatype_config["value"]
                         for (value, value_tag) in zip(values, values_tags):
-                            is_valid_value = "${" in datatype_config["value"] and "}" in \
-                                             datatype_config["value"]
-
-                            full_value = full_value.replace('${' + str(value_tag) + '}',
-                                                            str(value)) if is_valid_value else value
+                            is_valid_value = "${" in datatype_config["value"] and "}" in datatype_config["value"]
+                            full_value = full_value.replace('${' + str(value_tag) + '}', str(value)) if is_valid_value else value
 
                         if full_key != 'None' and full_value != 'None':
                             dict_result[datatypes[datatype]].append(
-                                self.create_timeseries_record(full_key, full_value, timestamp))
+                                self.create_timeseries_record(full_key, TBUtility.convert_data_type(
+                                    full_value, datatype_config["type"], self.__use_eval), timestamp))
         except Exception as e:
             self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config),
                             str(data), e)

diff --git a/thingsboard_gateway/tb_utility/tb_utility.py b/thingsboard_gateway/tb_utility/tb_utility.py
@@ -108,7 +108,8 @@ def get_value(expression, body=None, value_type="string", get_tag=False, express
                     full_value = body.get(target_str.split()[0])
             elif isinstance(body, (dict, list)):
                 try:
-                    jsonpath_expression = parse(target_str)
+                    # Wrap in quotes to support key name with spaces
+                    jsonpath_expression = parse('"' + target_str + '"')
                     jsonpath_match = jsonpath_expression.find(body)
                     if jsonpath_match:
                         full_value = jsonpath_match[0].value
@@ -126,7 +127,7 @@ def get_value(expression, body=None, value_type="string", get_tag=False, express
 
     @staticmethod
     def get_values(expression, body=None, value_type="string", get_tag=False, expression_instead_none=False):
-        expression_arr = findall(r'\$\{[${A-Za-z0-9.^\]\[*_:]*\}', expression)
+        expression_arr = findall(r'\$\{[${A-Za-z0-9. ^\]\[*_:]*\}', expression)
 
         values = [TBUtility.get_value(exp, body, value_type=value_type, get_tag=get_tag,
                                       expression_instead_none=expression_instead_none) for exp in expression_arr]
@@ -219,3 +220,20 @@ def check_certificate(certificate, key=None, generate_new=True, days_left=3):
                 return TBUtility.generate_certificate(certificate, key, cert_detail)
             else:
                 return True
+
+    @staticmethod
+    def convert_data_type(data, new_type, use_eval=False):
+        current_type = type(data)
+        # use 'in' check instead of equality for such case like 'str' and 'string'
+        if current_type.__name__ in new_type:
+            return data
+
+        evaluated_data = eval(data, globals(), {}) if use_eval else data
+        if 'int' in new_type or 'long' in new_type:
+            return int(float(evaluated_data))
+        elif 'float' in new_type or 'double' in new_type:
+            return float(evaluated_data)
+        elif 'bool' in new_type:
+            return bool(evaluated_data)
+        else:
+            return str(evaluated_data)