Merge pull request #1529 from samson0v/tests

Fixes for tests
thingsboard · Sep 19, 2024 · 1f289e1 · 1f289e1
2 parents cbbe8af + 2659ab9
commit 1f289e1
Show file tree

Hide file tree

Showing 15 changed files with 57 additions and 50 deletions.
diff --git a/tests/integration/connectors/modbus/test_modbus_connector.py b/tests/integration/connectors/modbus/test_modbus_connector.py
@@ -2,10 +2,13 @@
 import unittest
 from os import path
 from time import sleep
-from unittest.mock import Mock
+from unittest.mock import Mock, patch
 
 from simplejson import load
 
+from thingsboard_gateway.gateway.tb_client import TBClient
+from thingsboard_gateway.tb_utility.tb_logger import TbLogger
+
 try:
     from pymodbus.client import ModbusTcpClient as ModbusClient
 except (ImportError, ModuleNotFoundError):
@@ -27,26 +30,34 @@ class ModbusConnectorTestsBase(BaseTest):
 
     def setUp(self) -> None:
         super().setUp()
+        self.tb_client = Mock(spec=TBClient)
         self.gateway = Mock(spec=TBGatewayService)
+        self.gateway.tb_client = self.tb_client
+        self.tb_logger = Mock(spec=TbLogger)
         self.gateway.get_devices.return_value = []
         self.connector = None
         self.config = None
 
     def tearDown(self):
-        super().tearDown()
         self.connector.close()
+        super().tearDown()
 
-    def _create_connector(self, config_file_name):
-        with open(self.CONFIG_PATH + config_file_name, 'r', encoding="UTF-8") as file:
-            self.config = load(file)
-            self.config['master']['slaves'][0]['uplink_converter'] = BytesModbusUplinkConverter(
-                {**self.config['master']['slaves'][0], 'deviceName': 'Test'}, logger=logging.getLogger('converter'))
-            self.config['master']['slaves'][0]['downlink_converter'] = BytesModbusDownlinkConverter(
-                {**self.config['master']['slaves'][0], 'deviceName': 'Test'}, logger=logging.getLogger('converter'))
-            self.connector = ModbusConnector(self.gateway, self.config, "modbus")
-            self.connector._ModbusConnector__log = Mock()
-            self.connector.open()
-            sleep(1)  # some time to init
+    @patch('thingsboard_gateway.tb_utility.tb_logger.init_logger')
+    def _create_connector(self, config_file_name, test_patch):
+        test_patch.return_value = self.tb_logger
+        try:
+            with open(self.CONFIG_PATH + config_file_name, 'r', encoding="UTF-8") as file:
+                self.config = load(file)
+                self.config['master']['slaves'][0]['uplink_converter'] = BytesModbusUplinkConverter(
+                    {**self.config['master']['slaves'][0], 'deviceName': 'Test'}, logger=logging.getLogger('converter'))
+                self.config['master']['slaves'][0]['downlink_converter'] = BytesModbusDownlinkConverter(
+                    {**self.config['master']['slaves'][0], 'deviceName': 'Test'}, logger=logging.getLogger('converter'))
+                self.connector = ModbusConnector(self.gateway, self.config, "modbus")
+                self.connector._ModbusConnector__log = Mock()
+                self.connector.open()
+                sleep(1)  # some time to init
+        except Exception as e:
+            self.log.error("Error occurred during creating connector: %s", e)
 
 
 class ModbusReadRegisterTypesTests(ModbusConnectorTestsBase):

diff --git a/tests/integration/data/modbus/modbus_server.py b/tests/integration/data/modbus/modbus_server.py
@@ -67,7 +67,7 @@ def run_server():
         0x05: store1
     }
     context = ModbusServerContext(slaves=slaves, single=False)
-    StartTcpServer(context=context, address=("127.0.0.1", 5021))
+    StartTcpServer(context=context, address=("localhost", 5021))
 
 
 if __name__ == "__main__":

diff --git a/tests/unit/connectors/mqtt/data/attribute_requests/new_config.json b/tests/unit/connectors/mqtt/data/attribute_requests/new_config.json
@@ -12,7 +12,7 @@
       "type": "anonymous"
     }
   },
-  "dataMapping": [],
+  "mapping": [],
   "requestsMapping": {
     "serverSideRpc": {},
     "connectRequests": {},

diff --git a/tests/unit/connectors/mqtt/data/attribute_updates/new_config.json b/tests/unit/connectors/mqtt/data/attribute_updates/new_config.json
@@ -12,7 +12,7 @@
       "type": "anonymous"
     }
   },
-  "dataMapping": [],
+  "mapping": [],
   "requestsMapping": {
     "serverSideRpc": {},
     "connectRequests": {},

diff --git a/tests/unit/connectors/mqtt/data/connect_requests/new_config.json b/tests/unit/connectors/mqtt/data/connect_requests/new_config.json
@@ -12,7 +12,7 @@
       "type": "anonymous"
     }
   },
-  "dataMapping": [],
+  "mapping": [],
   "requestsMapping": {
     "serverSideRpc": {},
     "connectRequests": [

diff --git a/tests/unit/connectors/mqtt/data/disconnect_requests/new_config.json b/tests/unit/connectors/mqtt/data/disconnect_requests/new_config.json
@@ -12,7 +12,7 @@
       "type": "anonymous"
     }
   },
-  "dataMapping": [],
+  "mapping": [],
   "requestsMapping": {
     "serverSideRpc": {},
     "connectRequests": {},

diff --git a/tests/unit/connectors/mqtt/data/mapping/new_config.json b/tests/unit/connectors/mqtt/data/mapping/new_config.json
@@ -12,7 +12,7 @@
       "type": "anonymous"
     }
   },
-  "dataMapping": [
+  "mapping": [
     {
       "topicFilter": "sensor/data",
       "converter": {

diff --git a/tests/unit/connectors/mqtt/data/server_side_rpc/new_config.json b/tests/unit/connectors/mqtt/data/server_side_rpc/new_config.json
@@ -12,7 +12,7 @@
       "type": "anonymous"
     }
   },
-  "dataMapping": [],
+  "mapping": [],
   "requestsMapping": {
     "serverSideRpc": [
       {

diff --git a/tests/unit/connectors/mqtt/test_mqtt_backward_compatibility_adapter.py b/tests/unit/connectors/mqtt/test_mqtt_backward_compatibility_adapter.py
@@ -23,7 +23,7 @@ def setUp(self):
     def test_convert_with_empty_config(self):
         result = self.adapter.convert()
         expected_result = {
-            'dataMapping': {},
+            'mapping': {},
             'requestsMapping': {
                 'attributeRequests': {},
                 'attributeUpdates': {},
@@ -32,7 +32,7 @@ def test_convert_with_empty_config(self):
                 'serverSideRpc': {}
             }
         }
-        self.assertEqual(result, expected_result)
+        self.assertEqual(expected_result, result)
 
     def test_convert_with_mapping_in_config(self):
         self.adapter._config = self.convert_json(self.CONFIG_PATH + 'mapping/old_config.json')
@@ -71,12 +71,12 @@ def test_convert_with_serverSideRpc_in_config(self):
         self.assertEqual(result, expected_result)
 
     def test_is_old_config_format_with_mapping(self):
-        self.adapter._config = {'mapping': []}
+        self.adapter._config = {'mapping': [{'converter': {}}]}
         result = self.adapter.is_old_config_format(self.adapter._config)
         self.assertTrue(result)
 
     def test_is_old_config_format_without_mapping(self):
-        self.adapter._config = {'mappingData': []}
+        self.adapter._config = {'mapping': []}
         result = self.adapter.is_old_config_format(self.adapter._config)
         self.assertFalse(result)
 

diff --git a/thingsboard_gateway/connectors/mqtt/backward_compatibility_adapter.py b/thingsboard_gateway/connectors/mqtt/backward_compatibility_adapter.py
@@ -1,16 +1,17 @@
-from copy import copy
+from copy import deepcopy
 
 
 class BackwardCompatibilityAdapter:
     def __init__(self, config):
-        self._config = copy(config)
+        self._config = deepcopy(config)
 
     def convert(self):
         self._config['requestsMapping'] = {}
         config_mapping = {'requestsMapping': ('connectRequests', 'disconnectRequests', 'attributeRequests',
                                        'attributeUpdates', 'serverSideRpc'),
                    'mapping': ('mapping', 'dataMapping')}
 
+        has_mapping_section = False
         for (map_section, map_type) in config_mapping.items():
             for t in map_type:
                 section_config = self._config.pop(t, {})
@@ -26,11 +27,11 @@ def convert(self):
 
                         if t == 'attributeRequests':
                             self._parce_attribute_info(item)
-                    if not self._config.get(map_section):
-                        if t == 'mapping' or t == 'dataMapping':
-                            self._config[map_section] = section_config
-                        else:
-                            self._config[map_section][t] = section_config
+                    if not has_mapping_section and (t == 'mapping' or t == 'dataMapping'):
+                        self._config[map_section] = section_config
+                        break
+                    else:
+                        self._config[map_section][t] = section_config
                 except KeyError:
                     continue
                 except AttributeError as e:

diff --git a/thingsboard_gateway/connectors/odbc/odbc_connector.py b/thingsboard_gateway/connectors/odbc/odbc_connector.py
@@ -265,6 +265,11 @@ def __process_row(self, row):
                        "telemetry": {} if "timeseries" not in self.__config["mapping"] else
                        self.__converter.convert(self.__config["mapping"]["timeseries"], data)}
 
+            StatisticsService.count_connector_message(self._log.name, 'convertersAttrProduced',
+                                                      count=len(to_send["attributes"]))
+            StatisticsService.count_connector_message(self._log.name, 'convertersTsProduced',
+                                                      count=len(to_send["telemetry"]))
+
             if to_send['telemetry'].get('ts'):
                 to_send['ts'] = to_send['telemetry']['ts']
                 del to_send['telemetry']['ts']

diff --git a/thingsboard_gateway/connectors/odbc/odbc_uplink_converter.py b/thingsboard_gateway/connectors/odbc/odbc_uplink_converter.py
@@ -57,8 +57,4 @@ def convert(self, config, data):
         if data.get('ts'):
             converted_data['ts'] = data.get('ts')
 
-        StatisticsService.count_connector_message(self._log.name, 'convertersAttrProduced',
-                                                  count=len(converted_data["attributes"]))
-        StatisticsService.count_connector_message(self._log.name, 'convertersTsProduced',
-                                                  count=len(converted_data["telemetry"]))
         return converted_data
diff --git a/thingsboard_gateway/gateway/tb_client.py b/thingsboard_gateway/gateway/tb_client.py
@@ -297,16 +297,13 @@ def connect(self, min_reconnect_delay=5):
 
         keep_alive = self.__config.get("keep_alive", 120)
         previous_connection_time = time()
-        try_count = 0
+
         try:
             while not self.client.is_connected() and not self.__stopped:
                 if not self.__paused:
                     if self.__stopped:
                         break
-                    if try_count > 5:
-                        self.__logger.error("Failed to connect to ThingsBoard. Check ThingsBoard is running. Will try to connect later.")
-                        break
-                    self.__logger.debug("connecting to ThingsBoard")
+                    self.__logger.info("Connecting to ThingsBoard...")
                     try:
                         if time() - previous_connection_time > min_reconnect_delay:
                             self.client.connect(keepalive=keep_alive,
@@ -315,10 +312,8 @@ def connect(self, min_reconnect_delay=5):
                         else:
                             sleep(1)
                     except ConnectionRefusedError:
-                        try_count += 1
                         self.__logger.error("Connection refused. Check ThingsBoard is running.")
                     except Exception as e:
-                        try_count
                         self.__logger.exception(e)
                 sleep(1)
         except Exception as e:

diff --git a/thingsboard_gateway/gateway/tb_gateway_service.py b/thingsboard_gateway/gateway/tb_gateway_service.py
@@ -724,10 +724,11 @@ def __process_renamed_gateway_devices(self, renamed_device: dict):
                       renamed_device)
         return {'success': True}
 
-    def __process_remote_configuration(self, new_configuration):
-        if new_configuration is not None and self.__remote_configurator is not None:
+    @staticmethod
+    def __process_remote_configuration(new_configuration):
+        if new_configuration is not None:
             try:
-                self.__remote_configurator.process_config_request(new_configuration)
+                RemoteConfigurator.RECEIVED_UPDATE_QUEUE.put(new_configuration)
             except Exception as e:
                 log.exception(e)
 

diff --git a/thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py b/thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py
@@ -34,6 +34,7 @@
 
 class RemoteConfigurator:
     ALLOWED_BACKUPS_AMOUNT = 10
+    RECEIVED_UPDATE_QUEUE = Queue()
 
     DEFAULT_STATISTICS = {
         'enable': True,
@@ -216,14 +217,11 @@ def _load_logs_configuration(self):
             LOG.warning("Cannot open logs configuration file. Using default logs configuration.")
             return {}
 
-    def process_config_request(self, config):
-        self._request_queue.put(config)
-
     def _process_config_request(self):
         while not self._gateway.stopped:
-            if not self._request_queue.empty():
+            if not RemoteConfigurator.RECEIVED_UPDATE_QUEUE.empty():
                 self.in_process = True
-                config = self._request_queue.get()
+                config = RemoteConfigurator.RECEIVED_UPDATE_QUEUE.get()
                 LOG.info('Configuration update request received.')
                 LOG.debug('Got config update request: %s', config)