Added backing up configuration on remote configurator start and conne…

…ctor configuration update

for_build/etc/thingsboard-gateway/config/tb_gateway.json

@@ -0,0 +1,56 @@
+{
+  "thingsboard": {
+    "host": "thingsboard.cloud",
+    "port": 1883,
+    "remoteShell": false,
+    "remoteConfiguration": true,
+    "statistics": {
+      "enable": true,
+      "statsSendPeriodInSeconds": 3600
+    },
+    "deviceFiltering": {
+      "enable": false,
+      "filterFile": "list.json"
+    },
+    "maxPayloadSizeBytes": 1024,
+    "minPackSendDelayMS": 200,
+    "minPackSizeToSend": 500,
+    "checkConnectorsConfigurationInSeconds": 60,
+    "handleDeviceRenaming": true,
+    "security": {
+      "type": "accessToken",
+      "accessToken": "YOUR_ACCESS_TOKEN"
+    },
+    "qos": 1,
+    "checkingDeviceActivity": {
+      "checkDeviceInactivity": false,
+      "inactivityTimeoutSeconds": 200,
+      "inactivityCheckPeriodSeconds": 500
+    }
+  },
+  "storage": {
+    "type": "memory",
+    "read_records_count": 100,
+    "max_records_count": 100000,
+    "data_folder_path": "./data/",
+    "max_file_count": 10,
+    "max_read_records_count": 10,
+    "max_records_per_file": 10000,
+    "data_file_path": "./data/data.db",
+    "messages_ttl_check_in_hours": 1,
+    "messages_ttl_in_days": 7
+  },
+  "grpc": {
+    "enabled": false,
+    "serverPort": 9595,
+    "keepaliveTimeMs": 10000,
+    "keepaliveTimeoutMs": 5000,
+    "keepalivePermitWithoutCalls": true,
+    "maxPingsWithoutData": 0,
+    "minTimeBetweenPingsMs": 10000,
+    "minPingIntervalWithoutDataMs": 5000,
+    "keepAliveTimeMs": 10000,
+    "keepAliveTimeoutMs": 5000
+  },
+  "connectors": []
+}

thingsboard_gateway/config/logs.json

@@ -71,8 +71,8 @@
       "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler",
       "formatter": "LogFormatter",
       "filename": "./logs/extension.log",
-      "backupCount": 8,
-      "interval": 5,
+      "backupCount": 7,
+      "interval": 3,
       "when": "D",
       "encoding": "utf-8"
     }

thingsboard_gateway/config/tb_gateway.json

@@ -2,20 +2,20 @@
   "thingsboard": {
     "host": "thingsboard.cloud",
     "port": 1883,
-    "remoteShell": true,
+    "remoteShell": false,
     "remoteConfiguration": true,
     "statistics": {
       "enable": true,
-      "statsSendPeriodInSeconds": 60
+      "statsSendPeriodInSeconds": 3600
     },
     "deviceFiltering": {
       "enable": false,
       "filterFile": "list.json"
     },
     "maxPayloadSizeBytes": 1024,
-    "minPackSendDelayMS": 60,
+    "minPackSendDelayMS": 200,
     "minPackSizeToSend": 500,
-    "checkConnectorsConfigurationInSeconds": 10,
+    "checkConnectorsConfigurationInSeconds": 60,
     "handleDeviceRenaming": true,
     "security": {
       "type": "accessToken",
@@ -31,7 +31,7 @@
   "storage": {
     "type": "memory",
     "read_records_count": 100,
-    "max_records_count": 10000,
+    "max_records_count": 100000,
     "data_folder_path": "./data/",
     "max_file_count": 10,
     "max_read_records_count": 10,

thingsboard_gateway/gateway/tb_gateway_service.py

@@ -17,7 +17,6 @@
 import logging.handlers
 import multiprocessing.managers
 import os.path
-from signal import signal, SIGINT
 import subprocess
 from os import execv, listdir, path, pathsep, stat, system, environ
 from platform import system as platform_system
@@ -44,12 +43,12 @@
 from thingsboard_gateway.storage.memory.memory_event_storage import MemoryEventStorage
 from thingsboard_gateway.storage.sqlite.sqlite_event_storage import SQLiteEventStorage
 from thingsboard_gateway.tb_utility.tb_gateway_remote_configurator import RemoteConfigurator
-from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader
 from thingsboard_gateway.tb_utility.tb_handler import TBLoggerHandler
+from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader
+from thingsboard_gateway.tb_utility.tb_logger import TbLogger
 from thingsboard_gateway.tb_utility.tb_remote_shell import RemoteShell
 from thingsboard_gateway.tb_utility.tb_updater import TBUpdater
 from thingsboard_gateway.tb_utility.tb_utility import TBUtility
-from thingsboard_gateway.tb_utility.tb_logger import TbLogger
 
 GRPC_LOADED = False
 try:
@@ -339,7 +338,7 @@ def __load_general_config(config_file):
                         'Please, use JSON configuration instead.')
             log.warning(
                 'See default configuration on '
-                'https://thingsboard.io/docs/iot-gateway/configuration/?storageConfig=sqlite#storage-configuration')
+                'https://thingsboard.io/docs/iot-gateway/configuration/')
 
             config = {}
             try:

thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py

@@ -52,6 +52,7 @@ def __init__(self, gateway, config):
             'logs_configuration': 'logs.json'
         }
         LOG.info('Remote Configurator started')
+        self.create_configuration_file_backup(config, "tb_gateway.json")
 
     @property
     def general_configuration(self):
@@ -379,20 +380,21 @@ def _handle_connector_configuration_update(self, config):
         LOG.debug('Processing connectors configuration update...')
 
         try:
-            configuration = config['configuration']
+            config_file_name = config['configuration']
 
             found_connectors = list(filter(lambda item: item['name'] == config['name'], self.connectors_configuration))
             if not found_connectors:
                 connector_configuration = {'name': config['name'], 'type': config['type'],
-                                           'configuration': configuration}
+                                           'configuration': config_file_name}
                 if config.get('key'):
                     connector_configuration['key'] = config['key']
 
                 if config.get('class'):
                     connector_configuration['class'] = config['class']
 
-                with open(self._gateway.get_config_path() + configuration, 'w') as file:
+                with open(self._gateway.get_config_path() + config_file_name, 'w') as file:
                     config['configurationJson'].update({'logLevel': config['logLevel'], 'name': config['name']})
+                    self.create_configuration_file_backup(config, config_file_name)
                     file.writelines(dumps(config['configurationJson'], indent='  '))
 
                 self.connectors_configuration.append(connector_configuration)
@@ -405,13 +407,15 @@ def _handle_connector_configuration_update(self, config):
                 found_connector = found_connectors[0]
                 changed = False
 
-                config_path = self._gateway.get_config_path() + configuration
-                if os.path.exists(config_path):
-                    with open(config_path, 'r') as file:
-                        conf = load(file)
-                        config_hash = hash(str(conf))
+                config_file_path = self._gateway.get_config_path() + config_file_name
+                if os.path.exists(config_file_path):
+                    connector_config_data = None
+                    with open(config_file_path, 'r') as file:
+                        connector_config_data = load(file)
+                        config_hash = hash(str(connector_config_data))
 
                     if config_hash != hash(str(config['configurationJson'])):
+                        self.create_configuration_file_backup(connector_config_data, config_file_name)
                         changed = True
 
                 connector_configuration = None
@@ -421,7 +425,7 @@ def _handle_connector_configuration_update(self, config):
                         'logLevel') != config.get('logLevel'):
                     changed = True
                     connector_configuration = {'name': config['name'], 'type': config['type'],
-                                               'configuration': configuration}
+                                               'configuration': config_file_name}
 
                     if config.get('key'):
                         connector_configuration['key'] = config['key']
@@ -432,7 +436,7 @@ def _handle_connector_configuration_update(self, config):
                     found_connector.update(connector_configuration)
 
                 if changed:
-                    with open(self._gateway.get_config_path() + configuration, 'w') as file:
+                    with open(self._gateway.get_config_path() + config_file_name, 'w') as file:
                         config['configurationJson'].update({'logLevel': config['logLevel'], 'name': config['name']})
                         file.writelines(dumps(config['configurationJson'], indent='  '))
 
@@ -580,3 +584,14 @@ def _is_modified(self, attr_name, config):
             LOG.warning('File %s not exist', file_path)
 
         return True
+
+    def create_configuration_file_backup(self, config_data, config_file_name):
+        backup_folder_path = self._gateway.get_config_path() + "backup"
+        if not os.path.exists(backup_folder_path):
+            os.mkdir(backup_folder_path)
+
+        backup_file_path = backup_folder_path + os.path.sep + config_file_name + "_backup_" + str(int(time()))
+        with open(backup_file_path, "w") as backup_file:
+            LOG.debug(f"Backup file created for configuration file {config_file_name} in {backup_file_path}")
+            backup_file.writelines(dumps(config_data, indent='  '))
+