From 5deadef49a6316d0e9e5ca5799d66b1bbd7a7e8f Mon Sep 17 00:00:00 2001 From: imbeacon Date: Wed, 11 Oct 2023 11:38:39 +0300 Subject: [PATCH] Added backing up configuration on remote configurator start and connector configuration update --- .../config/tb_gateway.json | 56 +++++++++ .../config/tb_gateway.yaml | 109 ------------------ thingsboard_gateway/config/logs.json | 4 +- thingsboard_gateway/config/tb_gateway.json | 10 +- .../gateway/tb_gateway_service.py | 7 +- .../tb_gateway_remote_configurator.py | 35 ++++-- 6 files changed, 91 insertions(+), 130 deletions(-) create mode 100644 for_build/etc/thingsboard-gateway/config/tb_gateway.json delete mode 100644 for_build/etc/thingsboard-gateway/config/tb_gateway.yaml diff --git a/for_build/etc/thingsboard-gateway/config/tb_gateway.json b/for_build/etc/thingsboard-gateway/config/tb_gateway.json new file mode 100644 index 000000000..b839649da --- /dev/null +++ b/for_build/etc/thingsboard-gateway/config/tb_gateway.json @@ -0,0 +1,56 @@ +{ + "thingsboard": { + "host": "thingsboard.cloud", + "port": 1883, + "remoteShell": false, + "remoteConfiguration": true, + "statistics": { + "enable": true, + "statsSendPeriodInSeconds": 3600 + }, + "deviceFiltering": { + "enable": false, + "filterFile": "list.json" + }, + "maxPayloadSizeBytes": 1024, + "minPackSendDelayMS": 200, + "minPackSizeToSend": 500, + "checkConnectorsConfigurationInSeconds": 60, + "handleDeviceRenaming": true, + "security": { + "type": "accessToken", + "accessToken": "YOUR_ACCESS_TOKEN" + }, + "qos": 1, + "checkingDeviceActivity": { + "checkDeviceInactivity": false, + "inactivityTimeoutSeconds": 200, + "inactivityCheckPeriodSeconds": 500 + } + }, + "storage": { + "type": "memory", + "read_records_count": 100, + "max_records_count": 100000, + "data_folder_path": "./data/", + "max_file_count": 10, + "max_read_records_count": 10, + "max_records_per_file": 10000, + "data_file_path": "./data/data.db", + "messages_ttl_check_in_hours": 1, + "messages_ttl_in_days": 7 + }, + "grpc": { + "enabled": false, + "serverPort": 9595, + "keepaliveTimeMs": 10000, + "keepaliveTimeoutMs": 5000, + "keepalivePermitWithoutCalls": true, + "maxPingsWithoutData": 0, + "minTimeBetweenPingsMs": 10000, + "minPingIntervalWithoutDataMs": 5000, + "keepAliveTimeMs": 10000, + "keepAliveTimeoutMs": 5000 + }, + "connectors": [] +} \ No newline at end of file diff --git a/for_build/etc/thingsboard-gateway/config/tb_gateway.yaml b/for_build/etc/thingsboard-gateway/config/tb_gateway.yaml deleted file mode 100644 index 4ee7ea31c..000000000 --- a/for_build/etc/thingsboard-gateway/config/tb_gateway.yaml +++ /dev/null @@ -1,109 +0,0 @@ -thingsboard: - host: thingsboard.cloud - port: 1883 - remoteShell: false - remoteConfiguration: false - statsSendPeriodInSeconds: 3600 - minPackSendDelayMS: 0 - checkConnectorsConfigurationInSeconds: 60 - security: - accessToken: PUT_YOUR_GW_ACCESS_TOKEN_HERE - qos: 1 -storage: - type: memory - read_records_count: 100 - max_records_count: 100000 -# type: file -# data_folder_path: ./data/ -# max_file_count: 10 -# max_read_records_count: 10 -# max_records_per_file: 10000 -# type: sqlite -# data_file_path: ./data/data.db -# messages_ttl_check_in_hours: 1 -# messages_ttl_in_days: 7 -grpc: - enabled: false - serverPort: 9595 - keepaliveTimeMs: 10000 - keepaliveTimeoutMs: 5000 - keepalivePermitWithoutCalls: true - maxPingsWithoutData: 0 - minTimeBetweenPingsMs: 10000 - minPingIntervalWithoutDataMs: 5000 -connectors: - - - name: MQTT Broker Connector - type: mqtt - configuration: mqtt.json - -# - -# name: Modbus Connector -# type: modbus -# configuration: modbus.json -# -# - -# name: Modbus Connector -# type: modbus -# configuration: modbus_serial.json -# -# - -# name: OPC-UA Connector -# type: opcua -# configuration: opcua.json -# -# - -# name: BLE Connector -# type: ble -# configuration: ble.json -# -# - -# name: REQUEST Connector -# type: request -# configuration: request.json -# -# - -# name: CAN Connector -# type: can -# configuration: can.json -# -# - -# name: BACnet Connector -# type: bacnet -# configuration: bacnet.json -# -# - -# name: ODBC Connector -# type: odbc -# configuration: odbc.json -# -# - -# name: REST Connector -# type: rest -# configuration: rest.json -# -# - -# name: SNMP Connector -# type: snmp -# configuration: snmp.json -# -# - -# name: FTP Connector -# type: ftp -# configuration: ftp.json -# -# -# ========= Customization ========== -# -# -# - -# name: Custom Serial Connector -# type: serial -# configuration: custom_serial.json -# class: CustomSerialConnector -# -# - -# name: GRPC Connector 1 -# key: auto -# type: grpc -# configuration: grpc_connector_1.json diff --git a/thingsboard_gateway/config/logs.json b/thingsboard_gateway/config/logs.json index 1965d1c0a..797137547 100644 --- a/thingsboard_gateway/config/logs.json +++ b/thingsboard_gateway/config/logs.json @@ -71,8 +71,8 @@ "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", "formatter": "LogFormatter", "filename": "./logs/extension.log", - "backupCount": 8, - "interval": 5, + "backupCount": 7, + "interval": 3, "when": "D", "encoding": "utf-8" } diff --git a/thingsboard_gateway/config/tb_gateway.json b/thingsboard_gateway/config/tb_gateway.json index 3799ab770..b839649da 100644 --- a/thingsboard_gateway/config/tb_gateway.json +++ b/thingsboard_gateway/config/tb_gateway.json @@ -2,20 +2,20 @@ "thingsboard": { "host": "thingsboard.cloud", "port": 1883, - "remoteShell": true, + "remoteShell": false, "remoteConfiguration": true, "statistics": { "enable": true, - "statsSendPeriodInSeconds": 60 + "statsSendPeriodInSeconds": 3600 }, "deviceFiltering": { "enable": false, "filterFile": "list.json" }, "maxPayloadSizeBytes": 1024, - "minPackSendDelayMS": 60, + "minPackSendDelayMS": 200, "minPackSizeToSend": 500, - "checkConnectorsConfigurationInSeconds": 10, + "checkConnectorsConfigurationInSeconds": 60, "handleDeviceRenaming": true, "security": { "type": "accessToken", @@ -31,7 +31,7 @@ "storage": { "type": "memory", "read_records_count": 100, - "max_records_count": 10000, + "max_records_count": 100000, "data_folder_path": "./data/", "max_file_count": 10, "max_read_records_count": 10, diff --git a/thingsboard_gateway/gateway/tb_gateway_service.py b/thingsboard_gateway/gateway/tb_gateway_service.py index e5a86c18b..8ae266194 100644 --- a/thingsboard_gateway/gateway/tb_gateway_service.py +++ b/thingsboard_gateway/gateway/tb_gateway_service.py @@ -17,7 +17,6 @@ import logging.handlers import multiprocessing.managers import os.path -from signal import signal, SIGINT import subprocess from os import execv, listdir, path, pathsep, stat, system, environ from platform import system as platform_system @@ -44,12 +43,12 @@ from thingsboard_gateway.storage.memory.memory_event_storage import MemoryEventStorage from thingsboard_gateway.storage.sqlite.sqlite_event_storage import SQLiteEventStorage from thingsboard_gateway.tb_utility.tb_gateway_remote_configurator import RemoteConfigurator -from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader from thingsboard_gateway.tb_utility.tb_handler import TBLoggerHandler +from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader +from thingsboard_gateway.tb_utility.tb_logger import TbLogger from thingsboard_gateway.tb_utility.tb_remote_shell import RemoteShell from thingsboard_gateway.tb_utility.tb_updater import TBUpdater from thingsboard_gateway.tb_utility.tb_utility import TBUtility -from thingsboard_gateway.tb_utility.tb_logger import TbLogger GRPC_LOADED = False try: @@ -339,7 +338,7 @@ def __load_general_config(config_file): 'Please, use JSON configuration instead.') log.warning( 'See default configuration on ' - 'https://thingsboard.io/docs/iot-gateway/configuration/?storageConfig=sqlite#storage-configuration') + 'https://thingsboard.io/docs/iot-gateway/configuration/') config = {} try: diff --git a/thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py b/thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py index c966b00d1..a511c7936 100644 --- a/thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py +++ b/thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py @@ -52,6 +52,7 @@ def __init__(self, gateway, config): 'logs_configuration': 'logs.json' } LOG.info('Remote Configurator started') + self.create_configuration_file_backup(config, "tb_gateway.json") @property def general_configuration(self): @@ -379,20 +380,21 @@ def _handle_connector_configuration_update(self, config): LOG.debug('Processing connectors configuration update...') try: - configuration = config['configuration'] + config_file_name = config['configuration'] found_connectors = list(filter(lambda item: item['name'] == config['name'], self.connectors_configuration)) if not found_connectors: connector_configuration = {'name': config['name'], 'type': config['type'], - 'configuration': configuration} + 'configuration': config_file_name} if config.get('key'): connector_configuration['key'] = config['key'] if config.get('class'): connector_configuration['class'] = config['class'] - with open(self._gateway.get_config_path() + configuration, 'w') as file: + with open(self._gateway.get_config_path() + config_file_name, 'w') as file: config['configurationJson'].update({'logLevel': config['logLevel'], 'name': config['name']}) + self.create_configuration_file_backup(config, config_file_name) file.writelines(dumps(config['configurationJson'], indent=' ')) self.connectors_configuration.append(connector_configuration) @@ -405,13 +407,15 @@ def _handle_connector_configuration_update(self, config): found_connector = found_connectors[0] changed = False - config_path = self._gateway.get_config_path() + configuration - if os.path.exists(config_path): - with open(config_path, 'r') as file: - conf = load(file) - config_hash = hash(str(conf)) + config_file_path = self._gateway.get_config_path() + config_file_name + if os.path.exists(config_file_path): + connector_config_data = None + with open(config_file_path, 'r') as file: + connector_config_data = load(file) + config_hash = hash(str(connector_config_data)) if config_hash != hash(str(config['configurationJson'])): + self.create_configuration_file_backup(connector_config_data, config_file_name) changed = True connector_configuration = None @@ -421,7 +425,7 @@ def _handle_connector_configuration_update(self, config): 'logLevel') != config.get('logLevel'): changed = True connector_configuration = {'name': config['name'], 'type': config['type'], - 'configuration': configuration} + 'configuration': config_file_name} if config.get('key'): connector_configuration['key'] = config['key'] @@ -432,7 +436,7 @@ def _handle_connector_configuration_update(self, config): found_connector.update(connector_configuration) if changed: - with open(self._gateway.get_config_path() + configuration, 'w') as file: + with open(self._gateway.get_config_path() + config_file_name, 'w') as file: config['configurationJson'].update({'logLevel': config['logLevel'], 'name': config['name']}) file.writelines(dumps(config['configurationJson'], indent=' ')) @@ -580,3 +584,14 @@ def _is_modified(self, attr_name, config): LOG.warning('File %s not exist', file_path) return True + + def create_configuration_file_backup(self, config_data, config_file_name): + backup_folder_path = self._gateway.get_config_path() + "backup" + if not os.path.exists(backup_folder_path): + os.mkdir(backup_folder_path) + + backup_file_path = backup_folder_path + os.path.sep + config_file_name + "_backup_" + str(int(time())) + with open(backup_file_path, "w") as backup_file: + LOG.debug(f"Backup file created for configuration file {config_file_name} in {backup_file_path}") + backup_file.writelines(dumps(config_data, indent=' ')) +