diff --git a/tests/connectors/connector_tests_base.py b/tests/connectors/connector_tests_base.py index 563a481d7..3b26f8e9b 100644 --- a/tests/connectors/connector_tests_base.py +++ b/tests/connectors/connector_tests_base.py @@ -24,6 +24,7 @@ from thingsboard_gateway.gateway.tb_gateway_service import DEFAULT_CONNECTORS from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader + logging.basicConfig(level=logging.ERROR, format='%(asctime)s - %(levelname)s - %(module)s - %(lineno)d - %(message)s', datefmt='%Y-%m-%d %H:%M:%S') @@ -32,7 +33,7 @@ class ConnectorTestBase(unittest.TestCase): DATA_PATH = path.join(path.dirname(path.dirname(path.abspath(__file__))), - "data" + path.sep) + "data" + path.sep) def setUp(self): self.gateway = Mock(spec=TBGatewayService) @@ -49,7 +50,6 @@ def _load_data_file(self, filename, connector_type): datafile = load(required_file) return datafile - def _create_connector(self, config_filename, connector_type=None): if connector_type is None: class_name = self.__class__.__name__.lower() diff --git a/tests/connectors/test_can_connector.py b/tests/connectors/test_can_connector.py index ebccab01b..d2d4aa899 100644 --- a/tests/connectors/test_can_connector.py +++ b/tests/connectors/test_can_connector.py @@ -20,7 +20,7 @@ from random import choice, randint, uniform from string import ascii_lowercase from time import sleep -from unittest.mock import Mock +from unittest.mock import Mock, patch from thingsboard_gateway.gateway.tb_gateway_service import TBGatewayService from can import Notifier, BufferedReader, Bus, Message @@ -67,14 +67,16 @@ def _create_bus(self): def _create_connector(self, config_file_name): with open(self.CONFIG_PATH + config_file_name, 'r', encoding="UTF-8") as file: - self.config = load(file) - self.connector = CanConnector(self.gateway, self.config, "can") - self.connector.open() - sleep(1) # some time to init + try: + self.config = load(file) + self.connector = CanConnector(self.gateway, self.config, "can") + self.connector.open() + sleep(1) # some time to init + except Exception as e: + print(e) class CanConnectorPollingTests(CanConnectorTestsBase): - def test_polling_once(self): self._create_connector("polling_once.json") config = self.config["devices"][0]["attributes"][0] diff --git a/tests/connectors/test_odbc_connector.py b/tests/connectors/test_odbc_connector.py index f43396c48..38332f3d0 100644 --- a/tests/connectors/test_odbc_connector.py +++ b/tests/connectors/test_odbc_connector.py @@ -51,7 +51,7 @@ @unittest.skipIf(not IS_ODBC_DRIVER_INSTALLED, - "To run ODBC tests install " + ODBC_DRIVER_WITH_STORED_PROCEDURE + "or" + + "To run ODBC tests install " + ODBC_DRIVER_WITH_STORED_PROCEDURE + " or " + ODBC_DRIVER_WITHOUT_STORED_PROCEDURE + " ODBC driver") class OdbcConnectorTests(unittest.TestCase): CONFIG_PATH = path.join(path.dirname(path.dirname(path.abspath(__file__))), diff --git a/tests/connectors/test_opcua_connector.py b/tests/connectors/test_opcua_connector.py index d021b370d..a6b5acab3 100644 --- a/tests/connectors/test_opcua_connector.py +++ b/tests/connectors/test_opcua_connector.py @@ -49,7 +49,7 @@ def __server_run(self, test_server): class SubHandler(object): def datachange_notification(self, node, val, data): print("Python: New data change event", node, val) - + def event_notification(self, event): print("Python: New event", event) diff --git a/tests/converters/test_bytes_can_downlink_converter.py b/tests/converters/test_bytes_can_downlink_converter.py index 055ace916..ae44f7c84 100644 --- a/tests/converters/test_bytes_can_downlink_converter.py +++ b/tests/converters/test_bytes_can_downlink_converter.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import struct import unittest from random import randint, uniform, choice @@ -23,7 +24,7 @@ class BytesCanDownlinkConverterTests(unittest.TestCase): def setUp(self): - self.converter = BytesCanDownlinkConverter() + self.converter = BytesCanDownlinkConverter(logger=logging.getLogger('converter')) def test_data_in_hex_in_conf(self): expected_can_data = [0, 1, 2, 3] diff --git a/tests/converters/test_bytes_can_uplink_converter.py b/tests/converters/test_bytes_can_uplink_converter.py index cb8fc0138..dd8d88aee 100644 --- a/tests/converters/test_bytes_can_uplink_converter.py +++ b/tests/converters/test_bytes_can_uplink_converter.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import _struct import unittest from math import isclose @@ -24,7 +25,7 @@ class BytesCanUplinkConverterTests(unittest.TestCase): def setUp(self): - self.converter = BytesCanUplinkConverter() + self.converter = BytesCanUplinkConverter(logger=logging.getLogger('converter')) def _has_no_data(self, data): return bool(data is None or not data.get("attributes", []) and not data.get("telemetry", [])) diff --git a/tests/converters/test_bytes_modbus_uplink_converter.py b/tests/converters/test_bytes_modbus_uplink_converter.py index 67d32d623..1af96d18d 100644 --- a/tests/converters/test_bytes_modbus_uplink_converter.py +++ b/tests/converters/test_bytes_modbus_uplink_converter.py @@ -1,3 +1,4 @@ +import logging import unittest from pymodbus.constants import Endian @@ -175,7 +176,7 @@ def __init__(self, registers): {tag: {"input_data": DummyResponse(builder.to_registers()), "data_sent": tag_dict[tag]}}) builder.reset() - converter = BytesModbusUplinkConverter({"deviceName": "Modbus Test", "deviceType": "default", "unitId": 1}) + converter = BytesModbusUplinkConverter({"deviceName": "Modbus Test", "deviceType": "default", "unitId": 1}, logger=logging.getLogger('converter')) result = converter.convert(test_modbus_convert_config, test_modbus_body_to_convert) self.assertDictEqual(result, test_modbus_result) diff --git a/tests/converters/test_mqtt_json_uplink_converter.py b/tests/converters/test_mqtt_json_uplink_converter.py index 7eda62412..a4fae982d 100644 --- a/tests/converters/test_mqtt_json_uplink_converter.py +++ b/tests/converters/test_mqtt_json_uplink_converter.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import unittest from random import randint @@ -25,7 +26,7 @@ class JsonMqttUplinkConverterTests(unittest.TestCase): def test_topic_name_and_type(self): topic, config, data = self._get_device_1_test_data() - converter = JsonMqttUplinkConverter(config) + converter = JsonMqttUplinkConverter(config, logger=logging.getLogger('converter')) converted_data = converter.convert(topic, data) self.assertEqual(self.DEVICE_NAME, converted_data["deviceName"]) @@ -33,7 +34,7 @@ def test_topic_name_and_type(self): def test_json_name_and_type(self): topic, config, data = self._get_device_2_test_data() - converter = JsonMqttUplinkConverter(config) + converter = JsonMqttUplinkConverter(config, logger=logging.getLogger('converter')) converted_data = converter.convert(topic, data) self.assertEqual(self.DEVICE_NAME, converted_data["deviceName"]) @@ -41,15 +42,15 @@ def test_json_name_and_type(self): def test_glob_matching(self): topic, config, data = self._get_device_1_test_data() - converter = JsonMqttUplinkConverter(config) + converter = JsonMqttUplinkConverter(config, logger=logging.getLogger('converter')) converted_data = converter.convert(topic, data) self.assertDictEqual(data, self._convert_to_dict(converted_data.get('telemetry'))) self.assertDictEqual(data, self._convert_to_dict(converted_data.get('attributes'))) def test_array_result(self): topic, config, single_data = self._get_device_1_test_data() - array_data = [single_data, single_data]; - converter = JsonMqttUplinkConverter(config) + array_data = [single_data, single_data] + converter = JsonMqttUplinkConverter(config, logger=logging.getLogger('converter')) converted_array_data = converter.convert(topic, array_data) self.assertTrue(isinstance(converted_array_data, list)) @@ -59,21 +60,21 @@ def test_array_result(self): def test_without_send_on_change_option(self): topic, config, data = self._get_device_1_test_data() - converter = JsonMqttUplinkConverter(config) + converter = JsonMqttUplinkConverter(config, logger=logging.getLogger('converter')) converted_array_data = converter.convert(topic, data) self.assertIsNone(converted_array_data.get(SEND_ON_CHANGE_PARAMETER)) def test_with_send_on_change_option_disabled(self): topic, config, data = self._get_device_1_test_data() config["converter"][SEND_ON_CHANGE_PARAMETER] = False - converter = JsonMqttUplinkConverter(config) + converter = JsonMqttUplinkConverter(config, logger=logging.getLogger('converter')) converted_array_data = converter.convert(topic, data) self.assertFalse(converted_array_data.get(SEND_ON_CHANGE_PARAMETER)) def test_with_send_on_change_option_enabled(self): topic, config, data = self._get_device_1_test_data() config["converter"][SEND_ON_CHANGE_PARAMETER] = True - converter = JsonMqttUplinkConverter(config) + converter = JsonMqttUplinkConverter(config, logger=logging.getLogger('converter')) converted_array_data = converter.convert(topic, data) self.assertTrue(converted_array_data.get(SEND_ON_CHANGE_PARAMETER)) diff --git a/tests/converters/test_odbc_uplink_converter.py b/tests/converters/test_odbc_uplink_converter.py index 106a9be93..a983b0668 100644 --- a/tests/converters/test_odbc_uplink_converter.py +++ b/tests/converters/test_odbc_uplink_converter.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import unittest from random import randint, uniform, choice from string import ascii_lowercase @@ -22,7 +23,7 @@ class OdbcUplinkConverterTests(unittest.TestCase): def setUp(self): - self.converter = OdbcUplinkConverter() + self.converter = OdbcUplinkConverter(logger=logging.getLogger('converter')) self.db_data = {"boolValue": True, "intValue": randint(0, 256), "floatValue": uniform(-3.1415926535, 3.1415926535), diff --git a/thingsboard_gateway/config/logs.conf b/thingsboard_gateway/config/logs.conf deleted file mode 100644 index 4d62ce4da..000000000 --- a/thingsboard_gateway/config/logs.conf +++ /dev/null @@ -1,87 +0,0 @@ -[loggers] -keys=root, service, connector, converter, tb_connection, storage, extension -[handlers] -keys=consoleHandler, serviceHandler, connectorHandler, converterHandler, tb_connectionHandler, storageHandler, extensionHandler -[formatters] -keys=LogFormatter -[logger_root] -level=ERROR -handlers=consoleHandler -[logger_connector] -level=INFO -handlers=connectorHandler -formatter=LogFormatter -qualname=connector -[logger_storage] -level=INFO -handlers=storageHandler -formatter=LogFormatter -qualname=storage -[logger_database] -level=INFO -handlers=databaseHandler -formatter=LogFormatter -qualname=database -[logger_tb_connection] -level=INFO -handlers=tb_connectionHandler -formatter=LogFormatter -qualname=tb_connection -[logger_service] -level=INFO -handlers=serviceHandler -formatter=LogFormatter -qualname=service -[logger_converter] -level=INFO -handlers=converterHandler -formatter=LogFormatter -qualname=converter -[logger_extension] -level=INFO -handlers=connectorHandler -formatter=LogFormatter -qualname=extension -[handler_consoleHandler] -class=StreamHandler -level=INFO -formatter=LogFormatter -args=(sys.stdout,) -[handler_connectorHandler] -level=INFO -class=thingsboard_gateway.tb_utility.tb_logger.TimedRotatingFileHandler -formatter=LogFormatter -args=("./logs/connector.log", "d", 1, 7,) -[handler_storageHandler] -level=INFO -class=thingsboard_gateway.tb_utility.tb_logger.TimedRotatingFileHandler -formatter=LogFormatter -args=("./logs/storage.log", "d", 1, 7,) -[handler_databaseHandler] -level=INFO -class=thingsboard_gateway.tb_utility.tb_logger.TimedRotatingFileHandler -formatter=LogFormatter -args=("./logs/database.log", "d", 1, 7,) -[handler_serviceHandler] -level=INFO -class=thingsboard_gateway.tb_utility.tb_logger.TimedRotatingFileHandler -formatter=LogFormatter -args=("./logs/service.log", "d", 1, 7,) -[handler_converterHandler] -level=INFO -class=thingsboard_gateway.tb_utility.tb_logger.TimedRotatingFileHandler -formatter=LogFormatter -args=("./logs/converter.log", "d", 1, 3,) -[handler_extensionHandler] -level=INFO -class=thingsboard_gateway.tb_utility.tb_logger.TimedRotatingFileHandler -formatter=LogFormatter -args=("./logs/extension.log", "d", 1, 3,) -[handler_tb_connectionHandler] -level=INFO -class=thingsboard_gateway.tb_utility.tb_logger.TimedRotatingFileHandler -formatter=LogFormatter -args=("./logs/tb_connection.log", "d", 1, 3,) -[formatter_LogFormatter] -format="%(asctime)s - |%(levelname)s| - [%(filename)s] - %(module)s - %(funcName)s - %(lineno)d - %(message)s" -datefmt="%Y-%m-%d %H:%M:%S" diff --git a/thingsboard_gateway/config/logs.json b/thingsboard_gateway/config/logs.json new file mode 100644 index 000000000..1965d1c0a --- /dev/null +++ b/thingsboard_gateway/config/logs.json @@ -0,0 +1,144 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "LogFormatter": { + "class": "logging.Formatter", + "format": "%(asctime)s - |%(levelname)s| - [%(filename)s] - %(module)s - %(funcName)s - %(lineno)d - %(message)s", + "datefmt": "%Y-%m-%d %H:%M:%S" + } + }, + "handlers": { + "consoleHandler": { + "class": "logging.StreamHandler", + "formatter": "LogFormatter", + "level": "DEBUG", + "stream": "ext://sys.stdout" + }, + "databaseHandler": { + "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", + "formatter": "LogFormatter", + "filename": "./logs/database.log", + "backupCount": 1, + "encoding": "utf-8" + }, + "serviceHandler": { + "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", + "formatter": "LogFormatter", + "filename": "./logs/service.log", + "backupCount": 7, + "interval": 3, + "when": "D", + "encoding": "utf-8" + }, + "connectorHandler": { + "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", + "formatter": "LogFormatter", + "filename": "./logs/connector.log", + "backupCount": 7, + "interval": 3, + "when": "D", + "encoding": "utf-8" + }, + "converterHandler": { + "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", + "formatter": "LogFormatter", + "filename": "./logs/converter.log", + "backupCount": 7, + "interval": 3, + "when": "D", + "encoding": "utf-8" + }, + "tb_connectionHandler": { + "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", + "formatter": "LogFormatter", + "filename": "./logs/tb_connection.log", + "backupCount": 7, + "interval": 3, + "when": "D", + "encoding": "utf-8" + }, + "storageHandler": { + "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", + "formatter": "LogFormatter", + "filename": "./logs/storage.log", + "backupCount": 7, + "interval": 3, + "when": "D", + "encoding": "utf-8" + }, + "extensionHandler": { + "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", + "formatter": "LogFormatter", + "filename": "./logs/extension.log", + "backupCount": 8, + "interval": 5, + "when": "D", + "encoding": "utf-8" + } + }, + "loggers": { + "database": { + "handlers": [ + "databaseHandler", + "consoleHandler" + ], + "level": "INFO", + "propagate": false + }, + "service": { + "handlers": [ + "serviceHandler", + "consoleHandler" + ], + "level": "INFO", + "propagate": false + }, + "connector": { + "handlers": [ + "connectorHandler", + "consoleHandler" + ], + "level": "INFO", + "propagate": false + }, + "converter": { + "handlers": [ + "converterHandler", + "consoleHandler" + ], + "level": "INFO", + "propagate": false + }, + "tb_connection": { + "handlers": [ + "tb_connectionHandler", + "consoleHandler" + ], + "level": "INFO", + "propagate": false + }, + "storage": { + "handlers": [ + "storageHandler", + "consoleHandler" + ], + "level": "INFO", + "propagate": false + }, + "extension": { + "handlers": [ + "extensionHandler", + "consoleHandler" + ], + "level": "INFO", + "propagate": false + } + }, + "root": { + "level": "ERROR", + "handlers": [ + "consoleHandler" + ] + } +} \ No newline at end of file diff --git a/thingsboard_gateway/config/statistics/statistics_macos.json b/thingsboard_gateway/config/statistics/statistics_macos.json index 89a6abcdd..1a64c022e 100644 --- a/thingsboard_gateway/config/statistics/statistics_macos.json +++ b/thingsboard_gateway/config/statistics/statistics_macos.json @@ -1,12 +1,12 @@ [ { "timeout": 100, - "command": ["/bin/sh", "-c", "ps -A -o cpu,%mem | awk '{cpu += $1}END{print cpu}'"], + "command": "ps -A -o cpu,%mem | awk '{cpu += $1}END{print cpu}'", "attributeOnGateway": "CPU" }, { "timeout": 100, - "command": ["/bin/sh", "-c", "ps -A -o %cpu,%mem | awk '{mem += $2}END{print mem}'"], + "command": "ps -A -o %cpu,%mem | awk '{mem += $2}END{print mem}'", "attributeOnGateway": "Memory" }, { diff --git a/thingsboard_gateway/config/tb_gateway.json b/thingsboard_gateway/config/tb_gateway.json new file mode 100644 index 000000000..3799ab770 --- /dev/null +++ b/thingsboard_gateway/config/tb_gateway.json @@ -0,0 +1,56 @@ +{ + "thingsboard": { + "host": "thingsboard.cloud", + "port": 1883, + "remoteShell": true, + "remoteConfiguration": true, + "statistics": { + "enable": true, + "statsSendPeriodInSeconds": 60 + }, + "deviceFiltering": { + "enable": false, + "filterFile": "list.json" + }, + "maxPayloadSizeBytes": 1024, + "minPackSendDelayMS": 60, + "minPackSizeToSend": 500, + "checkConnectorsConfigurationInSeconds": 10, + "handleDeviceRenaming": true, + "security": { + "type": "accessToken", + "accessToken": "YOUR_ACCESS_TOKEN" + }, + "qos": 1, + "checkingDeviceActivity": { + "checkDeviceInactivity": false, + "inactivityTimeoutSeconds": 200, + "inactivityCheckPeriodSeconds": 500 + } + }, + "storage": { + "type": "memory", + "read_records_count": 100, + "max_records_count": 10000, + "data_folder_path": "./data/", + "max_file_count": 10, + "max_read_records_count": 10, + "max_records_per_file": 10000, + "data_file_path": "./data/data.db", + "messages_ttl_check_in_hours": 1, + "messages_ttl_in_days": 7 + }, + "grpc": { + "enabled": false, + "serverPort": 9595, + "keepaliveTimeMs": 10000, + "keepaliveTimeoutMs": 5000, + "keepalivePermitWithoutCalls": true, + "maxPingsWithoutData": 0, + "minTimeBetweenPingsMs": 10000, + "minPingIntervalWithoutDataMs": 5000, + "keepAliveTimeMs": 10000, + "keepAliveTimeoutMs": 5000 + }, + "connectors": [] +} \ No newline at end of file diff --git a/thingsboard_gateway/config/tb_gateway.yaml b/thingsboard_gateway/config/tb_gateway.yaml deleted file mode 100644 index da4768a52..000000000 --- a/thingsboard_gateway/config/tb_gateway.yaml +++ /dev/null @@ -1,141 +0,0 @@ -thingsboard: - host: thingsboard.cloud - port: 1883 - remoteShell: false - remoteConfiguration: false - statistics: - enable: true - statsSendPeriodInSeconds: 60 -# configuration: statistics/statistics_linux.json - deviceFiltering: - enable: false - filterFile: list.json - maxPayloadSizeBytes: 1024 - minPackSendDelayMS: 200 - minPackSizeToSend: 500 - checkConnectorsConfigurationInSeconds: 60 - handleDeviceRenaming: true - checkingDeviceActivity: - checkDeviceInactivity: false - inactivityTimeoutSeconds: 120 - inactivityCheckPeriodSeconds: 10 - security: - accessToken: PUT_YOUR_GW_ACCESS_TOKEN_HERE - qos: 1 -storage: - type: memory - read_records_count: 100 - max_records_count: 100000 -# type: file -# data_folder_path: ./data/ -# max_file_count: 10 -# max_read_records_count: 10 -# max_records_per_file: 10000 -# type: sqlite -# data_file_path: ./data/data.db -# messages_ttl_check_in_hours: 1 -# messages_ttl_in_days: 7 -grpc: - enabled: false - serverPort: 9595 - keepaliveTimeMs: 10000 - keepaliveTimeoutMs: 5000 - keepalivePermitWithoutCalls: true - maxPingsWithoutData: 0 - minTimeBetweenPingsMs: 10000 - minPingIntervalWithoutDataMs: 5000 -connectors: - - - name: MQTT Broker Connector - type: mqtt - configuration: mqtt.json - -# - -# name: Modbus Connector -# type: modbus -# configuration: modbus.json -# -# - -# name: Modbus Connector -# type: modbus -# configuration: modbus_serial.json -# -# - -# name: OPC-UA Connector -# type: opcua -# configuration: opcua.json -# -# - -# name: OPC-UA Connector -# type: opcua_asyncio -# configuration: opcua.json -# -# - -# name: BLE Connector -# type: ble -# configuration: ble.json -# -# - -# name: REQUEST Connector -# type: request -# configuration: request.json -# -# - -# name: CAN Connector -# type: can -# configuration: can.json -# -# - -# name: BACnet Connector -# type: bacnet -# configuration: bacnet.json -# -# - -# name: ODBC Connector -# type: odbc -# configuration: odbc.json -# -# - -# name: REST Connector -# type: rest -# configuration: rest.json -# -# - -# name: SNMP Connector -# type: snmp -# configuration: snmp.json -# -# - -# name: FTP Connector -# type: ftp -# configuration: ftp.json -# -# - -# name: Socket TCP/UDP Connector -# type: socket -# configuration: socket.json -# -# - -# name: XMPP Connector -# type: xmpp -# configuration: xmpp.json -# -# - -# name: OCPP Connector -# type: ocpp -# configuration: ocpp.json -# -# ========= Customization ========== -# -# -# - -# name: Custom Serial Connector -# type: serial -# configuration: custom_serial.json -# class: CustomSerialConnector -# -# - -# name: GRPC Connector 1 -# key: auto -# type: grpc -# configuration: grpc_connector_1.json diff --git a/thingsboard_gateway/connectors/bacnet/bacnet_connector.py b/thingsboard_gateway/connectors/bacnet/bacnet_connector.py index f1492a596..9a437eb07 100644 --- a/thingsboard_gateway/connectors/bacnet/bacnet_connector.py +++ b/thingsboard_gateway/connectors/bacnet/bacnet_connector.py @@ -21,6 +21,7 @@ from thingsboard_gateway.gateway.statistics_service import StatisticsService from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader from thingsboard_gateway.tb_utility.tb_utility import TBUtility +from thingsboard_gateway.tb_utility.tb_logger import init_logger try: from bacpypes.core import run, stop @@ -31,7 +32,7 @@ from bacpypes.pdu import Address, GlobalBroadcast, LocalBroadcast, LocalStation, RemoteStation -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector from thingsboard_gateway.connectors.bacnet.bacnet_utilities.tb_gateway_bacnet_application import TBBACnetApplication @@ -47,6 +48,7 @@ def __init__(self, gateway, config, connector_type): self.__device_indexes = {} self.__devices_address_name = {} self.__gateway = gateway + self._log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO')) self._application = TBBACnetApplication(self, self.__config) self.__bacnet_core_thread = Thread(target=run, name="BACnet core thread", daemon=True, kwargs={"sigterm": None, "sigusr1": None}) @@ -73,7 +75,7 @@ def run(self): self.__connected = True self.scan_network() self._application.do_whois() - log.debug("WhoIsRequest has been sent.") + self._log.debug("WhoIsRequest has been sent.") self.scan_network() while not self.__stopped: sleep(.2) @@ -96,7 +98,7 @@ def run(self): else: sleep(.2) except Exception as e: - log.exception(e) + self._log.exception(e) if not self.__convert_and_save_data_queue.empty(): for _ in range(self.__convert_and_save_data_queue.qsize()): @@ -107,18 +109,22 @@ def run(self): def close(self): self.__stopped = True self.__connected = False + self._log.reset() stop() def get_name(self): return self.name + def get_type(self): + return self._connector_type + def is_connected(self): return self.__connected @StatisticsService.CollectAllReceivedBytesStatistics(start_stat_type='allReceivedBytesFromTB') def on_attributes_update(self, content): try: - log.debug('Recieved Attribute Update Request: %r', str(content)) + self._log.debug('Recieved Attribute Update Request: %r', str(content)) for device in self.__devices: if device["deviceName"] == content["device"]: for request in device["attribute_updates"]: @@ -131,16 +137,16 @@ def on_attributes_update(self, content): self.__request_functions[request["config"]["requestType"]](iocb) return else: - log.error("\"requestType\" not found in request configuration for key %s device: %s", + self._log.error("\"requestType\" not found in request configuration for key %s device: %s", request.get("key", "[KEY IS EMPTY]"), device["deviceName"]) except Exception as e: - log.exception(e) + self._log.exception(e) @StatisticsService.CollectAllReceivedBytesStatistics(start_stat_type='allReceivedBytesFromTB') def server_side_rpc_handler(self, content): try: - log.debug('Recieved RPC Request: %r', str(content)) + self._log.debug('Recieved RPC Request: %r', str(content)) for device in self.__devices: if device["deviceName"] == content["device"]: method_found = False @@ -162,14 +168,14 @@ def server_side_rpc_handler(self, content): # iocb, # self.__rpc_cancel_processing) else: - log.error("\"requestType\" not found in request configuration for key %s device: %s", + self._log.error("\"requestType\" not found in request configuration for key %s device: %s", request.get("key", "[KEY IS EMPTY]"), device["deviceName"]) if not method_found: - log.error("RPC method %s not found in configuration", content["data"]["method"]) + self._log.error("RPC method %s not found in configuration", content["data"]["method"]) self.__gateway.send_rpc_reply(content["device"], content["data"]["id"], success_sent=False) except Exception as e: - log.exception(e) + self._log.exception(e) def __rpc_response_cb(self, iocb, callback_params=None): device = self.rpc_requests_in_progress[iocb] @@ -177,26 +183,26 @@ def __rpc_response_cb(self, iocb, callback_params=None): content = device["content"] if iocb.ioResponse: apdu = iocb.ioResponse - log.debug("Received callback with Response: %r", apdu) + self._log.debug("Received callback with Response: %r", apdu) converted_data = converter.convert(None, apdu) if converted_data is None: converted_data = {"success": True} self.__gateway.send_rpc_reply(content["device"], content["data"]["id"], converted_data) # self.__gateway.rpc_with_reply_processing(iocb, converted_data or {"success": True}) elif iocb.ioError: - log.exception("Received callback with Error: %r", iocb.ioError) + self._log.exception("Received callback with Error: %r", iocb.ioError) data = {"error": str(iocb.ioError)} self.__gateway.send_rpc_reply(content["device"], content["data"]["id"], data) - log.debug(iocb.ioError) + self._log.debug(iocb.ioError) else: - log.error("Received unknown RPC response callback from device: %r", iocb) + self._log.error("Received unknown RPC response callback from device: %r", iocb) def __rpc_cancel_processing(self, iocb): - log.info("RPC with iocb %r - cancelled.", iocb) + self._log.info("RPC with iocb %r - cancelled.", iocb) def scan_network(self): self._application.do_whois() - log.debug("WhoIsRequest has been sent.") + self._log.debug("WhoIsRequest has been sent.") for device in self.__config_devices: try: if self._application.check_or_add(device): @@ -212,7 +218,7 @@ def scan_network(self): } self._application.do_read_property(**data_to_application) except Exception as e: - log.exception(e) + self._log.exception(e) def __convert_and_save_data(self, queue): converter, mapping_type, config, iocb = queue.get() @@ -221,7 +227,7 @@ def __convert_and_save_data(self, queue): converted_data = converter.convert((mapping_type, config), iocb.ioResponse if iocb.ioResponse else iocb.ioError) except Exception as e: - log.exception(e) + self._log.exception(e) self.__gateway.send_to_storage(self.name, converted_data) def __bacnet_device_mapping_response_cb(self, iocb, callback_params): @@ -238,7 +244,7 @@ def __bacnet_device_mapping_response_cb(self, iocb, callback_params): converted_data = converter.convert((mapping_type, config), iocb.ioResponse if iocb.ioResponse else iocb.ioError) except Exception as e: - log.exception(e) + self._log.exception(e) self.__gateway.send_to_storage(self.name, converted_data) def __load_converters(self, device): @@ -250,9 +256,9 @@ def __load_converters(self, device): converter_object = self.default_converters[converter_type] if datatype_config.get( "class") is None else TBModuleLoader.import_module(self._connector_type, device.get("class")) - datatype_config[converter_type] = converter_object(device) + datatype_config[converter_type] = converter_object(device, self._log) except Exception as e: - log.exception(e) + self._log.exception(e) def add_device(self, data): if self.__devices_address_name.get(data["address"]) is None: @@ -282,9 +288,9 @@ def add_device(self, data): self.__devices_address_name[data["address"]] = device_information["deviceName"] self.__devices.append(device_information) - log.debug(data["address"].addrType) + self._log.debug(data["address"].addrType) except Exception as e: - log.exception(e) + self._log.exception(e) def __get_requests_configs(self, device): result = {"attribute_updates": [], "server_side_rpc": []} diff --git a/thingsboard_gateway/connectors/bacnet/bacnet_converter.py b/thingsboard_gateway/connectors/bacnet/bacnet_converter.py index 3ec040a30..4b8819aab 100644 --- a/thingsboard_gateway/connectors/bacnet/bacnet_converter.py +++ b/thingsboard_gateway/connectors/bacnet/bacnet_converter.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import Converter, log +from thingsboard_gateway.connectors.converter import Converter class BACnetConverter(Converter): diff --git a/thingsboard_gateway/connectors/bacnet/bacnet_downlink_converter.py b/thingsboard_gateway/connectors/bacnet/bacnet_downlink_converter.py index 5d42c2598..effe4455d 100644 --- a/thingsboard_gateway/connectors/bacnet/bacnet_downlink_converter.py +++ b/thingsboard_gateway/connectors/bacnet/bacnet_downlink_converter.py @@ -12,12 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.bacnet.bacnet_converter import Converter, log +from thingsboard_gateway.connectors.bacnet.bacnet_converter import Converter class BACnetDownlinkConverter(Converter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config def convert(self, config, data): - log.debug(config, data) + self._log.debug(config, data) diff --git a/thingsboard_gateway/connectors/bacnet/bacnet_uplink_converter.py b/thingsboard_gateway/connectors/bacnet/bacnet_uplink_converter.py index 296caf538..aca332d25 100644 --- a/thingsboard_gateway/connectors/bacnet/bacnet_uplink_converter.py +++ b/thingsboard_gateway/connectors/bacnet/bacnet_uplink_converter.py @@ -16,12 +16,13 @@ from bacpypes.constructeddata import ArrayOf from bacpypes.primitivedata import Tag -from thingsboard_gateway.connectors.bacnet.bacnet_converter import BACnetConverter, log +from thingsboard_gateway.connectors.bacnet.bacnet_converter import BACnetConverter from thingsboard_gateway.gateway.statistics_service import StatisticsService class BACnetUplinkConverter(BACnetConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config @StatisticsService.CollectStatistics(start_stat_type='receivedBytesFromDevices', @@ -40,7 +41,7 @@ def convert(self, config, data): dict_result[datatypes[config[0]]].append({config[1]["key"]: value}) else: dict_result = value - log.debug("%r %r", self, dict_result) + self._log.debug("%r %r", self, dict_result) return dict_result @staticmethod diff --git a/thingsboard_gateway/connectors/ble/ble_connector.py b/thingsboard_gateway/connectors/ble/ble_connector.py index 073dc3b12..2009e1231 100644 --- a/thingsboard_gateway/connectors/ble/ble_connector.py +++ b/thingsboard_gateway/connectors/ble/ble_connector.py @@ -21,6 +21,7 @@ from thingsboard_gateway.tb_utility.tb_utility import TBUtility from thingsboard_gateway.gateway.statistics_service import StatisticsService +from thingsboard_gateway.tb_utility.tb_logger import init_logger try: from bleak import BleakScanner @@ -28,7 +29,7 @@ print("BLE library not found - installing...") TBUtility.install_package("bleak") -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector from thingsboard_gateway.connectors.ble.device import Device @@ -42,9 +43,10 @@ def __init__(self, gateway, config, connector_type): self._connector_type = connector_type self.__gateway = gateway self.__config = config + self.setName(self.__config.get("name", 'BLE Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))) + self.__log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO')) self.daemon = True - self.setName(self.__config.get("name", 'BLE Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))) self.__stopped = False self.__connected = False @@ -62,20 +64,21 @@ async def __show_map(self): scanning_mode='passive' if self.__config.get('passiveScanMode', True) else 'active').discover( timeout=scanner.get('timeout', 10000) / 1000) - log.info('FOUND DEVICES') + self.__log.info('FOUND DEVICES') if scanner.get('deviceName'): found_devices = [x.__str__() for x in filter(lambda x: x.name == scanner['deviceName'], devices)] if found_devices: - log.info(', '.join(found_devices)) + self.__log.info(', '.join(found_devices)) else: - log.info('nothing to show') + self.__log.info('nothing to show') else: for device in devices: - log.info(device) + self.__log.info(device) def __configure_and_load_devices(self): - self.__devices = [Device({**device, 'callback': BLEConnector.callback, 'connector_type': self._connector_type}) - for device in self.__config.get('devices', [])] + self.__devices = [ + Device({**device, 'callback': BLEConnector.callback, 'connector_type': self._connector_type}, self.__log) + for device in self.__config.get('devices', [])] def open(self): self.__stopped = False @@ -93,11 +96,15 @@ def run(self): def close(self): self.__stopped = True - log.info('%s has been stopped.', self.get_name()) + self.__log.info('%s has been stopped.', self.get_name()) + self.__log.reset() def get_name(self): return self.name + def get_type(self): + return self._connector_type + def is_connected(self): return self.__connected @@ -110,17 +117,17 @@ def __process_data(self): converter = device_config.pop('converter') try: - converter = converter(device_config) + converter = converter(device_config, self.__log) converted_data = converter.convert(config, data) self.statistics['MessagesReceived'] = self.statistics['MessagesReceived'] + 1 - log.debug(converted_data) + self.__log.debug(converted_data) if converted_data is not None: self.__gateway.send_to_storage(self.get_name(), converted_data) self.statistics['MessagesSent'] = self.statistics['MessagesSent'] + 1 - log.info('Data to ThingsBoard %s', converted_data) + self.__log.info('Data to ThingsBoard %s', converted_data) except Exception as e: - log.exception(e) + self.__log.exception(e) else: sleep(.2) @@ -137,7 +144,7 @@ def on_attributes_update(self, content): 'utf-8')) except IndexError: - log.error('Device not found') + self.__log.error('Device not found') @StatisticsService.CollectAllReceivedBytesStatistics(start_stat_type='allReceivedBytesFromTB') def server_side_rpc_handler(self, content): @@ -163,7 +170,7 @@ def server_side_rpc_handler(self, content): return except IndexError: - log.error('Device not found') + self.__log.error('Device not found') def get_config(self): return self.__config diff --git a/thingsboard_gateway/connectors/ble/ble_uplink_converter.py b/thingsboard_gateway/connectors/ble/ble_uplink_converter.py index 97f2ba04d..05a8edf24 100644 --- a/thingsboard_gateway/connectors/ble/ble_uplink_converter.py +++ b/thingsboard_gateway/connectors/ble/ble_uplink_converter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import Converter, abstractmethod, log +from thingsboard_gateway.connectors.converter import Converter, abstractmethod class BLEUplinkConverter(Converter): diff --git a/thingsboard_gateway/connectors/ble/bytes_ble_uplink_converter.py b/thingsboard_gateway/connectors/ble/bytes_ble_uplink_converter.py index 043d0f892..f70379c1c 100644 --- a/thingsboard_gateway/connectors/ble/bytes_ble_uplink_converter.py +++ b/thingsboard_gateway/connectors/ble/bytes_ble_uplink_converter.py @@ -27,12 +27,13 @@ from pprint import pformat from re import findall -from thingsboard_gateway.connectors.ble.ble_uplink_converter import BLEUplinkConverter, log +from thingsboard_gateway.connectors.ble.ble_uplink_converter import BLEUplinkConverter from thingsboard_gateway.gateway.statistics_service import StatisticsService class BytesBLEUplinkConverter(BLEUplinkConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config @StatisticsService.CollectStatistics(start_stat_type='receivedBytesFromDevices', @@ -75,13 +76,12 @@ def convert(self, config, data): if item.get('key') is not None: dict_result[section].append({item['key']: converted_data}) else: - log.error('Key for %s not found in config: %s', config['type'], config[section]) + self._log.error('Key for %s not found in config: %s', config['type'], config[section]) except Exception as e: - log.error('\nException caught when processing data for %s\n\n', pformat(config)) - log.exception(e) + self._log.exception('\nException caught when processing data for %s\n\n%s', pformat(config), e) except Exception as e: - log.exception(e) + self._log.exception(e) - log.debug(dict_result) + self._log.debug(dict_result) return dict_result diff --git a/thingsboard_gateway/connectors/ble/device.py b/thingsboard_gateway/connectors/ble/device.py index e8b59e14e..eff18466b 100644 --- a/thingsboard_gateway/connectors/ble/device.py +++ b/thingsboard_gateway/connectors/ble/device.py @@ -31,7 +31,6 @@ from bleak import BleakClient, BleakScanner -from thingsboard_gateway.connectors.connector import log from thingsboard_gateway.gateway.statistics_service import StatisticsService from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader from thingsboard_gateway.connectors.ble.error_handler import ErrorHandler @@ -44,8 +43,9 @@ class Device(Thread): - def __init__(self, config): + def __init__(self, config, logger): super().__init__() + self._log = logger self.loop = None self.stopped = False self.name = config['name'] @@ -65,7 +65,7 @@ def __init__(self, config): except ValueError as e: self.client = None self.stopped = True - log.error(e) + self._log.error(e) self.poll_period = config.get('pollPeriod', 5000) / 1000 self.config = self._generate_config(config) @@ -123,10 +123,10 @@ def __load_converter(self, name): module = TBModuleLoader.import_module(self.__connector_type, name) if module: - log.debug('Converter %s for device %s - found!', name, self.name) + self._log.debug('Converter %s for device %s - found!', name, self.name) return module else: - log.error("Cannot find converter for %s device", self.name) + self._log.error("Cannot find converter for %s device", self.name) self.stopped = True async def timer(self): @@ -139,13 +139,12 @@ async def timer(self): else: await asyncio.sleep(.2) except Exception as e: - log.error('Problem with connection') - log.debug(e) + self._log.exception('Problem with connection: \n %s', e) try: await self.client.disconnect() except Exception as err: - log.exception(err) + self._log.exception(err) connect_try = 0 while not self.stopped and not self.client.is_connected: @@ -193,7 +192,7 @@ async def __process_self(self): except Exception as e: error = ErrorHandler(e) if error.is_char_not_found() or error.is_operation_not_supported(): - log.error(e) + self._log.error(e) pass else: raise e @@ -205,7 +204,7 @@ async def __process_self(self): except Exception as e: error = ErrorHandler(e) if error.is_char_not_found() or error.is_operation_not_supported(): - log.error(e) + self._log.error(e) pass else: raise e @@ -231,10 +230,10 @@ def __set_char_handle(self, item, char_id): async def _connect_to_device(self): try: - log.info('Trying to connect to %s with %s MAC address', self.name, self.mac_address) + self._log.info('Trying to connect to %s with %s MAC address', self.name, self.mac_address) await self.client.connect(timeout=self.timeout) except Exception as e: - log.error(e) + self._log.error(e) def filter_macaddress(self, device): macaddress, device = device @@ -249,13 +248,13 @@ async def _process_adv_data(self): try: device = tuple(filter(self.filter_macaddress, devices.items()))[0][-1] except IndexError: - log.error('Device with MAC address %s not found!', self.mac_address) + self._log.error('Device with MAC address %s not found!', self.mac_address) return try: advertisement_data = list(device[-1].manufacturer_data.values())[0] except (IndexError, AttributeError): - log.error('Device %s haven\'t advertisement data', self.name) + self._log.error('Device %s haven\'t advertisement data', self.name) return data_for_converter = { @@ -282,7 +281,7 @@ async def run_client(self): await self.connect_to_device() if self.client and self.client.is_connected: - log.info('Connected to %s device', self.name) + self._log.info('Connected to %s device', self.name) if self.show_map: await self.__show_map() @@ -326,7 +325,7 @@ async def __show_map(self, return_result=False): if return_result: return result else: - log.info(result) + self._log.info(result) def scan_self(self, return_result): task = self.loop.create_task(self.__show_map(return_result)) @@ -342,8 +341,7 @@ async def __write_char(self, char_id, data): await asyncio.sleep(1.0) return 'Ok' except Exception as e: - log.error('Can\'t write data to device') - log.exception(e) + self._log.exception('Can\'t write data to device: \n %s', e) return e @StatisticsService.CollectStatistics(start_stat_type='allBytesSentToDevices') @@ -359,7 +357,7 @@ async def __read_char(self, char_id): try: return await self.client.read_gatt_char(char_id) except Exception as e: - log.exception(e) + self._log.exception(e) def read_char(self, char_id): task = self.loop.create_task(self.__read_char(char_id)) diff --git a/thingsboard_gateway/connectors/ble/hex_bytes_ble_uplink_converter.py b/thingsboard_gateway/connectors/ble/hex_bytes_ble_uplink_converter.py index 0c2e41e9e..a19576996 100644 --- a/thingsboard_gateway/connectors/ble/hex_bytes_ble_uplink_converter.py +++ b/thingsboard_gateway/connectors/ble/hex_bytes_ble_uplink_converter.py @@ -1,12 +1,13 @@ from pprint import pformat from re import findall -from thingsboard_gateway.connectors.ble.ble_uplink_converter import BLEUplinkConverter, log +from thingsboard_gateway.connectors.ble.ble_uplink_converter import BLEUplinkConverter from thingsboard_gateway.gateway.statistics_service import StatisticsService class HexBytesBLEUplinkConverter(BLEUplinkConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config dict_result = {"deviceName": config['deviceName'], "deviceType": config['deviceType'] @@ -47,12 +48,11 @@ def convert(self, config, data): if item.get('key') is not None: dict_result[section].append({item['key']: value}) else: - log.error('Key for %s not found in config: %s', config['type'], config[section]) + self._log.error('Key for %s not found in config: %s', config['type'], config[section]) except Exception as e: - log.error('\nException caught when processing data for %s\n\n', pformat(config)) - log.exception(e) + self._log.error('\nException caught when processing data for %s\n\n %s', pformat(config), e) except Exception as e: - log.exception(e) + self._log.exception(e) - log.debug(dict_result) + self._log.debug(dict_result) return dict_result diff --git a/thingsboard_gateway/connectors/can/bytes_can_downlink_converter.py b/thingsboard_gateway/connectors/can/bytes_can_downlink_converter.py index d538914c1..32cc85eaa 100644 --- a/thingsboard_gateway/connectors/can/bytes_can_downlink_converter.py +++ b/thingsboard_gateway/connectors/can/bytes_can_downlink_converter.py @@ -15,11 +15,13 @@ import struct from thingsboard_gateway.connectors.can.can_converter import CanConverter -from thingsboard_gateway.connectors.converter import log from thingsboard_gateway.gateway.statistics_service import StatisticsService class BytesCanDownlinkConverter(CanConverter): + def __init__(self, logger): + self._log = logger + @StatisticsService.CollectStatistics(start_stat_type='allReceivedBytesFromTB', end_stat_type='allBytesSentToDevices') def convert(self, config, data): @@ -28,7 +30,7 @@ def convert(self, config, data): return list(bytearray.fromhex(config["dataInHex"])) if not isinstance(data, dict) or not data: - log.error("Failed to convert TB data to CAN payload: data is empty or not a dictionary") + self._log.error("Failed to convert TB data to CAN payload: data is empty or not a dictionary") return if data.get("dataInHex", ""): @@ -41,7 +43,7 @@ def convert(self, config, data): elif "value" in data: value = data["value"] else: - log.error("Failed to convert TB data to CAN payload: no `value` or `dataExpression` property") + self._log.error("Failed to convert TB data to CAN payload: no `value` or `dataExpression` property") return can_data = [] @@ -67,5 +69,5 @@ def convert(self, config, data): return can_data except Exception as e: - log.error("Failed to convert TB data to CAN payload: %s", str(e)) + self._log.error("Failed to convert TB data to CAN payload: %s", str(e)) return diff --git a/thingsboard_gateway/connectors/can/bytes_can_uplink_converter.py b/thingsboard_gateway/connectors/can/bytes_can_uplink_converter.py index 26b7122ee..73dd64cec 100644 --- a/thingsboard_gateway/connectors/can/bytes_can_uplink_converter.py +++ b/thingsboard_gateway/connectors/can/bytes_can_uplink_converter.py @@ -15,11 +15,13 @@ import struct from thingsboard_gateway.connectors.can.can_converter import CanConverter -from thingsboard_gateway.connectors.converter import log from thingsboard_gateway.gateway.statistics_service import StatisticsService class BytesCanUplinkConverter(CanConverter): + def __init__(self, logger): + self._log = logger + @StatisticsService.CollectStatistics(start_stat_type='receivedBytesFromDevices', end_stat_type='convertedBytesFromDevice') def convert(self, configs, can_data): @@ -51,8 +53,8 @@ def convert(self, configs, can_data): for hex_byte in can_data[config["start"]:config["start"] + data_length]: value += "%02x" % hex_byte else: - log.error("Failed to convert CAN data to TB %s '%s': unknown data type '%s'", - "time series key" if config["is_ts"] else "attribute", tb_key, config["type"]) + self._log.error("Failed to convert CAN data to TB %s '%s': unknown data type '%s'", + "time series key" if config["is_ts"] else "attribute", tb_key, config["type"]) continue if config.get("expression", ""): @@ -62,7 +64,7 @@ def convert(self, configs, can_data): else: result[tb_item][tb_key] = value except Exception as e: - log.error("Failed to convert CAN data to TB %s '%s': %s", - "time series key" if config["is_ts"] else "attribute", tb_key, str(e)) + self._log.error("Failed to convert CAN data to TB %s '%s': %s", + "time series key" if config["is_ts"] else "attribute", tb_key, str(e)) continue return result diff --git a/thingsboard_gateway/connectors/can/can_connector.py b/thingsboard_gateway/connectors/can/can_connector.py index 80eca34b9..050011891 100644 --- a/thingsboard_gateway/connectors/can/can_connector.py +++ b/thingsboard_gateway/connectors/can/can_connector.py @@ -22,6 +22,7 @@ from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader from thingsboard_gateway.tb_utility.tb_utility import TBUtility +from thingsboard_gateway.tb_utility.tb_logger import init_logger try: from can import Notifier, BufferedReader, Message, CanError, ThreadSafeBus @@ -32,7 +33,7 @@ from thingsboard_gateway.connectors.can.bytes_can_downlink_converter import BytesCanDownlinkConverter from thingsboard_gateway.connectors.can.bytes_can_uplink_converter import BytesCanUplinkConverter -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector class CanConnector(Connector, Thread): @@ -71,6 +72,7 @@ def __init__(self, gateway, config, connector_type): self.__gateway = gateway self._connector_type = connector_type self.__config = config + self._log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO')) self.__bus_conf = {} self.__bus = None self.__reconnect_count = 0 @@ -89,18 +91,22 @@ def __init__(self, gateway, config, connector_type): self.__parse_config(config) def open(self): - log.info("[%s] Starting...", self.get_name()) + self._log.info("[%s] Starting...", self.get_name()) self.__stopped = False self.start() def close(self): if not self.__stopped: self.__stopped = True - log.debug("[%s] Stopping", self.get_name()) + self._log.debug("[%s] Stopping", self.get_name()) + self._log.reset() def get_name(self): return self.name + def get_type(self): + return self._connector_type + def is_connected(self): return self.__connected @@ -108,47 +114,47 @@ def on_attributes_update(self, content): for attr_name, attr_value in content["data"].items(): attr_config = self.__shared_attributes.get(content["device"], {}).get(attr_name) if attr_config is None: - log.warning("[%s] No configuration for '%s' attribute, ignore its update", self.get_name(), attr_name) + self._log.warning("[%s] No configuration for '%s' attribute, ignore its update", self.get_name(), attr_name) return - log.debug("[%s] Processing attribute update for '%s' device: attribute=%s,value=%s", + self._log.debug("[%s] Processing attribute update for '%s' device: attribute=%s,value=%s", self.get_name(), content["device"], attr_name, attr_value) # Converter expects dictionary as the second parameter so pack an attribute value to a dictionary data = self.__converters[content["device"]]["downlink"].convert(attr_config, {"value": attr_value}) if data is None: - log.error("[%s] Failed to update '%s' attribute for '%s' device: data conversion failure", + self._log.error("[%s] Failed to update '%s' attribute for '%s' device: data conversion failure", self.get_name(), attr_name, content["device"]) return done = self.send_data_to_bus(data, attr_config, data_check=True) if done: - log.debug("[%s] Updated '%s' attribute for '%s' device", self.get_name(), attr_name, content["device"]) + self._log.debug("[%s] Updated '%s' attribute for '%s' device", self.get_name(), attr_name, content["device"]) else: - log.error("[%s] Failed to update '%s' attribute for '%s' device", + self._log.error("[%s] Failed to update '%s' attribute for '%s' device", self.get_name(), attr_name, content["device"]) def server_side_rpc_handler(self, content): rpc_config = self.__rpc_calls.get(content["device"], {}).get(content["data"]["method"]) if rpc_config is None: if not self.__devices[content["device"]]["enableUnknownRpc"]: - log.warning("[%s] No configuration for '%s' RPC request (id=%s), ignore it", + self._log.warning("[%s] No configuration for '%s' RPC request (id=%s), ignore it", self.get_name(), content["data"]["method"], content["data"]["id"]) return else: rpc_config = {} - log.debug("[%s] Processing %s '%s' RPC request (id=%s) for '%s' device: params=%s", + self._log.debug("[%s] Processing %s '%s' RPC request (id=%s) for '%s' device: params=%s", self.get_name(), "pre-configured" if rpc_config else "UNKNOWN", content["data"]["method"], content["data"]["id"], content["device"], content["data"].get("params")) if self.__devices[content["device"]]["overrideRpcConfig"]: if rpc_config: conversion_config = self.__merge_rpc_configs(content["data"].get("params", {}), rpc_config) - log.debug("[%s] RPC request (id=%s) params and connector config merged to conversion config %s", + self._log.debug("[%s] RPC request (id=%s) params and connector config merged to conversion config %s", self.get_name(), content["data"]["id"], conversion_config) else: - log.debug("[%s] RPC request (id=%s) will use its params as conversion config", + self._log.debug("[%s] RPC request (id=%s) will use its params as conversion config", self.get_name(), content["data"]["id"]) conversion_config = content["data"].get("params", {}) else: @@ -159,14 +165,14 @@ def server_side_rpc_handler(self, content): if data is not None: done = self.send_data_to_bus(data, conversion_config, data_check=True) if done: - log.debug("[%s] Processed '%s' RPC request (id=%s) for '%s' device", + self._log.debug("[%s] Processed '%s' RPC request (id=%s) for '%s' device", self.get_name(), content["data"]["method"], content["data"]["id"], content["device"]) else: - log.error("[%s] Failed to process '%s' RPC request (id=%s) for '%s' device", + self._log.error("[%s] Failed to process '%s' RPC request (id=%s) for '%s' device", self.get_name(), content["data"]["method"], content["data"]["id"], content["device"]) else: done = False - log.error("[%s] Failed to process '%s' RPC request (id=%s) for '%s' device: data conversion failure", + self._log.error("[%s] Failed to process '%s' RPC request (id=%s) for '%s' device: data conversion failure", self.get_name(), content["data"]["method"], content["data"]["id"], content["device"]) if conversion_config.get("response", self.DEFAULT_RPC_RESPONSE_SEND_FLAG): @@ -185,7 +191,7 @@ def run(self): reader = BufferedReader() bus_notifier = Notifier(self.__bus, [reader]) - log.info("[%s] Connected to CAN bus (interface=%s,channel=%s)", self.get_name(), interface, channel) + self._log.info("[%s] Connected to CAN bus (interface=%s,channel=%s)", self.get_name(), interface, channel) if self.__polling_messages: poller = Poller(self) @@ -205,7 +211,7 @@ def run(self): self.__process_message(message) self.__check_if_error_happened() except Exception as e: - log.error("[%s] Error on CAN bus: %s", self.get_name(), str(e)) + self._log.error("[%s] Error on CAN bus: %s", self.get_name(), str(e)) finally: try: if poller is not None: @@ -213,28 +219,28 @@ def run(self): if bus_notifier is not None: bus_notifier.stop() if self.__bus is not None: - log.debug("[%s] Shutting down connection to CAN bus (state=%s)", + self._log.debug("[%s] Shutting down connection to CAN bus (state=%s)", self.get_name(), self.__bus.state) self.__bus.shutdown() except Exception as e: - log.error("[%s] Error on shutdown connection to CAN bus: %s", self.get_name(), str(e)) + self._log.error("[%s] Error on shutdown connection to CAN bus: %s", self.get_name(), str(e)) self.__connected = False if not self.__stopped: if self.__is_reconnect_enabled(): retry_period = self.__reconnect_conf["period"] - log.info("[%s] Next attempt to connect will be in %f seconds (%s attempt left)", + self._log.info("[%s] Next attempt to connect will be in %f seconds (%s attempt left)", self.get_name(), retry_period, "infinite" if self.__reconnect_conf["maxCount"] is None else self.__reconnect_conf["maxCount"] - self.__reconnect_count + 1) time.sleep(retry_period) else: need_run = False - log.info("[%s] Last attempt to connect has failed. Exiting...", self.get_name()) + self._log.info("[%s] Last attempt to connect has failed. Exiting...", self.get_name()) else: need_run = False - log.info("[%s] Stopped", self.get_name()) + self._log.info("[%s] Stopped", self.get_name()) def is_stopped(self): return self.__stopped @@ -252,9 +258,9 @@ def send_data_to_bus(self, data, config, data_check=True, raise_exception=False) check=data_check)) return True except (ValueError, TypeError) as e: - log.error("[%s] Wrong CAN message data: %s", self.get_name(), str(e)) + self._log.error("[%s] Wrong CAN message data: %s", self.get_name(), str(e)) except CanError as e: - log.error("[%s] Failed to send CAN message: %s", self.get_name(), str(e)) + self._log.error("[%s] Failed to send CAN message: %s", self.get_name(), str(e)) if raise_exception: raise e else: @@ -262,7 +268,7 @@ def send_data_to_bus(self, data, config, data_check=True, raise_exception=False) return False def __on_bus_error(self, e): - log.warning("[%s] Notified about CAN bus error. Store it to later processing", self.get_name()) + self._log.warning("[%s] Notified about CAN bus error. Store it to later processing", self.get_name()) self.__bus_error = e def __check_if_error_happened(self): @@ -299,7 +305,7 @@ def __merge_rpc_configs(self, rpc_params, rpc_config): def __process_message(self, message): if message.arbitration_id not in self.__nodes: # Too lot log messages in case of high message generation frequency - log.debug("[%s] Ignoring CAN message. Unknown arbitration_id %d", self.get_name(), message.arbitration_id) + self._log.debug("[%s] Ignoring CAN message. Unknown arbitration_id %d", self.get_name(), message.arbitration_id) return cmd_conf = self.__commands[message.arbitration_id] @@ -310,16 +316,16 @@ def __process_message(self, message): cmd_id = self.NO_CMD_ID if cmd_id not in self.__nodes[message.arbitration_id]: - log.debug("[%s] Ignoring CAN message. Unknown cmd_id %d", self.get_name(), cmd_id) + self._log.debug("[%s] Ignoring CAN message. Unknown cmd_id %d", self.get_name(), cmd_id) return - log.debug("[%s] Processing CAN message (id=%d,cmd_id=%s): %s", + self._log.debug("[%s] Processing CAN message (id=%d,cmd_id=%s): %s", self.get_name(), message.arbitration_id, cmd_id, message) parsing_conf = self.__nodes[message.arbitration_id][cmd_id] data = self.__converters[parsing_conf["deviceName"]]["uplink"].convert(parsing_conf["configs"], message.data) if data is None or not data.get("attributes", []) and not data.get("telemetry", []): - log.warning("[%s] Failed to process CAN message (id=%d,cmd_id=%s): data conversion failure", + self._log.warning("[%s] Failed to process CAN message (id=%d,cmd_id=%s): data conversion failure", self.get_name(), message.arbitration_id, cmd_id) return @@ -340,12 +346,12 @@ def __check_and_send(self, conf, new_data): to_send["deviceName"] = conf["deviceName"] to_send["deviceType"] = conf["deviceType"] - log.debug("[%s] Pushing to TB server '%s' device data: %s", self.get_name(), conf["deviceName"], to_send) + self._log.debug("[%s] Pushing to TB server '%s' device data: %s", self.get_name(), conf["deviceName"], to_send) self.__gateway.send_to_storage(self.get_name(), to_send) self.statistics['MessagesSent'] += 1 else: - log.debug("[%s] '%s' device data has not been changed", self.get_name(), conf["deviceName"]) + self._log.debug("[%s] '%s' device data has not been changed", self.get_name(), conf["deviceName"]) def __is_reconnect_enabled(self): if self.__reconnect_conf["enabled"]: @@ -385,7 +391,7 @@ def __parse_config(self, config): self.__converters[device_name] = {} if not strict_eval: - log.info("[%s] Data converters for '%s' device will use non-strict eval", self.get_name(), device_name) + self._log.info("[%s] Data converters for '%s' device will use non-strict eval", self.get_name(), device_name) if "serverSideRpc" in device_config and device_config["serverSideRpc"]: is_device_config_valid = True @@ -428,7 +434,7 @@ def __parse_config(self, config): node_id = msg_config.get("nodeId", self.UNKNOWN_ARBITRATION_ID) if node_id == self.UNKNOWN_ARBITRATION_ID: - log.warning("[%s] Ignore '%s' %s configuration: no arbitration id", + self._log.warning("[%s] Ignore '%s' %s configuration: no arbitration id", self.get_name(), tb_key, config_key) continue @@ -436,14 +442,14 @@ def __parse_config(self, config): if value_config is not None: msg_config.update(value_config) else: - log.warning("[%s] Ignore '%s' %s configuration: no value configuration", + self._log.warning("[%s] Ignore '%s' %s configuration: no value configuration", self.get_name(), tb_key, config_key, ) continue if msg_config.get("command"): cmd_config = self.__parse_command_config(msg_config["command"]) if cmd_config is None: - log.warning("[%s] Ignore '%s' %s configuration: wrong command configuration", + self._log.warning("[%s] Ignore '%s' %s configuration: wrong command configuration", self.get_name(), tb_key, config_key, ) continue @@ -456,7 +462,7 @@ def __parse_config(self, config): if cmd_config["start"] != prev_cmd_config["start"] or \ cmd_config["length"] != prev_cmd_config["length"] or \ cmd_config["byteorder"] != prev_cmd_config["byteorder"]: - log.warning("[%s] Ignore '%s' %s configuration: " + self._log.warning("[%s] Ignore '%s' %s configuration: " "another command configuration already added for arbitration_id %d", self.get_name(), tb_key, config_key, node_id) continue @@ -498,25 +504,25 @@ def __parse_config(self, config): check=True) self.__polling_messages.append(polling_config) except (ValueError, TypeError) as e: - log.warning("[%s] Ignore '%s' %s polling configuration, wrong CAN data: %s", - self.get_name(), tb_key, config_key, str(e)) + self._log.warning("[%s] Ignore '%s' %s polling configuration, wrong CAN data: %s", + self.get_name(), tb_key, config_key, str(e)) continue if is_device_config_valid: - log.debug("[%s] Done parsing of '%s' device configuration", self.get_name(), device_name) + self._log.debug("[%s] Done parsing of '%s' device configuration", self.get_name(), device_name) self.__gateway.add_device(device_name, {"connector": self}) else: - log.warning("[%s] Ignore '%s' device configuration, because it doesn't have attributes," - "attributeUpdates,timeseries or serverSideRpc", self.get_name(), device_name) + self._log.warning("[%s] Ignore '%s' device configuration, because it doesn't have attributes," + "attributeUpdates,timeseries or serverSideRpc", self.get_name(), device_name) def __parse_value_config(self, config): if config is None: - log.warning("[%s] Wrong value configuration: no data", self.get_name()) + self._log.warning("[%s] Wrong value configuration: no data", self.get_name()) return if isinstance(config, str): value_matches = re.search(self.VALUE_REGEX, config) if not value_matches: - log.warning("[%s] Wrong value configuration: '%s' doesn't match pattern", self.get_name(), config) + self._log.warning("[%s] Wrong value configuration: '%s' doesn't match pattern", self.get_name(), config) return value_config = { @@ -549,20 +555,20 @@ def __parse_value_config(self, config): return value_config except (KeyError, ValueError) as e: - log.warning("[%s] Wrong value configuration: %s", self.get_name(), str(e)) + self._log.warning("[%s] Wrong value configuration: %s", self.get_name(), str(e)) return - log.warning("[%s] Wrong value configuration: unknown type", self.get_name()) + self._log.warning("[%s] Wrong value configuration: unknown type", self.get_name()) return def __parse_command_config(self, config): if config is None: - log.warning("[%s] Wrong command configuration: no data", self.get_name()) + self._log.warning("[%s] Wrong command configuration: no data", self.get_name()) return if isinstance(config, str): cmd_matches = re.search(self.CMD_REGEX, config) if not cmd_matches: - log.warning("[%s] Wrong command configuration: '%s' doesn't match pattern", self.get_name(), config) + self._log.warning("[%s] Wrong command configuration: '%s' doesn't match pattern", self.get_name(), config) return return { @@ -580,22 +586,22 @@ def __parse_command_config(self, config): "value": int(config["value"]) } except (KeyError, ValueError) as e: - log.warning("[%s] Wrong command configuration: %s", self.get_name(), str(e)) + self._log.warning("[%s] Wrong command configuration: %s", self.get_name(), str(e)) return - log.warning("[%s] Wrong command configuration: unknown type", self.get_name()) + self._log.warning("[%s] Wrong command configuration: unknown type", self.get_name()) return def __get_converter(self, config, need_uplink): if config is None: - return BytesCanUplinkConverter() if need_uplink else BytesCanDownlinkConverter() + return BytesCanUplinkConverter(self._log) if need_uplink else BytesCanDownlinkConverter(self._log) else: if need_uplink: uplink = config.get("uplink") - return BytesCanUplinkConverter() if uplink is None \ + return BytesCanUplinkConverter(self._log) if uplink is None \ else TBModuleLoader.import_module(self._connector_type, uplink) else: downlink = config.get("downlink") - return BytesCanDownlinkConverter() if downlink is None \ + return BytesCanDownlinkConverter(self._log) if downlink is None \ else TBModuleLoader.import_module(self._connector_type, downlink) def get_config(self): @@ -613,15 +619,16 @@ def __init__(self, connector: CanConnector): def poll(self): if self.first_run: - log.info("[%s] Starting poller", self.connector.get_name()) + self.connector._log.info("[%s] Starting poller", self.connector.get_name()) for polling_config in self.connector.get_polling_messages(): key = polling_config["key"] if polling_config["type"] == "always": - log.info("[%s] Polling '%s' key every %f sec", self.connector.get_name(), key, polling_config["period"]) + self.connector._log.info("[%s] Polling '%s' key every %f sec", self.connector.get_name(), key, + polling_config["period"]) self.__poll_and_schedule(bytearray.fromhex(polling_config["dataInHex"]), polling_config) elif self.first_run: - log.info("[%s] Polling '%s' key once", self.connector.get_name(), key) + self.connector._log.info("[%s] Polling '%s' key once", self.connector.get_name(), key) self.connector.send_data_to_bus(bytearray.fromhex(polling_config["dataInHex"]), polling_config, raise_exception=self.first_run) @@ -631,10 +638,10 @@ def poll(self): def run(self): self.scheduler.run() - log.info("[%s] Poller stopped", self.connector.get_name()) + self.connector._log.info("[%s] Poller stopped", self.connector.get_name()) def stop(self): - log.debug("[%s] Stopping poller", self.connector.get_name()) + self.connector._log.debug("[%s] Stopping poller", self.connector.get_name()) for event in self.events: self.scheduler.cancel(event) @@ -644,8 +651,8 @@ def __poll_and_schedule(self, data, config): if self.events: self.events.pop(0) - log.debug("[%s] Sending periodic (%f sec) CAN message (arbitration_id=%d, data=%s)", - self.connector.get_name(), config["period"], config["nodeId"], data) + self.connector._log.debug("[%s] Sending periodic (%f sec) CAN message (arbitration_id=%d, data=%s)", + self.connector.get_name(), config["period"], config["nodeId"], data) self.connector.send_data_to_bus(data, config, raise_exception=self.first_run) event = self.scheduler.enter(config["period"], 1, self.__poll_and_schedule, argument=(data, config)) diff --git a/thingsboard_gateway/connectors/can/can_converter.py b/thingsboard_gateway/connectors/can/can_converter.py index 9b8d77b30..feb82d26c 100644 --- a/thingsboard_gateway/connectors/can/can_converter.py +++ b/thingsboard_gateway/connectors/can/can_converter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import ABC, abstractmethod, log +from thingsboard_gateway.connectors.converter import ABC, abstractmethod class CanConverter(ABC): diff --git a/thingsboard_gateway/connectors/connector.py b/thingsboard_gateway/connectors/connector.py index b2016eb21..ae645e4ca 100644 --- a/thingsboard_gateway/connectors/connector.py +++ b/thingsboard_gateway/connectors/connector.py @@ -12,11 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging from abc import ABC, abstractmethod -from thingsboard_gateway.gateway.constants import DEFAULT_SEND_ON_CHANGE_INFINITE_TTL_VALUE, DEFAULT_SEND_ON_CHANGE_VALUE - -log = logging.getLogger("connector") +from thingsboard_gateway.gateway.constants import DEFAULT_SEND_ON_CHANGE_INFINITE_TTL_VALUE, \ + DEFAULT_SEND_ON_CHANGE_VALUE class Connector(ABC): @@ -33,6 +31,10 @@ def close(self): def get_name(self): pass + @abstractmethod + def get_type(self): + pass + @abstractmethod def get_config(self): pass diff --git a/thingsboard_gateway/connectors/converter.py b/thingsboard_gateway/connectors/converter.py index 8d24f8f67..f7ae73af5 100644 --- a/thingsboard_gateway/connectors/converter.py +++ b/thingsboard_gateway/connectors/converter.py @@ -12,11 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging from abc import ABC, abstractmethod -log = logging.getLogger("converter") - class Converter(ABC): diff --git a/thingsboard_gateway/connectors/ftp/ftp_connector.py b/thingsboard_gateway/connectors/ftp/ftp_connector.py index d979c974a..016dbc1c6 100644 --- a/thingsboard_gateway/connectors/ftp/ftp_connector.py +++ b/thingsboard_gateway/connectors/ftp/ftp_connector.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import io import re from ftplib import FTP, FTP_TLS @@ -28,6 +29,7 @@ from thingsboard_gateway.connectors.ftp.path import Path from thingsboard_gateway.gateway.statistics_service import StatisticsService from thingsboard_gateway.tb_utility.tb_utility import TBUtility +from thingsboard_gateway.tb_utility.tb_logger import init_logger try: from requests import Timeout, request @@ -36,7 +38,7 @@ TBUtility.install_package("requests") from requests import Timeout, request -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector class FTPConnector(Connector, Thread): @@ -44,7 +46,6 @@ def __init__(self, gateway, config, connector_type): super().__init__() self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} - self.__log = log self.__config = config self._connector_type = connector_type self.__gateway = gateway @@ -52,6 +53,7 @@ def __init__(self, gateway, config, connector_type): 'username': 'anonymous', "password": 'anonymous@'} self.__tls_support = self.__config.get("TLSSupport", False) self.setName(self.__config.get("name", "".join(choice(ascii_lowercase) for _ in range(5)))) + self.__log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO')) self.daemon = True self.__stopped = False self.__requests_in_progress = [] @@ -134,7 +136,7 @@ def __process_paths(self, ftp): time_point = timer() if time_point - path.last_polled_time >= path.poll_period or path.last_polled_time == 0: configuration = path.config - converter = FTPUplinkConverter(configuration) + converter = FTPUplinkConverter(configuration, self.__log) path.last_polled_time = time_point if '*' in path.path: @@ -186,14 +188,18 @@ def __send_data(self, converted_data): if converted_data: self.__gateway.send_to_storage(self.getName(), converted_data) self.statistics['MessagesSent'] = self.statistics['MessagesSent'] + 1 - log.debug("Data to ThingsBoard: %s", converted_data) + self.__log.debug("Data to ThingsBoard: %s", converted_data) def close(self): self.__stopped = True + self.__log.reset() def get_name(self): return self.name + def get_type(self): + return self._connector_type + def is_connected(self): return self._connected @@ -235,7 +241,7 @@ def on_attributes_update(self, content): ftp.storbinary('STOR ' + file.path_to_file, io_stream) io_stream.close() else: - log.error('Invalid json data') + self.__log.error('Invalid json data') else: if attribute_request['writingMode'] == 'OVERRIDE': io_stream = self._get_io_stream(data_expression) @@ -252,7 +258,7 @@ def on_attributes_update(self, content): io_stream.close() except Exception as e: - log.exception(e) + self.__log.exception(e) @StatisticsService.CollectAllReceivedBytesStatistics('allBytesSentToDevices') def _get_io_stream(self, data_expression): @@ -282,7 +288,7 @@ def server_side_rpc_handler(self, content): io_stream.close() success_sent = True except Exception as e: - log.error(e) + self.__log.error(e) converted_data = '{"error": "' + str(e) + '"}' else: handle_stream = io.BytesIO() @@ -293,7 +299,7 @@ def server_side_rpc_handler(self, content): self.__gateway.send_rpc_reply(device=content["device"], req_id=content["data"]["id"], success_sent=success_sent, content=converted_data) except Exception as e: - log.exception(e) + self.__log.exception(e) def get_config(self): return self.__config diff --git a/thingsboard_gateway/connectors/ftp/ftp_uplink_converter.py b/thingsboard_gateway/connectors/ftp/ftp_uplink_converter.py index 32b4c80ec..648febb2f 100644 --- a/thingsboard_gateway/connectors/ftp/ftp_uplink_converter.py +++ b/thingsboard_gateway/connectors/ftp/ftp_uplink_converter.py @@ -17,14 +17,14 @@ from simplejson import dumps -from thingsboard_gateway.connectors.converter import log from thingsboard_gateway.connectors.ftp.ftp_converter import FTPConverter from thingsboard_gateway.tb_utility.tb_utility import TBUtility from thingsboard_gateway.gateway.statistics_service import StatisticsService class FTPUplinkConverter(FTPConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config self.__data_types = {"attributes": "attributes", "timeseries": "telemetry"} @@ -73,8 +73,8 @@ def _convert_table_view_data(self, config, data): dict_result['deviceType'] = arr[index] except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), data) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config), data, + e) return dict_result @@ -108,8 +108,8 @@ def _convert_slices_view_data(self, data): if self.__config['devicePatternType'] == information['value']: dict_result['deviceType'] = val except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), data) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config), data, + e) return dict_result @@ -132,7 +132,7 @@ def _convert_json_file(self, data): if is_valid_key else device_name_tag else: - log.error("The expression for looking \"deviceName\" not found in config %s", dumps(self.__config)) + self._log.error("The expression for looking \"deviceName\" not found in config %s", dumps(self.__config)) if self.__config.get("devicePatternType") is not None: device_type_tags = TBUtility.get_values(self.__config.get("devicePatternType"), data, @@ -148,8 +148,8 @@ def _convert_json_file(self, data): str(device_type_value)) \ if is_valid_key else device_type_tag except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), data) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config), data, + e) try: for datatype in self.__data_types: @@ -188,8 +188,8 @@ def _convert_json_file(self, data): else: dict_result[self.__data_types[datatype]].append({full_key: full_value}) except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), str(data)) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config), + str(data), e) return dict_result diff --git a/thingsboard_gateway/connectors/modbus/backward_compability_adapter.py b/thingsboard_gateway/connectors/modbus/backward_compability_adapter.py index f20c13d57..d5d4981b4 100644 --- a/thingsboard_gateway/connectors/modbus/backward_compability_adapter.py +++ b/thingsboard_gateway/connectors/modbus/backward_compability_adapter.py @@ -12,16 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging from simplejson import dumps -from thingsboard_gateway.connectors.connector import log - class BackwardCompatibilityAdapter: config_files_count = 1 CONFIG_PATH = None - def __init__(self, config, config_dir): + def __init__(self, config, config_dir, logger=None): + if logger: + self._log = logger + else: + self._log = logging.getLogger('BackwardCompatibilityAdapter') + self.__config = config self.__config_dir = config_dir BackwardCompatibilityAdapter.CONFIG_PATH = self.__config_dir @@ -36,8 +40,7 @@ def __save_json_config_file(config): file.writelines(dumps(config, sort_keys=False, indent=' ', separators=(',', ': '))) BackwardCompatibilityAdapter.config_files_count += 1 - @staticmethod - def __check_slaves_type_connection(config): + def __check_slaves_type_connection(self, config): is_tcp_or_udp_connection = False is_serial_connection = False @@ -48,7 +51,7 @@ def __check_slaves_type_connection(config): is_serial_connection = True if is_tcp_or_udp_connection and is_serial_connection: - log.warning('It seems that your slaves using different connection type (tcp/udp and serial). ' + self._log.warning('It seems that your slaves using different connection type (tcp/udp and serial). ' 'It is recommended to separate tcp/udp slaves and serial slaves in different connectors ' 'to avoid problems with reading data.') @@ -58,10 +61,10 @@ def convert(self): self.__check_slaves_type_connection(self.__config) return self.__config - log.warning( + self._log.warning( 'You are using old configuration structure for Modbus connector. It will be DEPRECATED in the future ' 'version! New config file "modbus_new.json" was generated in %s folder. Please, use it.', self.CONFIG_PATH) - log.warning('You have to manually connect the new generated config file to tb_gateway.yaml!') + self._log.warning('You have to manually connect the new generated config file to tb_gateway.yaml!') slaves = [] for device in self.__config['server'].get('devices', []): diff --git a/thingsboard_gateway/connectors/modbus/bytes_modbus_downlink_converter.py b/thingsboard_gateway/connectors/modbus/bytes_modbus_downlink_converter.py index bc149557b..9d36bed7f 100644 --- a/thingsboard_gateway/connectors/modbus/bytes_modbus_downlink_converter.py +++ b/thingsboard_gateway/connectors/modbus/bytes_modbus_downlink_converter.py @@ -15,13 +15,14 @@ from pymodbus.constants import Endian from pymodbus.payload import BinaryPayloadBuilder -from thingsboard_gateway.connectors.modbus.modbus_converter import ModbusConverter, log +from thingsboard_gateway.connectors.modbus.modbus_converter import ModbusConverter from thingsboard_gateway.gateway.statistics_service import StatisticsService class BytesModbusDownlinkConverter(ModbusConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config @StatisticsService.CollectStatistics(start_stat_type='allReceivedBytesFromTB', @@ -55,7 +56,7 @@ def convert(self, config, data): lower_type = config.get("type", config.get("tag", "error")).lower() if lower_type == "error": - log.error('"type" and "tag" - not found in configuration.') + self._log.error('"type" and "tag" - not found in configuration.') variable_size = config.get("objectsCount", config.get("registersCount", config.get("registerCount", 1))) * 16 if lower_type in ["integer", "dword", "dword/integer", "word", "int"]: @@ -87,7 +88,7 @@ def convert(self, config, data): elif lower_type in builder_functions: builder_functions[lower_type](value) else: - log.error("Unknown variable type") + self._log.error("Unknown variable type") builder_converting_functions = {5: builder.to_coils, 15: builder.to_coils, @@ -98,9 +99,9 @@ def convert(self, config, data): if function_code in builder_converting_functions: builder = builder_converting_functions[function_code]() - log.debug(builder) + self._log.debug(builder) if "Exception" in str(builder): - log.exception(builder) + self._log.exception(builder) builder = str(builder) # if function_code is 5 , is using first coils value if function_code == 5: @@ -112,8 +113,8 @@ def convert(self, config, data): builder = builder[0] else: if isinstance(builder, list) and len(builder) not in (2, 4): - log.warning("There is a problem with the value builder. Only the first register is written.") + self._log.warning("There is a problem with the value builder. Only the first register is written.") builder = builder[0] return builder - log.warning("Unsupported function code, for the device %s in the Modbus Downlink converter", config["device"]) + self._log.warning("Unsupported function code, for the device %s in the Modbus Downlink converter", config["device"]) return None diff --git a/thingsboard_gateway/connectors/modbus/bytes_modbus_uplink_converter.py b/thingsboard_gateway/connectors/modbus/bytes_modbus_uplink_converter.py index 6660ba994..0eef83191 100644 --- a/thingsboard_gateway/connectors/modbus/bytes_modbus_uplink_converter.py +++ b/thingsboard_gateway/connectors/modbus/bytes_modbus_uplink_converter.py @@ -17,12 +17,13 @@ from pymodbus.payload import BinaryPayloadDecoder from pymodbus.pdu import ExceptionResponse -from thingsboard_gateway.connectors.modbus.modbus_converter import ModbusConverter, log +from thingsboard_gateway.connectors.modbus.modbus_converter import ModbusConverter from thingsboard_gateway.gateway.statistics_service import StatisticsService class BytesModbusUplinkConverter(ModbusConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__datatypes = { "timeseries": "telemetry", "attributes": "attributes" @@ -69,7 +70,7 @@ def convert(self, config, data): elif configuration["functionCode"] in [3, 4]: decoder = None registers = response.registers - log.debug("Tag: %s Config: %s registers: %s", tag, str(configuration), str(registers)) + self._log.debug("Tag: %s Config: %s registers: %s", tag, str(configuration), str(registers)) try: decoder = BinaryPayloadDecoder.fromRegisters(registers, byteorder=endian_order, wordorder=word_endian_order) @@ -84,21 +85,19 @@ def convert(self, config, data): if configuration.get("multiplier"): decoded_data = decoded_data * configuration["multiplier"] else: - log.exception(response) + self._log.exception(response) decoded_data = None if config_data == "rpc": return decoded_data - log.debug("datatype: %s \t key: %s \t value: %s", self.__datatypes[config_data], tag, - str(decoded_data)) + self._log.debug("datatype: %s \t key: %s \t value: %s", self.__datatypes[config_data], tag, str(decoded_data)) if decoded_data is not None: self.__result[self.__datatypes[config_data]].append({tag: decoded_data}) except Exception as e: - log.exception(e) - log.debug(self.__result) + self._log.exception(e) + self._log.debug(self.__result) return self.__result - @staticmethod - def decode_from_registers(decoder, configuration): + def decode_from_registers(self, decoder, configuration): type_ = configuration["type"] objects_count = configuration.get("objectsCount", configuration.get("registersCount", configuration.get("registerCount", 1))) @@ -154,7 +153,7 @@ def decode_from_registers(decoder, configuration): decoded = decoder_functions[type_]() else: - log.error("Unknown type: %s", type_) + self._log.error("Unknown type: %s", type_) if isinstance(decoded, int): result_data = decoded diff --git a/thingsboard_gateway/connectors/modbus/modbus_connector.py b/thingsboard_gateway/connectors/modbus/modbus_connector.py index fb4207be3..48d3a4f5f 100644 --- a/thingsboard_gateway/connectors/modbus/modbus_connector.py +++ b/thingsboard_gateway/connectors/modbus/modbus_connector.py @@ -20,6 +20,7 @@ from packaging import version from thingsboard_gateway.tb_utility.tb_utility import TBUtility +from thingsboard_gateway.tb_utility.tb_logger import init_logger # Try import Pymodbus library or install it and import installation_required = False @@ -69,7 +70,7 @@ from pymodbus.datastore import ModbusSparseDataBlock from pymodbus.pdu import ExceptionResponse -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector from thingsboard_gateway.connectors.modbus.constants import * from thingsboard_gateway.connectors.modbus.slave import Slave from thingsboard_gateway.connectors.modbus.backward_compability_adapter import BackwardCompatibilityAdapter @@ -114,11 +115,13 @@ def __init__(self, gateway, config, connector_type): super().__init__() self.__gateway = gateway self._connector_type = connector_type - - self.__backward_compatibility_adapter = BackwardCompatibilityAdapter(config, gateway.get_config_path()) + self.__log = init_logger(self.__gateway, config.get('name', self.name), + config.get('logLevel', 'INFO')) + self.__backward_compatibility_adapter = BackwardCompatibilityAdapter(config, gateway.get_config_path(), + logger=self.__log) self.__config = self.__backward_compatibility_adapter.convert() - self.setName(self.__config.get("name", 'Modbus Default ' + ''.join(choice(ascii_lowercase) for _ in range(5)))) + self.__connected = False self.__stopped = False self.daemon = True @@ -151,6 +154,9 @@ def open(self): self.__stopped = False self.start() + def get_type(self): + return self._connector_type + def run(self): self.__connected = True @@ -162,8 +168,7 @@ def run(self): sleep(.001) - @staticmethod - def __configure_and_run_slave(config): + def __configure_and_run_slave(self, config): identity = None if config.get('identity'): identity = ModbusDeviceIdentification() @@ -177,7 +182,7 @@ def __configure_and_run_slave(config): blocks = {} for (key, value) in config.get('values').items(): values = {} - converter = BytesModbusDownlinkConverter({}) + converter = BytesModbusDownlinkConverter({}, self.__log) for item in value: for section in ('attributes', 'timeseries', 'attributeUpdates', 'rpc'): for val in item.get(section, []): @@ -219,7 +224,8 @@ def __modify_main_config(self): def __load_slaves(self): self.__slaves = [ - Slave(**{**device, 'connector': self, 'gateway': self.__gateway, 'callback': ModbusConnector.callback}) for + Slave(**{**device, 'connector': self, 'gateway': self.__gateway, 'logger': self.__log, + 'callback': ModbusConnector.callback}) for device in self.__config.get('master', {'slaves': []}).get('slaves', [])] @classmethod @@ -232,7 +238,7 @@ def connector_type(self): def __thread_manager(self): if len(self.__workers_thread_pool) == 0: - worker = ModbusConnector.ConverterWorker("Main", self._convert_msg_queue, self._save_data) + worker = ModbusConnector.ConverterWorker("Main", self._convert_msg_queue, self._save_data, self.__log) self.__workers_thread_pool.append(worker) worker.start() @@ -241,7 +247,7 @@ def __thread_manager(self): if number_of_needed_threads > threads_count < self.__max_number_of_workers: thread = ModbusConnector.ConverterWorker( "Worker " + ''.join(choice(ascii_lowercase) for _ in range(5)), self._convert_msg_queue, - self._save_data) + self._save_data, self.__log) self.__workers_thread_pool.append(thread) thread.start() elif number_of_needed_threads < threads_count and threads_count > 1: @@ -259,7 +265,7 @@ def __convert_data(self, params): config=config, data=device_responses) except Exception as e: - log.error(e) + self.__log.error(e) to_send = {DEVICE_NAME_PARAMETER: converted_data[DEVICE_NAME_PARAMETER], DEVICE_TYPE_PARAMETER: converted_data[DEVICE_TYPE_PARAMETER], @@ -298,7 +304,8 @@ def close(self): self.__stop_connections_to_masters() if reactor.running: ServerStop() - log.info('%s has been stopped.', self.get_name()) + self.__log.info('%s has been stopped.', self.get_name()) + self.__log.reset() def get_name(self): return self.name @@ -308,7 +315,7 @@ def __process_slaves(self): if not self.__stopped and not ModbusConnector.process_requests.empty(): device = ModbusConnector.process_requests.get() - log.debug("Device connector type: %s", device.config.get(TYPE_PARAMETER)) + self.__log.debug("Device connector type: %s", device.config.get(TYPE_PARAMETER)) if device.config.get(TYPE_PARAMETER).lower() == 'serial': self.lock.acquire() @@ -325,7 +332,7 @@ def __process_slaves(self): current_device_config[config_section]): error = 'Socket is closed' if not device.config[ 'master'].is_socket_open() else 'Config is invalid' - log.error(error) + self.__log.error(error) continue # Reading data from device @@ -344,8 +351,8 @@ def __process_slaves(self): "data_sent": current_data, "input_data": input_data} - log.debug("Checking %s for device %s", config_section, device) - log.debug('Device response: ', device_responses) + self.__log.debug("Checking %s for device %s", config_section, device) + self.__log.debug('Device response: ', device_responses) if device_responses.get('timeseries') or device_responses.get('attributes'): self._convert_msg_queue.put((self.__convert_data, (device, current_device_config, { @@ -358,9 +365,9 @@ def __process_slaves(self): except ConnectionException: sleep(5) - log.error("Connection lost! Reconnecting...") + self.__log.error("Connection lost! Reconnecting...") except Exception as e: - log.exception(e) + self.__log.exception(e) # Release mutex if "serial" type only if device.config.get(TYPE_PARAMETER) == 'serial': @@ -404,12 +411,13 @@ def __connect_to_current_master(self, device=None): device.config['connection_attempt'] = device.config[ 'connection_attempt'] + 1 device.config['last_connection_attempt_time'] = current_time - log.debug("Modbus trying connect to %s", device) + self.__log.debug("Modbus trying connect to %s", device) device.config['master'].connect() if device.config['connection_attempt'] == connect_attempt_count: - log.warn("Maximum attempt count (%i) for device \"%s\" - encountered.", connect_attempt_count, - device) + self.__log.warn("Maximum attempt count (%i) for device \"%s\" - encountered.", + connect_attempt_count, + device) if device.config['connection_attempt'] >= 0 and device.config['master'].is_socket_open(): device.config['connection_attempt'] = 0 @@ -477,8 +485,7 @@ def __stop_connections_to_masters(self): if slave.config.get('master') is not None and slave.config.get('master').is_socket_open(): slave.config['master'].close() - @staticmethod - def __function_to_device(device, config): + def __function_to_device(self, device, config): function_code = config.get('functionCode') result = None if function_code == 1: @@ -519,12 +526,12 @@ def __function_to_device(device, config): values=config[PAYLOAD_PARAMETER], slave=device.config['unitId']) else: - log.error("Unknown Modbus function with code: %s", function_code) + self.__log.error("Unknown Modbus function with code: %s", function_code) - log.debug("With result %s", str(result)) + self.__log.debug("With result %s", str(result)) if "Exception" in str(result): - log.exception(result) + self.__log.exception(result) return result @@ -547,22 +554,35 @@ def on_attributes_update(self, content): self.__process_request(to_process, attribute_updates_command_config, request_type='attributeUpdates') except Exception as e: - log.exception(e) + self.__log.exception(e) def server_side_rpc_handler(self, server_rpc_request): try: + if server_rpc_request.get('data') is None: + server_rpc_request['data'] = {'params': server_rpc_request['params'], + 'method': server_rpc_request['method']} + + rpc_method = server_rpc_request['data']['method'] + + # check if RPC type is connector RPC (can be only 'set') + try: + (connector_type, rpc_method_name) = rpc_method.split('_') + if connector_type == self._connector_type: + rpc_method = rpc_method_name + server_rpc_request['device'] = server_rpc_request['params'].split(' ')[0].split('=')[-1] + except (IndexError, ValueError): + pass + if server_rpc_request.get(DEVICE_SECTION_PARAMETER) is not None: - log.debug("Modbus connector received rpc request for %s with server_rpc_request: %s", - server_rpc_request[DEVICE_SECTION_PARAMETER], - server_rpc_request) + self.__log.debug("Modbus connector received rpc request for %s with server_rpc_request: %s", + server_rpc_request[DEVICE_SECTION_PARAMETER], + server_rpc_request) device = tuple( filter( lambda slave: slave.name == server_rpc_request[DEVICE_SECTION_PARAMETER], self.__slaves ) )[0] - rpc_method = server_rpc_request[DATA_PARAMETER].get(RPC_METHOD_PARAMETER) - # check if RPC method is reserved get/set if rpc_method == 'get' or rpc_method == 'set': params = {} @@ -589,19 +609,19 @@ def server_side_rpc_handler(self, server_rpc_request): self.__process_request(server_rpc_request, rpc_command_config) break else: - log.error("Received rpc request, but method %s not found in config for %s.", + self.__log.error("Received rpc request, but method %s not found in config for %s.", rpc_method, self.get_name()) self.__gateway.send_rpc_reply(server_rpc_request[DEVICE_SECTION_PARAMETER], server_rpc_request[DATA_PARAMETER][RPC_ID_PARAMETER], {rpc_method: "METHOD NOT FOUND!"}) else: - log.debug("Received RPC to connector: %r", server_rpc_request) + self.__log.debug("Received RPC to connector: %r", server_rpc_request) except Exception as e: - log.exception(e) + self.__log.exception(e) def __process_request(self, content, rpc_command_config, request_type='RPC'): - log.debug('Processing %s request', request_type) + self.__log.debug('Processing %s request', request_type) if rpc_command_config is not None: device = tuple(filter(lambda slave: slave.name == content[DEVICE_SECTION_PARAMETER], self.__slaves))[0] rpc_command_config[UNIT_ID_PARAMETER] = device.config['unitId'] @@ -625,7 +645,7 @@ def __process_request(self, content, rpc_command_config, request_type='RPC'): try: response = self.__function_to_device(device, rpc_command_config) except Exception as e: - log.exception(e) + self.__log.exception(e) response = e if isinstance(response, (ReadRegistersResponseBase, ReadBitsResponseBase)): @@ -639,42 +659,43 @@ def __process_request(self, content, rpc_command_config, request_type='RPC'): WORD_ORDER_PARAMETER: device.word_order }, data=to_converter) - log.debug("Received %s method: %s, result: %r", request_type, - content[DATA_PARAMETER][RPC_METHOD_PARAMETER], - response) + self.__log.debug("Received %s method: %s, result: %r", request_type, + content[DATA_PARAMETER][RPC_METHOD_PARAMETER], + response) elif isinstance(response, (WriteMultipleRegistersResponse, WriteMultipleCoilsResponse, WriteSingleCoilResponse, WriteSingleRegisterResponse)): - log.debug("Write %r", str(response)) + self.__log.debug("Write %r", str(response)) response = {"success": True} if content.get(RPC_ID_PARAMETER) or ( content.get(DATA_PARAMETER) is not None and content[DATA_PARAMETER].get( RPC_ID_PARAMETER)) is not None: if isinstance(response, Exception): - self.__gateway.send_rpc_reply(content[DEVICE_SECTION_PARAMETER], - content[DATA_PARAMETER][RPC_ID_PARAMETER], - {content[DATA_PARAMETER][RPC_METHOD_PARAMETER]: str(response)}) + self.__gateway.send_rpc_reply(device=content[DEVICE_SECTION_PARAMETER], + req_id=content[DATA_PARAMETER].get(RPC_ID_PARAMETER), + content={ + content[DATA_PARAMETER][RPC_METHOD_PARAMETER]: str(response)}) else: - self.__gateway.send_rpc_reply(content[DEVICE_SECTION_PARAMETER], - content[DATA_PARAMETER][RPC_ID_PARAMETER], - response) + self.__gateway.send_rpc_reply(device=content[DEVICE_SECTION_PARAMETER], + req_id=content[DATA_PARAMETER].get(RPC_ID_PARAMETER), + content=response) - log.debug("%r", response) + self.__log.debug("%r", response) def get_config(self): return self.__config def update_converter_config(self, converter_name, config): - log.debug('Received remote converter configuration update for %s with configuration %s', converter_name, + self.__log.debug('Received remote converter configuration update for %s with configuration %s', converter_name, config) for slave in self.__slaves: try: if slave.config[UPLINK_PREFIX + CONVERTER_PARAMETER].__class__.__name__ == converter_name: slave.config.update(config) - log.info('Updated converter configuration for: %s with configuration %s', - converter_name, config) + self.__log.info('Updated converter configuration for: %s with configuration %s', + converter_name, config) for slave_config in self.__config['master']['slaves']: if slave_config['deviceName'] == slave.name: @@ -685,8 +706,9 @@ def update_converter_config(self, converter_name, config): continue class ConverterWorker(Thread): - def __init__(self, name, incoming_queue, send_result): + def __init__(self, name, incoming_queue, send_result, logger): super().__init__() + self._log = logger self.stopped = False self.setName(name) self.setDaemon(True) @@ -701,7 +723,7 @@ def run(self): convert_function, params = self.__msg_queue.get(True, 10) converted_data = convert_function(params) if converted_data: - log.info(converted_data) + self._log.info(converted_data) self.__send_result(converted_data) self.in_progress = False else: diff --git a/thingsboard_gateway/connectors/modbus/modbus_converter.py b/thingsboard_gateway/connectors/modbus/modbus_converter.py index 6c8044100..ab4b31fa0 100644 --- a/thingsboard_gateway/connectors/modbus/modbus_converter.py +++ b/thingsboard_gateway/connectors/modbus/modbus_converter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import Converter, abstractmethod, log +from thingsboard_gateway.connectors.converter import Converter, abstractmethod class ModbusConverter(Converter): diff --git a/thingsboard_gateway/connectors/modbus/slave.py b/thingsboard_gateway/connectors/modbus/slave.py index 43ff0b262..c168a0e76 100644 --- a/thingsboard_gateway/connectors/modbus/slave.py +++ b/thingsboard_gateway/connectors/modbus/slave.py @@ -18,7 +18,6 @@ from pymodbus.constants import Defaults from thingsboard_gateway.connectors.modbus.constants import * -from thingsboard_gateway.connectors.connector import log from thingsboard_gateway.connectors.modbus.bytes_modbus_uplink_converter import BytesModbusUplinkConverter from thingsboard_gateway.connectors.modbus.bytes_modbus_downlink_converter import BytesModbusDownlinkConverter from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader @@ -29,6 +28,7 @@ def __init__(self, **kwargs): super().__init__() self.timeout = kwargs.get('timeout') self.name = kwargs['deviceName'] + self._log = kwargs['logger'] self.poll_period = kwargs['pollPeriod'] / 1000 self.byte_order = kwargs['byteOrder'] @@ -97,13 +97,13 @@ def __load_converters(self, connector, gateway): converter = TBModuleLoader.import_module(connector.connector_type, self.config[UPLINK_PREFIX + CONVERTER_PARAMETER])(self) else: - converter = BytesModbusUplinkConverter({**self.config, 'deviceName': self.name}) + converter = BytesModbusUplinkConverter({**self.config, 'deviceName': self.name}, self._log) if self.config.get(DOWNLINK_PREFIX + CONVERTER_PARAMETER) is not None: downlink_converter = TBModuleLoader.import_module(connector.connector_type, self.config[ DOWNLINK_PREFIX + CONVERTER_PARAMETER])(self) else: - downlink_converter = BytesModbusDownlinkConverter(self.config) + downlink_converter = BytesModbusDownlinkConverter(self.config, self._log) if self.name not in gateway.get_devices(): gateway.add_device(self.name, {CONNECTOR_PARAMETER: connector}, @@ -112,7 +112,7 @@ def __load_converters(self, connector, gateway): self.config[UPLINK_PREFIX + CONVERTER_PARAMETER] = converter self.config[DOWNLINK_PREFIX + CONVERTER_PARAMETER] = downlink_converter except Exception as e: - log.exception(e) + self._log.exception(e) def __str__(self): return f'{self.name}' diff --git a/thingsboard_gateway/connectors/mqtt/bytes_mqtt_uplink_converter.py b/thingsboard_gateway/connectors/mqtt/bytes_mqtt_uplink_converter.py index 76150a915..8c8fcc7f0 100644 --- a/thingsboard_gateway/connectors/mqtt/bytes_mqtt_uplink_converter.py +++ b/thingsboard_gateway/connectors/mqtt/bytes_mqtt_uplink_converter.py @@ -3,13 +3,14 @@ from simplejson import dumps -from thingsboard_gateway.connectors.mqtt.mqtt_uplink_converter import MqttUplinkConverter, log +from thingsboard_gateway.connectors.mqtt.mqtt_uplink_converter import MqttUplinkConverter from thingsboard_gateway.gateway.statistics_service import StatisticsService class BytesMqttUplinkConverter(MqttUplinkConverter): - def __init__(self, config): + def __init__(self, config, logger): self.__config = config.get('converter') + self._log = logger @property def config(self): @@ -41,10 +42,10 @@ def convert(self, topic, data): else: dict_result[datatypes[datatype]].append(value_item) except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), str(data)) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config), + str(data), e) - log.debug('Converted data: %s', dict_result) + self._log.debug('Converted data: %s', dict_result) return dict_result @staticmethod diff --git a/thingsboard_gateway/connectors/mqtt/json_mqtt_uplink_converter.py b/thingsboard_gateway/connectors/mqtt/json_mqtt_uplink_converter.py index 212713b8d..f28b34130 100644 --- a/thingsboard_gateway/connectors/mqtt/json_mqtt_uplink_converter.py +++ b/thingsboard_gateway/connectors/mqtt/json_mqtt_uplink_converter.py @@ -17,13 +17,14 @@ from simplejson import dumps from thingsboard_gateway.gateway.constants import SEND_ON_CHANGE_PARAMETER -from thingsboard_gateway.connectors.mqtt.mqtt_uplink_converter import MqttUplinkConverter, log +from thingsboard_gateway.connectors.mqtt.mqtt_uplink_converter import MqttUplinkConverter from thingsboard_gateway.tb_utility.tb_utility import TBUtility from thingsboard_gateway.gateway.statistics_service import StatisticsService class JsonMqttUplinkConverter(MqttUplinkConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config.get('converter') self.__send_data_on_change = self.__config.get(SEND_ON_CHANGE_PARAMETER) @@ -42,6 +43,7 @@ def convert(self, topic, data): converted_data = [] for item in data: converted_data.append(self._convert_single_item(topic, item)) + self._log.debug(converted_data) return converted_data else: return self._convert_single_item(topic, data) @@ -97,8 +99,9 @@ def _convert_single_item(self, topic, data): dict_result[datatypes[datatype]].append( self.create_timeseries_record(full_key, full_value, timestamp)) except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), str(data)) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config), + str(data), e) + self._log.debug(dict_result) return dict_result @staticmethod @@ -106,18 +109,15 @@ def create_timeseries_record(key, value, timestamp): value_item = {key: value} return {"ts": timestamp, 'values': value_item} if timestamp else value_item - @staticmethod - def parse_device_name(topic, data, config): - return JsonMqttUplinkConverter.parse_device_info( + def parse_device_name(self, topic, data, config): + return self.parse_device_info( topic, data, config, "deviceNameJsonExpression", "deviceNameTopicExpression") - @staticmethod - def parse_device_type(topic, data, config): - return JsonMqttUplinkConverter.parse_device_info( + def parse_device_type(self, topic, data, config): + return self.parse_device_info( topic, data, config, "deviceTypeJsonExpression", "deviceTypeTopicExpression") - @staticmethod - def parse_device_info(topic, data, config, json_expression_config_name, topic_expression_config_name): + def parse_device_info(self, topic, data, config, json_expression_config_name, topic_expression_config_name): result = None try: if config.get(json_expression_config_name) is not None: @@ -136,13 +136,12 @@ def parse_device_info(topic, data, config, json_expression_config_name, topic_ex if search_result is not None: result = search_result.group(0) else: - log.debug( + self._log.debug( "Regular expression result is None. deviceNameTopicExpression parameter will be interpreted " "as a deviceName\n Topic: %s\nRegex: %s", topic, expression) result = expression else: - log.error("The expression for looking \"deviceName\" not found in config %s", dumps(config)) + self._log.error("The expression for looking \"deviceName\" not found in config %s", dumps(config)) except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(config), data) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(config), data, e) return result diff --git a/thingsboard_gateway/connectors/mqtt/mqtt_connector.py b/thingsboard_gateway/connectors/mqtt/mqtt_connector.py index 4c3ed2491..b60a0cf15 100644 --- a/thingsboard_gateway/connectors/mqtt/mqtt_connector.py +++ b/thingsboard_gateway/connectors/mqtt/mqtt_connector.py @@ -1,4 +1,4 @@ -# Copyright 2022. ThingsBoard +# Copyright 2023. ThingsBoard # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,11 +25,12 @@ from thingsboard_gateway.gateway.constants import SEND_ON_CHANGE_PARAMETER, DEFAULT_SEND_ON_CHANGE_VALUE, \ ATTRIBUTES_PARAMETER, TELEMETRY_PARAMETER, SEND_ON_CHANGE_TTL_PARAMETER, DEFAULT_SEND_ON_CHANGE_INFINITE_TTL_VALUE from thingsboard_gateway.gateway.constant_enums import Status -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector from thingsboard_gateway.connectors.mqtt.mqtt_decorators import CustomCollectStatistics from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader from thingsboard_gateway.tb_utility.tb_utility import TBUtility from thingsboard_gateway.gateway.statistics_service import StatisticsService +from thingsboard_gateway.tb_utility.tb_logger import init_logger try: from paho.mqtt.client import Client @@ -108,14 +109,15 @@ def __init__(self, gateway, config, connector_type): self._connector_type = connector_type # Should be "mqtt" self.config = config # mqtt.json contents - self.__log = log + self.__log = init_logger(self.__gateway, self.config['name'], self.config.get('logLevel', 'INFO')) self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} self.__subscribes_sent = {} # Extract main sections from configuration --------------------------------------------------------------------- self.__broker = config.get('broker') self.__send_data_only_on_change = self.__broker.get(SEND_ON_CHANGE_PARAMETER, DEFAULT_SEND_ON_CHANGE_VALUE) - self.__send_data_only_on_change_ttl = self.__broker.get(SEND_ON_CHANGE_TTL_PARAMETER, DEFAULT_SEND_ON_CHANGE_INFINITE_TTL_VALUE) + self.__send_data_only_on_change_ttl = self.__broker.get(SEND_ON_CHANGE_TTL_PARAMETER, + DEFAULT_SEND_ON_CHANGE_INFINITE_TTL_VALUE) # for sendDataOnlyOnChange param self.__topic_content = {} @@ -199,9 +201,8 @@ def __init__(self, gateway, config, connector_type): ciphers=None) except Exception as e: self.__log.error("Cannot setup connection to broker %s using SSL. " - "Please check your configuration.\nError: ", - self.get_name()) - self.__log.exception(e) + "Please check your configuration.\nError: %s", + self.get_name(), e) if self.__broker["security"].get("insecure", False): self._client.tls_insecure_set(True) else: @@ -233,7 +234,10 @@ def is_filtering_enable(self, device_name): def get_config(self): return self.config - + + def get_type(self): + return self._connector_type + def get_ttl_for_duplicates(self, device_name): return self.__send_data_only_on_change_ttl @@ -312,9 +316,10 @@ def close(self): try: self._client.disconnect() except Exception as e: - log.exception(e) + self.__log.exception(e) self._client.loop_stop() self.__log.info('%s has been stopped.', self.get_name()) + self.__log.reset() def get_name(self): return self.name @@ -379,7 +384,7 @@ def _on_connect(self, client, userdata, flags, result_code, *extra_params): if module: self.__log.debug('Converter %s for topic %s - found!', converter_class_name, mapping["topicFilter"]) - converter = module(mapping) + converter = module(mapping, self.__log) if sharing_id: self.__shared_custom_converters[sharing_id] = converter else: @@ -452,7 +457,7 @@ def _on_log(self, *args): self.__log.debug(args) def _on_subscribe(self, _, __, mid, granted_qos, *args): - log.info(args) + self.__log.info(args) try: if granted_qos[0] == 128: self.__log.error('"%s" subscription failed to topic %s subscription message id = %i', @@ -536,7 +541,7 @@ def _process_on_message(self): request_handled = self.put_data_to_convert(converter, message, content) except Exception as e: - log.exception(e) + self.__log.exception(e) if not request_handled: self.__log.error('Cannot find converter for the topic:"%s"! Client: %s, User data: %s', @@ -684,7 +689,7 @@ def _process_on_message(self): break except Exception as e: - log.exception(e) + self.__log.exception(e) # Note: if I'm in this branch, this was for sure an attribute request message # => Execution must end here both in case of failure and success @@ -694,7 +699,7 @@ def _process_on_message(self): # The gateway is expecting for this message => no wildcards here, the topic must be evaluated as is if self.__gateway.is_rpc_in_progress(message.topic): - log.info("RPC response arrived. Forwarding it to thingsboard.") + self.__log.info("RPC response arrived. Forwarding it to thingsboard.") self.__gateway.rpc_with_reply_processing(message.topic, content) continue @@ -736,14 +741,14 @@ def on_attributes_update(self, content): .replace("${attributeKey}", str(attribute_key)) \ .replace("${attributeValue}", str(content["data"][attribute_key])) except KeyError as e: - log.exception("Cannot form topic, key %s - not found", e) + self.__log.exception("Cannot form topic, key %s - not found", e) raise e try: data = attribute_update["valueExpression"] \ .replace("${attributeKey}", str(attribute_key)) \ .replace("${attributeValue}", str(content["data"][attribute_key])) except KeyError as e: - log.exception("Cannot form topic, key %s - not found", e) + self.__log.exception("Cannot form topic, key %s - not found", e) raise e self._publish(topic, data, attribute_update.get('retain', False)) @@ -766,10 +771,12 @@ def __process_rpc_request(self, content, rpc_config): # 2-way RPC setup if expects_response and defines_timeout: expected_response_topic = rpc_config["responseTopicExpression"] \ - .replace("${deviceName}", str(content["device"])) \ .replace("${methodName}", str(content['data']['method'])) \ .replace("${requestId}", str(content["data"]["id"])) + if content.get('device'): + expected_response_topic.replace("${deviceName}", str(content["device"])) + expected_response_topic = TBUtility.replace_params_tags(expected_response_topic, content) timeout = time() * 1000 + rpc_config.get("responseTimeout") @@ -802,10 +809,12 @@ def __process_rpc_request(self, content, rpc_config): # Actually reach out for the device request_topic: str = rpc_config.get("requestTopicExpression") \ - .replace("${deviceName}", str(content["device"])) \ .replace("${methodName}", str(content['data']['method'])) \ .replace("${requestId}", str(content["data"]["id"])) + if content['data'].get('device'): + request_topic.replace("${deviceName}", str(content["device"])) + request_topic = TBUtility.replace_params_tags(request_topic, content) data_to_send_tags = TBUtility.get_values(rpc_config.get('valueExpression'), content['data'], @@ -837,8 +846,19 @@ def __process_rpc_request(self, content, rpc_config): def server_side_rpc_handler(self, content): self.__log.info("Incoming server-side RPC: %s", content) + if content.get('data') is None: + content['data'] = {'params': content['params'], 'method': content['method']} + rpc_method = content['data']['method'] + # check if RPC type is connector RPC (can be only 'get' or 'set') + try: + (connector_type, rpc_method_name) = rpc_method.split('_') + if connector_type == self._connector_type: + rpc_method = rpc_method_name + except ValueError: + pass + # check if RPC method is reserved get/set if rpc_method == 'get' or rpc_method == 'set': params = {} @@ -867,7 +887,7 @@ def _publish(self, request_topic, data_to_send, retain): self._client.publish(request_topic, data_to_send, retain).wait_for_publish() def rpc_cancel_processing(self, topic): - log.info("RPC canceled or terminated. Unsubscribing from %s", topic) + self.__log.info("RPC canceled or terminated. Unsubscribing from %s", topic) self._client.unsubscribe(topic) def get_converters(self): @@ -883,7 +903,7 @@ def update_converter_config(self, converter_name, config): if converter_class_name == converter_name: converter_obj.config = config self._send_current_converter_config(self.name + ':' + converter_name, config) - log.info('Updated converter configuration for: %s with configuration %s', + self.__log.info('Updated converter configuration for: %s with configuration %s', converter_name, converter_obj.config) for device_config in self.config['mapping']: @@ -926,7 +946,7 @@ def run(self): self.in_progress = True convert_function, config, incoming_data = self.__msg_queue.get(True, 100) converted_data = convert_function(config, incoming_data) - log.debug(converted_data) + # log.debug(converted_data) if converted_data and (converted_data.get(ATTRIBUTES_PARAMETER) or converted_data.get(TELEMETRY_PARAMETER)): self.__send_result(config, converted_data) diff --git a/thingsboard_gateway/connectors/mqtt/mqtt_uplink_converter.py b/thingsboard_gateway/connectors/mqtt/mqtt_uplink_converter.py index 9991ae5dd..e9c732a6b 100644 --- a/thingsboard_gateway/connectors/mqtt/mqtt_uplink_converter.py +++ b/thingsboard_gateway/connectors/mqtt/mqtt_uplink_converter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import Converter, abstractmethod, log +from thingsboard_gateway.connectors.converter import Converter, abstractmethod class MqttUplinkConverter(Converter): diff --git a/thingsboard_gateway/connectors/ocpp/charge_point.py b/thingsboard_gateway/connectors/ocpp/charge_point.py index 5bcf374d5..79ecaf354 100644 --- a/thingsboard_gateway/connectors/ocpp/charge_point.py +++ b/thingsboard_gateway/connectors/ocpp/charge_point.py @@ -23,11 +23,12 @@ class ChargePoint(CP): - def __init__(self, charge_point_id, websocket, config, callback): + def __init__(self, charge_point_id, websocket, config, callback, logger): super(ChargePoint, self).__init__(charge_point_id, websocket) + self._log = logger self._config = config self._callback = callback - self._uplink_converter = self._load_converter(config['uplink_converter_name'])(self._config) + self._uplink_converter = self._load_converter(config['uplink_converter_name'])(self._config, self._log) self._profile = {} self.name = None self.type = None diff --git a/thingsboard_gateway/connectors/ocpp/ocpp_connector.py b/thingsboard_gateway/connectors/ocpp/ocpp_connector.py index b183edba3..48eed137d 100644 --- a/thingsboard_gateway/connectors/ocpp/ocpp_connector.py +++ b/thingsboard_gateway/connectors/ocpp/ocpp_connector.py @@ -24,9 +24,10 @@ from simplejson import dumps -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector from thingsboard_gateway.gateway.statistics_service import StatisticsService from thingsboard_gateway.tb_utility.tb_utility import TBUtility +from thingsboard_gateway.tb_utility.tb_logger import init_logger try: import ocpp @@ -56,15 +57,15 @@ class OcppConnector(Connector, Thread): def __init__(self, gateway, config, connector_type): super().__init__() - self._log = log + self._config = config self._central_system_config = config['centralSystem'] self._charge_points_config = config.get('chargePoints', []) self._connector_type = connector_type self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} self._gateway = gateway - self.setName(self._central_system_config.get("name", 'OCPP Connector ' + ''.join( - choice(ascii_lowercase) for _ in range(5)))) + self.setName(self._config.get("name", 'OCPP Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))) + self._log = init_logger(self._gateway, self.name, self._config.get('logLevel', 'INFO')) self._default_converters = {'uplink': 'OcppUplinkConverter'} self._server = None @@ -95,7 +96,10 @@ def __init__(self, gateway, config, connector_type): def open(self): self.__stopped = False self.start() - log.info("Starting OCPP Connector") + self._log.info("Starting OCPP Connector") + + def get_type(self): + return self._connector_type def run(self): self._data_convert_thread.start() @@ -175,7 +179,7 @@ async def on_connect(self, websocket, path): if is_valid: uplink_converter_name = cp_config.get('extension', self._default_converters['uplink']) cp = ChargePoint(charge_point_id, websocket, {**cp_config, 'uplink_converter_name': uplink_converter_name}, - OcppConnector._callback) + OcppConnector._callback, self._log) cp.authorized = True self._log.info('Connected Charge Point with id: %s', charge_point_id) @@ -208,6 +212,7 @@ def close(self): sleep(.2) self._log.info('%s has been stopped.', self.get_name()) + self._log.reset() async def _close_cp_connections(self): for cp in self._connected_charge_points: diff --git a/thingsboard_gateway/connectors/ocpp/ocpp_converter.py b/thingsboard_gateway/connectors/ocpp/ocpp_converter.py index aa5eddfa9..b9e12920c 100644 --- a/thingsboard_gateway/connectors/ocpp/ocpp_converter.py +++ b/thingsboard_gateway/connectors/ocpp/ocpp_converter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import Converter, abstractmethod, log +from thingsboard_gateway.connectors.converter import Converter, abstractmethod class OcppConverter(Converter): diff --git a/thingsboard_gateway/connectors/ocpp/ocpp_uplink_converter.py b/thingsboard_gateway/connectors/ocpp/ocpp_uplink_converter.py index bd4a6f091..51635816e 100644 --- a/thingsboard_gateway/connectors/ocpp/ocpp_uplink_converter.py +++ b/thingsboard_gateway/connectors/ocpp/ocpp_uplink_converter.py @@ -15,12 +15,13 @@ from simplejson import dumps from time import time -from thingsboard_gateway.connectors.ocpp.ocpp_converter import OcppConverter, log +from thingsboard_gateway.connectors.ocpp.ocpp_converter import OcppConverter from thingsboard_gateway.tb_utility.tb_utility import TBUtility class OcppUplinkConverter(OcppConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config def get_device_name(self, config): @@ -42,11 +43,11 @@ def get_device_name(self, config): return device_name else: - log.error("The expression for looking \"deviceName\" not found in config %s", dumps(self.__config)) + self._log.error("The expression for looking \"deviceName\" not found in config %s", dumps(self.__config)) except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), config) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config), + config, e) def get_device_type(self, config): try: @@ -66,8 +67,8 @@ def get_device_type(self, config): return device_type except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), config) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config), + config, e) def convert(self, config, data): datatypes = {"attributes": "attributes", @@ -113,8 +114,8 @@ def convert(self, config, data): else: dict_result[datatypes[datatype]].append({full_key: full_value}) except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), str(data)) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config), + str(data), e) - log.debug(dict_result) + self._log.debug(dict_result) return dict_result diff --git a/thingsboard_gateway/connectors/odbc/odbc_connector.py b/thingsboard_gateway/connectors/odbc/odbc_connector.py index f70944f4a..e22a85f79 100644 --- a/thingsboard_gateway/connectors/odbc/odbc_connector.py +++ b/thingsboard_gateway/connectors/odbc/odbc_connector.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + from time import time from hashlib import sha1 from os import path @@ -24,6 +25,7 @@ from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader from thingsboard_gateway.tb_utility.tb_utility import TBUtility +from thingsboard_gateway.tb_utility.tb_logger import init_logger try: import pyodbc @@ -34,7 +36,7 @@ from thingsboard_gateway.connectors.odbc.odbc_uplink_converter import OdbcUplinkConverter -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector from thingsboard_gateway.gateway.statistics_service import StatisticsService @@ -56,8 +58,9 @@ def __init__(self, gateway, config, connector_type): self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} self.__gateway = gateway - self._connector_type = connector_type self.__config = config + self._log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO')) + self._connector_type = connector_type self.__stopped = False self.__config_dir = self.__gateway.get_config_path() + "odbc" + path.sep @@ -74,25 +77,29 @@ def __init__(self, gateway, config, connector_type): self.__attribute_columns = [] self.__timeseries_columns = [] - self.__converter = OdbcUplinkConverter() if not self.__config.get("converter", "") else \ + self.__converter = OdbcUplinkConverter(self._log) if not self.__config.get("converter", "") else \ TBModuleLoader.import_module(self._connector_type, self.__config["converter"]) self.__configure_pyodbc() self.__parse_rpc_config() def open(self): - log.debug("[%s] Starting...", self.get_name()) + self._log.debug("[%s] Starting...", self.get_name()) self.__stopped = False self.start() def close(self): if not self.__stopped: self.__stopped = True - log.debug("[%s] Stopping", self.get_name()) + self._log.debug("[%s] Stopping", self.get_name()) + self._log.reset() def get_name(self): return self.name + def get_type(self): + return self._connector_type + def is_connected(self): return self.__connection is not None @@ -104,15 +111,15 @@ def server_side_rpc_handler(self, content): done = False try: if not self.is_connected(): - log.warning("[%s] Cannot process RPC request: not connected to database", self.get_name()) + self._log.warning("[%s] Cannot process RPC request: not connected to database", self.get_name()) raise Exception("no connection") is_rpc_unknown = False rpc_config = self.__config["serverSideRpc"]["methods"].get(content["data"]["method"]) if rpc_config is None: if not self.__config["serverSideRpc"]["enableUnknownRpc"]: - log.warning("[%s] Ignore unknown RPC request '%s' (id=%s)", - self.get_name(), content["data"]["method"], content["data"]["id"]) + self._log.warning("[%s] Ignore unknown RPC request '%s' (id=%s)", + self.get_name(), content["data"]["method"], content["data"]["id"]) raise Exception("unknown RPC request") else: is_rpc_unknown = True @@ -127,9 +134,9 @@ def server_side_rpc_handler(self, content): sql_params = rpc_config.get("args") or rpc_config.get("params", []) query = rpc_config.get("query", "") - log.debug("[%s] Processing %s '%s' RPC request (id=%s) for '%s' device: params=%s, query=%s", - self.get_name(), "unknown" if is_rpc_unknown else "", content["data"]["method"], - content["data"]["id"], content["device"], sql_params, query) + self._log.debug("[%s] Processing %s '%s' RPC request (id=%s) for '%s' device: params=%s, query=%s", + self.get_name(), "unknown" if is_rpc_unknown else "", content["data"]["method"], + content["data"]["id"], content["device"], sql_params, query) if self.__rpc_cursor is None: self.__rpc_cursor = self.__connection.cursor() @@ -148,14 +155,16 @@ def server_side_rpc_handler(self, content): self.__rpc_cursor.execute("{{CALL {}}}".format(content["data"]["method"])) done = True - log.debug("[%s] Processed '%s' RPC request (id=%s) for '%s' device", - self.get_name(), content["data"]["method"], content["data"]["id"], content["device"]) + self._log.debug("[%s] Processed '%s' RPC request (id=%s) for '%s' device", + self.get_name(), content["data"]["method"], content["data"]["id"], content["device"]) except pyodbc.Warning as w: - log.warning("[%s] Warning while processing '%s' RPC request (id=%s) for '%s' device: %s", - self.get_name(), content["data"]["method"], content["data"]["id"], content["device"], str(w)) + self._log.warning("[%s] Warning while processing '%s' RPC request (id=%s) for '%s' device: %s", + self.get_name(), content["data"]["method"], content["data"]["id"], content["device"], + str(w)) except Exception as e: - log.error("[%s] Failed to process '%s' RPC request (id=%s) for '%s' device: %s", - self.get_name(), content["data"]["method"], content["data"]["id"], content["device"], str(e)) + self._log.error("[%s] Failed to process '%s' RPC request (id=%s) for '%s' device: %s", + self.get_name(), content["data"]["method"], content["data"]["id"], content["device"], + str(e)) finally: if done and rpc_config.get("result", self.DEFAULT_PROCESS_RPC_RESULT): response = self.row_to_dict(self.__rpc_cursor.fetchone()) @@ -171,15 +180,15 @@ def run(self): not self.__init_connection() and \ self.__config["connection"].get("reconnect", self.DEFAULT_RECONNECT_STATE): reconnect_period = self.__config["connection"].get("reconnectPeriod", self.DEFAULT_RECONNECT_PERIOD) - log.info("[%s] Will reconnect to database in %d second(s)", self.get_name(), reconnect_period) + self._log.info("[%s] Will reconnect to database in %d second(s)", self.get_name(), reconnect_period) sleep(reconnect_period) if not self.is_connected(): - log.error("[%s] Cannot connect to database so exit from main loop", self.get_name()) + self._log.error("[%s] Cannot connect to database so exit from main loop", self.get_name()) break if not self.__init_iterator(): - log.error("[%s] Cannot init database iterator so exit from main loop", self.get_name()) + self._log.error("[%s] Cannot init database iterator so exit from main loop", self.get_name()) break # Polling phase @@ -187,17 +196,17 @@ def run(self): self.__poll() if not self.__stopped: polling_period = self.__config["polling"].get("period", self.DEFAULT_POLL_PERIOD) - log.debug("[%s] Next polling iteration will be in %d second(s)", self.get_name(), polling_period) + self._log.debug("[%s] Next polling iteration will be in %d second(s)", self.get_name(), polling_period) sleep(polling_period) except pyodbc.Warning as w: - log.warning("[%s] Warning while polling database: %s", self.get_name(), str(w)) + self._log.warning("[%s] Warning while polling database: %s", self.get_name(), str(w)) except pyodbc.Error as e: - log.error("[%s] Error while polling database: %s", self.get_name(), str(e)) + self._log.error("[%s] Error while polling database: %s", self.get_name(), str(e)) self.__close() self.__close() self.__stopped = False - log.info("[%s] Stopped", self.get_name()) + self._log.info("[%s] Stopped", self.get_name()) def __close(self): if self.is_connected(): @@ -207,7 +216,7 @@ def __close(self): self.__rpc_cursor.close() self.__connection.close() finally: - log.info("[%s] Connection to database closed", self.get_name()) + self._log.info("[%s] Connection to database closed", self.get_name()) self.__connection = None self.__cursor = None self.__rpc_cursor = None @@ -218,18 +227,18 @@ def __poll(self): if not self.__column_names: for column in self.__cursor.description: self.__column_names.append(column[0]) - log.info("[%s] Fetch column names: %s", self.get_name(), self.__column_names) + self._log.info("[%s] Fetch column names: %s", self.get_name(), self.__column_names) # For some reason pyodbc.Cursor.rowcount may be 0 (sqlite) so use our own row counter row_count = 0 for row in rows: - log.debug("[%s] Fetch row: %s", self.get_name(), row) + self._log.debug("[%s] Fetch row: %s", self.get_name(), row) self.__process_row(row) row_count += 1 self.__iterator["total"] += row_count - log.info("[%s] Polling iteration finished. Processed rows: current %d, total %d", - self.get_name(), row_count, self.__iterator["total"]) + self._log.info("[%s] Polling iteration finished. Processed rows: current %d, total %d", + self.get_name(), row_count, self.__iterator["total"]) if self.__config["polling"]["iterator"]["persistent"] and row_count > 0: self.__save_iterator_config() @@ -262,7 +271,7 @@ def __process_row(self, row): self.__iterator["value"] = getattr(row, self.__iterator["name"]) self.__check_and_send(device_name, device_type, to_send) except Exception as e: - log.warning("[%s] Failed to process database row: %s", self.get_name(), str(e)) + self._log.warning("[%s] Failed to process database row: %s", self.get_name(), str(e)) @staticmethod def row_to_dict(row): @@ -301,45 +310,47 @@ def __check_and_send(self, device_name, device_type, new_data): to_send['telemetry'] = {'ts': new_data.get('ts', int(time()) * 1000), 'values': new_data['telemetry']} - log.debug("[%s] Pushing to TB server '%s' device data: %s", self.get_name(), device_name, to_send) + self._log.debug("[%s] Pushing to TB server '%s' device data: %s", self.get_name(), device_name, to_send) to_send['telemetry'] = [to_send['telemetry']] self.__gateway.send_to_storage(self.get_name(), to_send) self.statistics['MessagesSent'] += 1 else: - log.debug("[%s] '%s' device data has not been changed", self.get_name(), device_name) + self._log.debug("[%s] '%s' device data has not been changed", self.get_name(), device_name) def __init_connection(self): try: - log.debug("[%s] Opening connection to database", self.get_name()) + self._log.debug("[%s] Opening connection to database", self.get_name()) connection_config = self.__config["connection"] self.__connection = pyodbc.connect(connection_config["str"], **connection_config.get("attributes", {})) if connection_config.get("encoding", ""): - log.info("[%s] Setting encoding to %s", self.get_name(), connection_config["encoding"]) + self._log.info("[%s] Setting encoding to %s", self.get_name(), connection_config["encoding"]) self.__connection.setencoding(connection_config["encoding"]) decoding_config = connection_config.get("decoding") if decoding_config is not None: if isinstance(decoding_config, dict): if decoding_config.get("char", ""): - log.info("[%s] Setting SQL_CHAR decoding to %s", self.get_name(), decoding_config["char"]) + self._log.info("[%s] Setting SQL_CHAR decoding to %s", self.get_name(), decoding_config["char"]) self.__connection.setdecoding(pyodbc.SQL_CHAR, decoding_config["char"]) if decoding_config.get("wchar", ""): - log.info("[%s] Setting SQL_WCHAR decoding to %s", self.get_name(), decoding_config["wchar"]) + self._log.info("[%s] Setting SQL_WCHAR decoding to %s", self.get_name(), + decoding_config["wchar"]) self.__connection.setdecoding(pyodbc.SQL_WCHAR, decoding_config["wchar"]) if decoding_config.get("metadata", ""): - log.info("[%s] Setting SQL_WMETADATA decoding to %s", - self.get_name(), decoding_config["metadata"]) + self._log.info("[%s] Setting SQL_WMETADATA decoding to %s", + self.get_name(), decoding_config["metadata"]) self.__connection.setdecoding(pyodbc.SQL_WMETADATA, decoding_config["metadata"]) else: - log.warning("[%s] Unknown decoding configuration %s. Read data may be misdecoded", self.get_name(), - decoding_config) + self._log.warning("[%s] Unknown decoding configuration %s. Read data may be misdecoded", + self.get_name(), + decoding_config) self.__cursor = self.__connection.cursor() - log.info("[%s] Connection to database opened, attributes %s", - self.get_name(), connection_config.get("attributes", {})) + self._log.info("[%s] Connection to database opened, attributes %s", + self.get_name(), connection_config.get("attributes", {})) except pyodbc.Error as e: - log.error("[%s] Failed to connect to database: %s", self.get_name(), str(e)) + self._log.error("[%s] Failed to connect to database: %s", self.get_name(), str(e)) self.__close() return self.is_connected() @@ -357,9 +368,9 @@ def __resolve_iterator_file(self): file_name += self.__config["polling"]["iterator"]["column"] self.__iterator_file_name = sha1(file_name.encode()).hexdigest() + ".json" - log.debug("[%s] Iterator file name resolved to %s", self.get_name(), self.__iterator_file_name) + self._log.debug("[%s] Iterator file name resolved to %s", self.get_name(), self.__iterator_file_name) except Exception as e: - log.warning("[%s] Failed to resolve iterator file name: %s", self.get_name(), str(e)) + self._log.warning("[%s] Failed to resolve iterator file name: %s", self.get_name(), str(e)) return bool(self.__iterator_file_name) def __init_iterator(self): @@ -369,10 +380,10 @@ def __init_iterator(self): else: save_iterator = self.__config["polling"]["iterator"]["persistent"] - log.info("[%s] Iterator saving %s", self.get_name(), "enabled" if save_iterator else "disabled") + self._log.info("[%s] Iterator saving %s", self.get_name(), "enabled" if save_iterator else "disabled") if save_iterator and self.__load_iterator_config(): - log.info("[%s] Init iterator from file '%s': column=%s, start_value=%s", + self._log.info("[%s] Init iterator from file '%s': column=%s, start_value=%s", self.get_name(), self.__iterator_file_name, self.__iterator["name"], self.__iterator["value"]) return True @@ -382,20 +393,20 @@ def __init_iterator(self): if "value" in self.__config["polling"]["iterator"]: self.__iterator["value"] = self.__config["polling"]["iterator"]["value"] - log.info("[%s] Init iterator from configuration: column=%s, start_value=%s", - self.get_name(), self.__iterator["name"], self.__iterator["value"]) + self._log.info("[%s] Init iterator from configuration: column=%s, start_value=%s", + self.get_name(), self.__iterator["name"], self.__iterator["value"]) elif "query" in self.__config["polling"]["iterator"]: try: self.__iterator["value"] = \ self.__cursor.execute(self.__config["polling"]["iterator"]["query"]).fetchone()[0] - log.info("[%s] Init iterator from database: column=%s, start_value=%s", - self.get_name(), self.__iterator["name"], self.__iterator["value"]) + self._log.info("[%s] Init iterator from database: column=%s, start_value=%s", + self.get_name(), self.__iterator["name"], self.__iterator["value"]) except pyodbc.Warning as w: - log.warning("[%s] Warning on init iterator from database: %s", self.get_name(), str(w)) + self._log.warning("[%s] Warning on init iterator from database: %s", self.get_name(), str(w)) except pyodbc.Error as e: - log.error("[%s] Failed to init iterator from database: %s", self.get_name(), str(e)) + self._log.error("[%s] Failed to init iterator from database: %s", self.get_name(), str(e)) else: - log.error("[%s] Failed to init iterator: value/query param is absent", self.get_name()) + self._log.error("[%s] Failed to init iterator: value/query param is absent", self.get_name()) return "value" in self.__iterator @@ -404,16 +415,16 @@ def __save_iterator_config(self): Path(self.__config_dir).mkdir(exist_ok=True) with Path(self.__config_dir + self.__iterator_file_name).open("w") as iterator_file: iterator_file.write(dumps(self.__iterator, indent=2, sort_keys=True)) - log.debug("[%s] Saved iterator configuration to %s", self.get_name(), self.__iterator_file_name) + self._log.debug("[%s] Saved iterator configuration to %s", self.get_name(), self.__iterator_file_name) except Exception as e: - log.error("[%s] Failed to save iterator configuration to %s: %s", - self.get_name(), self.__iterator_file_name, str(e)) + self._log.error("[%s] Failed to save iterator configuration to %s: %s", + self.get_name(), self.__iterator_file_name, str(e)) def __load_iterator_config(self): if not self.__iterator_file_name: if not self.__resolve_iterator_file(): - log.error("[%s] Unable to load iterator configuration from file: file name is not resolved", - self.get_name()) + self._log.error("[%s] Unable to load iterator configuration from file: file name is not resolved", + self.get_name()) return False try: @@ -423,10 +434,10 @@ def __load_iterator_config(self): with iterator_file_path.open("r") as iterator_file: self.__iterator = load(iterator_file) - log.debug("[%s] Loaded iterator configuration from %s", self.get_name(), self.__iterator_file_name) + self._log.debug("[%s] Loaded iterator configuration from %s", self.get_name(), self.__iterator_file_name) except Exception as e: - log.error("[%s] Failed to load iterator configuration from %s: %s", - self.get_name(), self.__iterator_file_name, str(e)) + self._log.error("[%s] Failed to load iterator configuration from %s: %s", + self.get_name(), self.__iterator_file_name, str(e)) return bool(self.__iterator) @@ -438,7 +449,7 @@ def __configure_pyodbc(self): for name, value in pyodbc_config.items(): pyodbc.__dict__[name] = value - log.info("[%s] Set pyodbc attributes: %s", self.get_name(), pyodbc_config) + self._log.info("[%s] Set pyodbc attributes: %s", self.get_name(), pyodbc_config) def __parse_rpc_config(self): if "serverSideRpc" not in self.__config: @@ -446,14 +457,14 @@ def __parse_rpc_config(self): if "enableUnknownRpc" not in self.__config["serverSideRpc"]: self.__config["serverSideRpc"]["enableUnknownRpc"] = self.DEFAULT_ENABLE_UNKNOWN_RPC - log.info("[%s] Processing unknown RPC %s", self.get_name(), - "enabled" if self.__config["serverSideRpc"]["enableUnknownRpc"] else "disabled") + self._log.info("[%s] Processing unknown RPC %s", self.get_name(), + "enabled" if self.__config["serverSideRpc"]["enableUnknownRpc"] else "disabled") if "overrideRpcConfig" not in self.__config["serverSideRpc"]: self.__config["serverSideRpc"]["overrideRpcConfig"] = self.DEFAULT_OVERRIDE_RPC_PARAMS - log.info("[%s] Overriding RPC config %s", self.get_name(), - "enabled" if self.__config["serverSideRpc"]["overrideRpcConfig"] else "disabled") + self._log.info("[%s] Overriding RPC config %s", self.get_name(), + "enabled" if self.__config["serverSideRpc"]["overrideRpcConfig"] else "disabled") if "serverSideRpc" not in self.__config or not self.__config["serverSideRpc"].get("methods", []): self.__config["serverSideRpc"] = {"methods": {}} @@ -466,8 +477,8 @@ def __parse_rpc_config(self): elif isinstance(rpc_config, dict): reformatted_config[rpc_config["name"]] = rpc_config else: - log.warning("[%s] Wrong RPC config format. Expected str or dict, get %s", self.get_name(), - type(rpc_config)) + self._log.warning("[%s] Wrong RPC config format. Expected str or dict, get %s", self.get_name(), + type(rpc_config)) self.__config["serverSideRpc"]["methods"] = reformatted_config diff --git a/thingsboard_gateway/connectors/odbc/odbc_uplink_converter.py b/thingsboard_gateway/connectors/odbc/odbc_uplink_converter.py index 003c65da4..dcd3b114c 100644 --- a/thingsboard_gateway/connectors/odbc/odbc_uplink_converter.py +++ b/thingsboard_gateway/connectors/odbc/odbc_uplink_converter.py @@ -12,13 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import log from thingsboard_gateway.connectors.odbc.odbc_converter import OdbcConverter from thingsboard_gateway.gateway.statistics_service import StatisticsService class OdbcUplinkConverter(OdbcConverter): + def __init__(self, logger): + self._log = logger + @StatisticsService.CollectStatistics(start_stat_type='receivedBytesFromDevices', end_stat_type='convertedBytesFromDevice') def convert(self, config, data): @@ -41,12 +43,12 @@ def convert(self, config, data): elif "value" in config_item: converted_data[name] = eval(config_item["value"], globals(), data) else: - log.error("Failed to convert SQL data to TB format: no column/value configuration item") + self._log.error("Failed to convert SQL data to TB format: no column/value configuration item") else: - log.error("Failed to convert SQL data to TB format: unexpected configuration type '%s'", + self._log.error("Failed to convert SQL data to TB format: unexpected configuration type '%s'", type(config_item)) except Exception as e: - log.error("Failed to convert SQL data to TB format: %s", str(e)) + self._log.error("Failed to convert SQL data to TB format: %s", str(e)) if data.get('ts'): converted_data['ts'] = data.get('ts') diff --git a/thingsboard_gateway/connectors/opcua/opcua_connector.py b/thingsboard_gateway/connectors/opcua/opcua_connector.py index d190db977..5f15b4c58 100644 --- a/thingsboard_gateway/connectors/opcua/opcua_connector.py +++ b/thingsboard_gateway/connectors/opcua/opcua_connector.py @@ -27,6 +27,7 @@ from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader from thingsboard_gateway.tb_utility.tb_utility import TBUtility from thingsboard_gateway.gateway.statistics_service import StatisticsService +from thingsboard_gateway.tb_utility.tb_logger import init_logger try: from opcua import Client, Node, ua @@ -41,7 +42,7 @@ TBUtility.install_package("cryptography") from opcua.crypto import uacrypto -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector from thingsboard_gateway.connectors.opcua.opcua_uplink_converter import OpcUaUplinkConverter @@ -52,7 +53,11 @@ def __init__(self, gateway, config, connector_type): 'MessagesSent': 0} super().__init__() self.__gateway = gateway + self._config = config self.__server_conf = config.get("server") + self.setName(self._config.get("name", 'OPC-UA ' + ''.join( + choice(ascii_lowercase) for _ in range(5))) + " Connector") + self._log = init_logger(self.__gateway, self.name, self._config.get('logLevel', 'INFO')) self.__interest_nodes = [] self.__available_object_resources = {} self.__show_map = self.__server_conf.get("showMap", False) @@ -61,8 +66,9 @@ def __init__(self, gateway, config, connector_type): if mapping.get("deviceNodePattern") is not None: self.__interest_nodes.append({mapping["deviceNodePattern"]: mapping}) else: - log.error("deviceNodePattern in mapping: %s - not found, add property deviceNodePattern to processing this mapping", - dumps(mapping)) + self._log.error( + "deviceNodePattern in mapping: %s - not found, add property deviceNodePattern to processing this mapping", + dumps(mapping)) if "opc.tcp" not in self.__server_conf.get("url"): self.__opcua_url = "opc.tcp://" + self.__server_conf.get("url") else: @@ -71,7 +77,6 @@ def __init__(self, gateway, config, connector_type): self.client = None self.__connected = False - self.setName(self.__server_conf.get("name", 'OPC-UA ' + ''.join(choice(ascii_lowercase) for _ in range(5))) + " Connector") self.__sub_handler = SubHandler(self) self.data_to_send = [] self.__stopped = False @@ -80,10 +85,13 @@ def __init__(self, gateway, config, connector_type): def is_connected(self): return self.__connected + def get_type(self): + return self._connector_type + def open(self): self.__stopped = False self.start() - log.info("Starting OPC-UA Connector") + self._log.info("Starting OPC-UA Connector") def __create_client(self): if self.client: @@ -119,27 +127,29 @@ def __connect(self): try: self.client.load_type_definitions() except Exception as e: - log.error("Error on loading type definitions:") - log.error(e) - log.debug(self.client.get_namespace_array()[-1]) - log.debug(self.client.get_namespace_index(self.client.get_namespace_array()[-1])) + self._log.error("Error on loading type definitions:\n %s", e) + self._log.debug(self.client.get_namespace_array()[-1]) + self._log.debug(self.client.get_namespace_index(self.client.get_namespace_array()[-1])) self.__initialize_client() if not self.__server_conf.get("disableSubscriptions", False): - self.__sub = self.client.create_subscription(self.__server_conf.get("subCheckPeriodInMillis", 500), self.__sub_handler) + self.__sub = self.client.create_subscription(self.__server_conf.get("subCheckPeriodInMillis", 500), + self.__sub_handler) self.__connected = True - log.info("OPC-UA connector %s connected to server %s", self.get_name(), self.__server_conf.get("url")) + self._log.info("OPC-UA connector %s connected to server %s", self.get_name(), + self.__server_conf.get("url")) except ConnectionRefusedError: - log.error("Connection refused on connection to OPC-UA server with url %s", self.__server_conf.get("url")) + self._log.error("Connection refused on connection to OPC-UA server with url %s", + self.__server_conf.get("url")) time.sleep(10) except OSError: - log.error("Connection refused on connection to OPC-UA server with url %s", self.__server_conf.get("url")) + self._log.error("Connection refused on connection to OPC-UA server with url %s", + self.__server_conf.get("url")) time.sleep(10) except Exception as e: - log.debug("error on connection to OPC-UA server.") - log.error(e) + self._log.error("error on connection to OPC-UA server.\n %s", e) time.sleep(10) def run(self): @@ -170,9 +180,8 @@ def run(self): except FuturesTimeoutError: self.__check_connection() except Exception as e: - log.error("Connection failed on connection to OPC-UA server with url %s", - self.__server_conf.get("url")) - log.exception(e) + self._log.error("Connection failed on connection to OPC-UA server with url %s\n %s", + self.__server_conf.get("url"), e) time.sleep(10) @@ -184,7 +193,7 @@ def __set_auth_settings_by_cert(self): security_mode = self.__server_conf["identity"].get("mode", "SignAndEncrypt") policy = self.__server_conf["security"] if cert is None or private_key is None: - log.exception("Error in ssl configuration - cert or privateKey parameter not found") + self._log.exception("Error in ssl configuration - cert or privateKey parameter not found") raise RuntimeError("Error in ssl configuration - cert or privateKey parameter not found") security_string = policy + ',' + security_mode + ',' + cert + ',' + private_key if ca_cert is not None: @@ -192,7 +201,7 @@ def __set_auth_settings_by_cert(self): self.client.set_security_string(security_string) except Exception as e: - log.exception(e) + self._log.exception(e) def __set_auth_settings_by_username(self): self.client.set_user(self.__server_conf["identity"].get("username")) @@ -216,7 +225,7 @@ def __check_connection(self): self.__connected = False except Exception as e: self.__connected = False - log.exception(e) + self._log.exception(e) def close(self): self.__stopped = True @@ -227,39 +236,55 @@ def close(self): except: pass self.__connected = False - log.info('%s has been stopped.', self.get_name()) + self._log.info('%s has been stopped.', self.get_name()) + self._log.reset() def get_name(self): return self.name @StatisticsService.CollectAllReceivedBytesStatistics(start_stat_type='allReceivedBytesFromTB') def on_attributes_update(self, content): - log.debug(content) + self._log.debug(content) try: for server_variables in self.__available_object_resources[content["device"]]['variables']: for attribute in content["data"]: for variable in server_variables: if attribute == variable: try: - if ( isinstance(content["data"][variable], int) ): - dv = ua.DataValue(ua.Variant(content["data"][variable], server_variables[variable].get_data_type_as_variant_type())) + if isinstance(content["data"][variable], int): + dv = ua.DataValue(ua.Variant(content["data"][variable], server_variables[ + variable].get_data_type_as_variant_type())) server_variables[variable].set_value(dv) else: server_variables[variable].set_value(content["data"][variable]) except Exception: - server_variables[variable].set_attribute(ua.AttributeIds.Value, ua.DataValue(content["data"][variable])) + server_variables[variable].set_attribute(ua.AttributeIds.Value, + ua.DataValue(content["data"][variable])) except Exception as e: - log.exception(e) + self._log.exception(e) @StatisticsService.CollectAllReceivedBytesStatistics(start_stat_type='allReceivedBytesFromTB') def server_side_rpc_handler(self, content): try: + if content.get('data') is None: + content['data'] = {'params': content['params'], 'method': content['method']} + rpc_method = content["data"].get("method") + # check if RPC type is connector RPC (can be only 'get' or 'set') + try: + (connector_type, rpc_method_name) = rpc_method.split('_') + if connector_type == self._connector_type: + rpc_method = rpc_method_name + content['device'] = content['params'].split(' ')[0].split('=')[-1] + except (ValueError, IndexError): + pass + # firstly check if a method is not service if rpc_method == 'set' or rpc_method == 'get': full_path = '' args_list = [] + device = content.get('device') try: args_list = content['data']['params'].split(';') @@ -268,45 +293,46 @@ def server_side_rpc_handler(self, content): else: full_path = args_list[0].split('=')[-1] except IndexError: - log.error('Not enough arguments. Expected min 2.') - self.__gateway.send_rpc_reply(content['device'], - content['data']['id'], - {content['data']['method']: 'Not enough arguments. Expected min 2.', - 'code': 400}) + self._log.error('Not enough arguments. Expected min 2.') + self.__gateway.send_rpc_reply(device=device, + req_id=content['data'].get('id'), + content={content['data'][ + 'method']: 'Not enough arguments. Expected min 2.', + 'code': 400}) node_list = [] - self.__search_node(current_node=content['device'], fullpath=full_path, result=node_list) + self.__search_node(current_node=device, fullpath=full_path, result=node_list) node = None try: node = node_list[0] except IndexError: - self.__gateway.send_rpc_reply(content['device'], content['data']['id'], - {content['data']['method']: 'Node didn\'t find!', - 'code': 500}) + self.__gateway.send_rpc_reply(device=device, req_id=content['data'].get('id'), + content={content['data']['method']: 'Node didn\'t find!', + 'code': 500}) if rpc_method == 'get': - self.__gateway.send_rpc_reply(content['device'], - content['data']['id'], - {content['data']['method']: node.get_value(), - 'code': 200}) + self.__gateway.send_rpc_reply(device=device, + req_id=content['data'].get('id'), + content={content['data']['method']: node.get_value(), 'code': 200}) else: try: value = args_list[2].split('=')[-1] node.set_value(value) - self.__gateway.send_rpc_reply(content['device'], - content['data']['id'], - {'success': 'true', 'code': 200}) + self.__gateway.send_rpc_reply(device=device, + req_id=content['data'].get('id'), + content={'success': 'true', 'code': 200}) except ValueError: - log.error('Method SET take three arguments!') - self.__gateway.send_rpc_reply(content['device'], - content['data']['id'], - {'error': 'Method SET take three arguments!', 'code': 400}) + self._log.error('Method SET take three arguments!') + self.__gateway.send_rpc_reply(device=device, + req_id=content['data'].get('id'), + content={'error': 'Method SET take three arguments!', + 'code': 400}) except ua.UaStatusCodeError: - log.error('Write method doesn\'t allow!') - self.__gateway.send_rpc_reply(content['device'], - content['data']['id'], - {'error': 'Write method doesn\'t allow!', 'code': 400}) + self._log.error('Write method doesn\'t allow!') + self.__gateway.send_rpc_reply(device=device, + req_id=content['data'].get('id'), + content={'error': 'Write method doesn\'t allow!', 'code': 400}) for method in self.__available_object_resources[content["device"]]['methods']: if rpc_method is not None and method.get(rpc_method) is not None: @@ -328,24 +354,25 @@ def server_side_rpc_handler(self, content): self.__gateway.send_rpc_reply(content["device"], content["data"]["id"], {content["data"]["method"]: result, "code": 200}) - log.debug("method %s result is: %s", method[rpc_method], result) + self._log.debug("method %s result is: %s", method[rpc_method], result) except Exception as e: - log.exception(e) + self._log.exception(e) self.__gateway.send_rpc_reply(content["device"], content["data"]["id"], {"error": str(e), "code": 500}) else: - log.error("Method %s not found for device %s", rpc_method, content["device"]) - self.__gateway.send_rpc_reply(content["device"], content["data"]["id"], {"error": "%s - Method not found" % (rpc_method), "code": 404}) + self._log.error("Method %s not found for device %s", rpc_method, content["device"]) + self.__gateway.send_rpc_reply(content["device"], content["data"]["id"], + {"error": "%s - Method not found" % rpc_method, "code": 404}) except Exception as e: - log.exception(e) + self._log.exception(e) def __initialize_client(self): self.__opcua_nodes["root"] = self.client.get_objects_node() self.__opcua_nodes["objects"] = self.client.get_objects_node() self.scan_nodes_from_config() self.__previous_scan_time = time.time() * 1000 - log.debug('Subscriptions: %s', self.subscribed) - log.debug("Available methods: %s", self.__available_object_resources) + self._log.debug('Subscriptions: %s', self.subscribed) + self._log.debug("Available methods: %s", self.__available_object_resources) def scan_nodes_from_config(self): try: @@ -361,20 +388,21 @@ def scan_nodes_from_config(self): self.__save_methods(device_info) self.__search_attribute_update_variables(device_info) else: - log.error("Device node is None, please check your configuration.") - log.debug("Current device node is: %s", str(device_configuration.get("deviceNodePattern"))) + self._log.error("Device node is None, please check your configuration.") + self._log.debug("Current device node is: %s", + str(device_configuration.get("deviceNodePattern"))) break except BrokenPipeError: - log.debug("Broken Pipe. Connection lost.") + self._log.debug("Broken Pipe. Connection lost.") except OSError: - log.debug("Stop on scanning.") + self._log.debug("Stop on scanning.") except FuturesTimeoutError: self.__check_connection() except Exception as e: - log.exception(e) - log.debug(self.__interest_nodes) + self._log.exception(e) + self._log.debug(self.__interest_nodes) except Exception as e: - log.exception(e) + self._log.exception(e) def __search_nodes_and_subscribe(self, device_info): sub_nodes = [] @@ -394,7 +422,7 @@ def __search_nodes_and_subscribe(self, device_info): try: information_value = information_node.get_value() except: - log.error("Err get_value: %s", str(information_node)) + self._log.error("Err get_value: %s", str(information_node)) continue if device_info.get("uplink_converter") is None: @@ -402,9 +430,11 @@ def __search_nodes_and_subscribe(self, device_info): "deviceName": device_info["deviceName"], "deviceType": device_info["deviceType"]} if device_info["configuration"].get('converter') is None: - converter = OpcUaUplinkConverter(configuration) + converter = OpcUaUplinkConverter(configuration, self._log) else: - converter = TBModuleLoader.import_module(self._connector_type, device_info["configuration"].get('converter'))(configuration) + converter = TBModuleLoader.import_module(self._connector_type, + device_info["configuration"].get('converter'))( + configuration, self._log) device_info["uplink_converter"] = converter else: converter = device_info["uplink_converter"] @@ -421,11 +451,11 @@ def __search_nodes_and_subscribe(self, device_info): information_key = information['key'] - log.debug("Node for %s \"%s\" with path: %s - FOUND! Current values is: %s", - information_type, - information_key, - information_path, - str(information_value)) + self._log.debug("Node for %s \"%s\" with path: %s - FOUND! Current values is: %s", + information_type, + information_key, + information_path, + str(information_value)) if not device_info.get(information_types[information_type]): device_info[information_types[information_type]] = [] @@ -434,12 +464,13 @@ def __search_nodes_and_subscribe(self, device_info): self.statistics['MessagesReceived'] = self.statistics['MessagesReceived'] + 1 self.data_to_send.append(converted_data) self.statistics['MessagesSent'] = self.statistics['MessagesSent'] + 1 - log.debug("Data to ThingsBoard: %s", converted_data) + self._log.debug("Data to ThingsBoard: %s", converted_data) if not self.__server_conf.get("disableSubscriptions", False): sub_nodes.append(information_node) else: - log.error("Node for %s \"%s\" with path %s - NOT FOUND!", information_type, information_key, information_path) + self._log.error("Node for %s \"%s\" with path %s - NOT FOUND!", information_type, + information_key, information_path) if changed_key: information['key'] = None @@ -450,7 +481,7 @@ def __search_nodes_and_subscribe(self, device_info): self.__sub_handler) if sub_nodes: self.__sub.subscribe_data_change(sub_nodes) - log.debug("Added subscription to nodes: %s", str(sub_nodes)) + self._log.debug("Added subscription to nodes: %s", str(sub_nodes)) def __save_methods(self, device_info): try: @@ -470,9 +501,9 @@ def __save_methods(self, device_info): self.__available_object_resources[device_info["deviceName"]]["methods"].append( {node_method_name: method, "node": node, "arguments": method_object.get("arguments")}) else: - log.error("Node for method with path %s - NOT FOUND!", method_node_path) + self._log.error("Node for method with path %s - NOT FOUND!", method_node_path) except Exception as e: - log.exception(e) + self._log.exception(e) def __search_attribute_update_variables(self, device_info): try: @@ -489,24 +520,27 @@ def __search_attribute_update_variables(self, device_info): self.__search_node(node, attribute_path, result=attribute_nodes) for attribute_node in attribute_nodes: if attribute_node is not None: - if self.get_node_path(attribute_node) == attribute_path: - self.__available_object_resources[device_name]["variables"].append({attribute_update["attributeOnThingsBoard"]: attribute_node}) + if self.get_node_path(attribute_node) == attribute_path: + self.__available_object_resources[device_name]["variables"].append( + {attribute_update["attributeOnThingsBoard"]: attribute_node}) else: - log.error("Attribute update node with path \"%s\" - NOT FOUND!", attribute_path) + self._log.error("Attribute update node with path \"%s\" - NOT FOUND!", attribute_path) except Exception as e: - log.exception(e) + self._log.exception(e) def __search_general_info(self, device): result = [] match_devices = [] - self.__search_node(self.__opcua_nodes["root"], TBUtility.get_value(device["deviceNodePattern"], get_tag=True), result=match_devices) + self.__search_node(self.__opcua_nodes["root"], TBUtility.get_value(device["deviceNodePattern"], get_tag=True), + result=match_devices) for device_node in match_devices: if device_node is not None: - result_device_dict = {"deviceName": None, "deviceType": None, "deviceNode": device_node, "configuration": deepcopy(device)} + result_device_dict = {"deviceName": None, "deviceType": None, "deviceNode": device_node, + "configuration": deepcopy(device)} name_pattern_config = device["deviceNamePattern"] name_expression = TBUtility.get_value(name_pattern_config, get_tag=True) if "${" in name_pattern_config and "}" in name_pattern_config: - log.debug("Looking for device name") + self._log.debug("Looking for device name") device_name_from_node = "" if name_expression == "$DisplayName": device_name_from_node = device_node.get_display_name().Text @@ -519,20 +553,21 @@ def __search_general_info(self, device): device_name_node = [] self.__search_node(device_node, name_path, result=device_name_node) if len(device_name_node) == 0: - log.warn("Device name node - not found, skipping device...") + self._log.warn("Device name node - not found, skipping device...") continue device_name_node = device_name_node[0] if device_name_node is not None: device_name_from_node = device_name_node.get_value() if device_name_from_node == "": - log.error("Device name node not found with expression: %s", name_expression) + self._log.error("Device name node not found with expression: %s", name_expression) return None - full_device_name = name_pattern_config.replace("${" + name_expression + "}", str(device_name_from_node)).replace( + full_device_name = name_pattern_config.replace("${" + name_expression + "}", + str(device_name_from_node)).replace( name_expression, str(device_name_from_node)) else: full_device_name = name_expression result_device_dict["deviceName"] = full_device_name - log.debug("Device name: %s", full_device_name) + self._log.debug("Device name: %s", full_device_name) if device.get("deviceTypePattern"): device_type_expression = TBUtility.get_value(device["deviceTypePattern"], get_tag=True) @@ -544,20 +579,22 @@ def __search_general_info(self, device): if device_type_node is not None: device_type = device_type_node.get_value() full_device_type = device_type_expression.replace("${" + device_type_expression + "}", - device_type).replace(device_type_expression, - device_type) + device_type).replace( + device_type_expression, + device_type) else: - log.error("Device type node not found with expression: %s", device_type_expression) + self._log.error("Device type node not found with expression: %s", device_type_expression) full_device_type = "default" else: full_device_type = device_type_expression result_device_dict["deviceType"] = full_device_type - log.debug("Device type: %s", full_device_type) + self._log.debug("Device type: %s", full_device_type) else: result_device_dict["deviceType"] = "default" result.append(result_device_dict) else: - log.error("Device node not found with expression: %s", TBUtility.get_value(device["deviceNodePattern"], get_tag=True)) + self._log.error("Device node not found with expression: %s", + TBUtility.get_value(device["deviceNodePattern"], get_tag=True)) return result @cached(cache=TTLCache(maxsize=1000, ttl=10 * 60)) @@ -570,12 +607,12 @@ def __search_node(self, current_node, fullpath, search_method=False, result=None try: if regex.match(r"ns=\d*;[isgb]=.*", fullpath, regex.IGNORECASE): if self.__show_map: - log.debug("Looking for node with config") + self._log.debug("Looking for node with config") node = self.client.get_node(fullpath) if node is None: - log.warning("NODE NOT FOUND - using configuration %s", fullpath) + self._log.warning("NODE NOT FOUND - using configuration %s", fullpath) else: - log.debug("Found in %s", node) + self._log.debug("Found in %s", node) result.append(node) else: fullpath_pattern = regex.compile(fullpath) @@ -604,14 +641,14 @@ def __search_node(self, current_node, fullpath, search_method=False, result=None nnp1 = new_node_path.replace('\\\\.', '.') nnp2 = new_node_path.replace('\\\\', '\\') if self.__show_map: - log.debug("SHOW MAP: Current node path: %s", new_node_path) + self._log.debug("SHOW MAP: Current node path: %s", new_node_path) regex_fullmatch = regex.fullmatch(fullpath_pattern, nnp1) or \ nnp2 == full1 or \ nnp2 == fullpath or \ nnp1 == full1 if regex_fullmatch: if self.__show_map: - log.debug("SHOW MAP: Current node path: %s - NODE FOUND", nnp2) + self._log.debug("SHOW MAP: Current node path: %s - NODE FOUND", nnp2) result.append(child_node) else: regex_search = fullpath_pattern.fullmatch(nnp1, partial=True) or \ @@ -619,25 +656,25 @@ def __search_node(self, current_node, fullpath, search_method=False, result=None nnp1 in full1 if regex_search: if self.__show_map: - log.debug("SHOW MAP: Current node path: %s - NODE FOUND", new_node_path) + self._log.debug("SHOW MAP: Current node path: %s - NODE FOUND", new_node_path) if new_node_class == ua.NodeClass.Object: if self.__show_map: - log.debug("SHOW MAP: Search in %s", new_node_path) + self._log.debug("SHOW MAP: Search in %s", new_node_path) self.__search_node(child_node, fullpath, result=result) # elif new_node_class == ua.NodeClass.Variable: - # log.debug("Found in %s", new_node_path) + # self._log.debug("Found in %s", new_node_path) # result.append(child_node) elif new_node_class == ua.NodeClass.Method and search_method: - log.debug("Found in %s", new_node_path) + self._log.debug("Found in %s", new_node_path) result.append(child_node) except CancelledError: - log.error("Request during search has been canceled by the OPC-UA server.") + self._log.error("Request during search has been canceled by the OPC-UA server.") except BrokenPipeError: - log.error("Broken Pipe. Connection lost.") + self._log.error("Broken Pipe. Connection lost.") except OSError: - log.debug("Stop on scanning.") + self._log.debug("Stop on scanning.") except Exception as e: - log.exception(e) + self._log.exception(e) def _check_path(self, config_path, node): if regex.match(r"ns=\d*;[isgb]=.*", config_path, regex.IGNORECASE): @@ -665,7 +702,7 @@ def get_converters(self): return [item['converter'] for _, item in self.subscribed.items()] def update_converter_config(self, converter_name, config): - log.debug('Received remote converter configuration update for %s with configuration %s', converter_name, + self._log.debug('Received remote converter configuration update for %s with configuration %s', converter_name, config) converters = self.get_converters() for converter_class_obj in converters: @@ -673,7 +710,7 @@ def update_converter_config(self, converter_name, config): converter_obj = converter_class_obj if converter_class_name == converter_name: converter_obj.config = config - log.info('Updated converter configuration for: %s with configuration %s', + self._log.info('Updated converter configuration for: %s with configuration %s', converter_name, converter_obj.config) for node_config in self.__server_conf['mapping']: @@ -682,27 +719,28 @@ def update_converter_config(self, converter_name, config): self.__gateway.update_connector_config_file(self.name, {'server': self.__server_conf}) + class SubHandler(object): def __init__(self, connector: OpcUaConnector): self.connector = connector def datachange_notification(self, node, val, data): try: - log.debug("Python: New data change event on node %s, with val: %s and data %s", node, val, str(data)) + self.connector._log.debug("Python: New data change event on node %s, with val: %s and data %s", node, val, str(data)) subscription = self.connector.subscribed[node] converted_data = subscription["converter"].convert((subscription["config_path"], subscription["path"]), val, data, key=subscription.get('key')) self.connector.statistics['MessagesReceived'] = self.connector.statistics['MessagesReceived'] + 1 self.connector.data_to_send.append(converted_data) self.connector.statistics['MessagesSent'] = self.connector.statistics['MessagesSent'] + 1 - log.debug("[SUBSCRIPTION] Data to ThingsBoard: %s", converted_data) + self.connector._log.debug("[SUBSCRIPTION] Data to ThingsBoard: %s", converted_data) except KeyError: self.connector.scan_nodes_from_config() except Exception as e: - log.exception(e) + self.connector._log.exception(e) def event_notification(self, event): try: - log.debug("Python: New event %s", event) + self.connector._log.debug("Python: New event %s", event) except Exception as e: - log.exception(e) + self.connector._log.exception(e) diff --git a/thingsboard_gateway/connectors/opcua/opcua_converter.py b/thingsboard_gateway/connectors/opcua/opcua_converter.py index 6237fdfb8..618d0533a 100644 --- a/thingsboard_gateway/connectors/opcua/opcua_converter.py +++ b/thingsboard_gateway/connectors/opcua/opcua_converter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import Converter, abstractmethod, log +from thingsboard_gateway.connectors.converter import Converter, abstractmethod class OpcUaConverter(Converter): diff --git a/thingsboard_gateway/connectors/opcua/opcua_uplink_converter.py b/thingsboard_gateway/connectors/opcua/opcua_uplink_converter.py index bebbede71..17c7c25ef 100644 --- a/thingsboard_gateway/connectors/opcua/opcua_uplink_converter.py +++ b/thingsboard_gateway/connectors/opcua/opcua_uplink_converter.py @@ -16,13 +16,14 @@ from time import time from datetime import timezone -from thingsboard_gateway.connectors.opcua.opcua_converter import OpcUaConverter, log +from thingsboard_gateway.connectors.opcua.opcua_converter import OpcUaConverter from thingsboard_gateway.tb_utility.tb_utility import TBUtility from thingsboard_gateway.gateway.statistics_service import StatisticsService class OpcUaUplinkConverter(OpcUaConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config @property @@ -70,4 +71,4 @@ def convert(self, config, val, data=None, key=None): result[information_types[information_type]].append({full_key: full_value}) return result except Exception as e: - log.exception(e) + self._log.exception(e) diff --git a/thingsboard_gateway/connectors/opcua_asyncio/device.py b/thingsboard_gateway/connectors/opcua_asyncio/device.py index 1311c1e1c..2fd425f4b 100644 --- a/thingsboard_gateway/connectors/opcua_asyncio/device.py +++ b/thingsboard_gateway/connectors/opcua_asyncio/device.py @@ -13,11 +13,11 @@ # limitations under the License. import re -from thingsboard_gateway.connectors.connector import log class Device: - def __init__(self, path, name, config, converter, converter_for_sub): + def __init__(self, path, name, config, converter, converter_for_sub, logger): + self._log = logger self.path = path self.name = name self.config = config @@ -46,4 +46,4 @@ def load_values(self): {'path': self.path + child.groups()[0].split('\\.'), 'key': node_config['key']}) except KeyError as e: - log.error('Invalid config for %s (key %s not found)', node_config, e) + self._log.error('Invalid config for %s (key %s not found)', node_config, e) diff --git a/thingsboard_gateway/connectors/opcua_asyncio/opcua_connector.py b/thingsboard_gateway/connectors/opcua_asyncio/opcua_connector.py index e489e0143..d826794f4 100644 --- a/thingsboard_gateway/connectors/opcua_asyncio/opcua_connector.py +++ b/thingsboard_gateway/connectors/opcua_asyncio/opcua_connector.py @@ -20,10 +20,11 @@ from time import sleep, time from queue import Queue -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector from thingsboard_gateway.connectors.opcua_asyncio.device import Device from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader from thingsboard_gateway.tb_utility.tb_utility import TBUtility +from thingsboard_gateway.tb_utility.tb_logger import init_logger try: import asyncua @@ -50,14 +51,14 @@ class OpcUaConnectorAsyncIO(Connector, Thread): def __init__(self, gateway, config, connector_type): self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} - self.__log = log super().__init__() self._connector_type = connector_type self.__gateway = gateway + self.__config = config self.__server_conf = config['server'] - self.setName( - self.__server_conf.get("name", 'OPC-UA Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))) + self.__config.get("name", 'OPC-UA Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))) + self.__log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO')) if "opc.tcp" not in self.__server_conf.get("url"): self.__opcua_url = "opc.tcp://" + self.__server_conf.get("url") @@ -82,7 +83,10 @@ def __init__(self, gateway, config, connector_type): def open(self): self.__stopped = False self.start() - log.info("Starting OPC-UA Connector (Async IO)") + self.__log.info("Starting OPC-UA Connector (Async IO)") + + def get_type(self): + return self._connector_type def close(self): task = self.__loop.create_task(self.__reset_nodes()) @@ -93,6 +97,7 @@ def close(self): self.__stopped = True self.__connected = False self.__log.info('%s has been stopped.', self.get_name()) + self.__log.reset() async def __reset_node(self, node): node['valid'] = False @@ -156,8 +161,7 @@ async def start_client(self): try: await self.__client.load_data_type_definitions() except Exception as e: - self.__log.error("Error on loading type definitions:") - self.__log.error(e) + self.__log.error("Error on loading type definitions:\n %s", e) while not self.__stopped: if time() - self.__last_poll >= self.__server_conf.get('scanPeriodInMillis', 5000) / 1000: @@ -184,7 +188,7 @@ async def __set_auth_settings_by_cert(self): policy = self.__server_conf["security"] if cert is None or private_key is None: - log.exception("Error in ssl configuration - cert or privateKey parameter not found") + self.__log.exception("Error in ssl configuration - cert or privateKey parameter not found") raise RuntimeError("Error in ssl configuration - cert or privateKey parameter not found") await self.__client.set_security( @@ -206,10 +210,10 @@ def __load_converter(self, device): module = TBModuleLoader.import_module(self._connector_type, converter_class_name) if module: - log.debug('Converter %s for device %s - found!', converter_class_name, self.name) + self.__log.debug('Converter %s for device %s - found!', converter_class_name, self.name) return module - log.error("Cannot find converter for %s device", self.name) + self.__log.error("Cannot find converter for %s device", self.name) return None @staticmethod @@ -290,10 +294,11 @@ async def __scan_device_nodes(self): converter = self.__load_converter(device) device_config = {**device, 'device_name': device_name} self.__device_nodes.append( - Device(path=node, name=device_name, config=device_config, converter=converter(device_config), - converter_for_sub=converter(device_config) if not self.__server_conf.get( + Device(path=node, name=device_name, config=device_config, + converter=converter(device_config, self.__log), + converter_for_sub=converter(device_config, self.__log) if not self.__server_conf.get( 'disableSubscriptions', - False) else None)) + False) else None, logger=self.__log)) self.__log.info('Added device node: %s', device_name) for device_name in existing_devices: @@ -334,11 +339,14 @@ async def __poll_nodes(self): qualified_path = await self.find_node_name_space_index(path) if len(qualified_path) == 0: if node.get('valid', True): - self.__log.warning('Node not found; device: %s, key: %s, path: %s', device.name, node['key'], node['path']) + self.__log.warning('Node not found; device: %s, key: %s, path: %s', device.name, + node['key'], node['path']) await self.__reset_node(node) continue elif len(qualified_path) > 1: - self.__log.warning('Multiple matching nodes found; device: %s, key: %s, path: %s; %s', device.name, node['key'], node['path'], qualified_path) + self.__log.warning( + 'Multiple matching nodes found; device: %s, key: %s, path: %s; %s', device.name, + node['key'], node['path'], qualified_path) node['qualified_path'] = qualified_path[0] path = qualified_path[0] @@ -352,12 +360,13 @@ async def __poll_nodes(self): False): if self.__subscription is None: self.__subscription = await self.__client.create_subscription(1, SubHandler( - self.__sub_data_to_convert)) + self.__sub_data_to_convert, self.__log)) handle = await self.__subscription.subscribe_data_change(var) node['subscription'] = handle node['sub_on'] = True node['id'] = var.nodeid.to_string() - self.__log.info("Subscribed on data change; device: %s, key: %s, path: %s", device.name, node['key'], node['id']) + self.__log.info("Subscribed on data change; device: %s, key: %s, path: %s", device.name, + node['key'], node['id']) node['valid'] = True except ConnectionError: @@ -365,7 +374,8 @@ async def __poll_nodes(self): except (BadNodeIdUnknown, BadConnectionClosed, BadInvalidState, BadAttributeIdInvalid, BadCommunicationError, BadOutOfService): if node.get('valid', True): - self.__log.warning('Node not found (2); device: %s, key: %s, path: %s', device.name, node['key'], node['path']) + self.__log.warning('Node not found (2); device: %s, key: %s, path: %s', device.name, + node['key'], node['path']) await self.__reset_node(node) except UaStatusCodeError as uae: if node.get('valid', True): @@ -430,12 +440,25 @@ def on_attributes_update(self, content): def server_side_rpc_handler(self, content): try: + if content.get('data') is None: + content['data'] = {'params': content['params'], 'method': content['method']} + rpc_method = content["data"].get("method") + # check if RPC type is connector RPC (can be only 'get' or 'set') + try: + (connector_type, rpc_method_name) = rpc_method.split('_') + if connector_type == self._connector_type: + rpc_method = rpc_method_name + content['device'] = content['params'].split(' ')[0].split('=')[-1] + except (ValueError, IndexError): + pass + # firstly check if a method is not service if rpc_method == 'set' or rpc_method == 'get': full_path = '' args_list = [] + device = content.get('device') try: args_list = content['data']['params'].split(';') @@ -445,11 +468,12 @@ def server_side_rpc_handler(self, content): else: full_path = args_list[0].split('=')[-1] except IndexError: - log.error('Not enough arguments. Expected min 2.') - self.__gateway.send_rpc_reply(content['device'], - content['data']['id'], - {content['data']['method']: 'Not enough arguments. Expected min 2.', - 'code': 400}) + self.__log.error('Not enough arguments. Expected min 2.') + self.__gateway.send_rpc_reply(device=device, + req_id=content['data'].get('id'), + content={content['data'][ + 'method']: 'Not enough arguments. Expected min 2.', + 'code': 400}) result = {} if rpc_method == 'get': @@ -464,9 +488,9 @@ def server_side_rpc_handler(self, content): while not task.done(): sleep(.2) - self.__gateway.send_rpc_reply(content['device'], - content['data']['id'], - {content['data']['method']: result}) + self.__gateway.send_rpc_reply(device=device, + req_id=content['data'].get('id'), + content={content['data']['method']: result}) else: device = tuple(filter(lambda i: i.name == content['device'], self.__device_nodes))[0] @@ -486,13 +510,13 @@ def server_side_rpc_handler(self, content): self.__gateway.send_rpc_reply(content["device"], content["data"]["id"], {content["data"]["method"]: result, "code": 200}) - log.debug("method %s result is: %s", rpc['method'], result) + self.__log.debug("method %s result is: %s", rpc['method'], result) except Exception as e: - log.exception(e) + self.__log.exception(e) self.__gateway.send_rpc_reply(content["device"], content["data"]["id"], {"error": str(e), "code": 500}) else: - log.error("Method %s not found for device %s", rpc_method, content["device"]) + self.__log.error("Method %s not found for device %s", rpc_method, content["device"]) self.__gateway.send_rpc_reply(content["device"], content["data"]["id"], {"error": "%s - Method not found" % rpc_method, "code": 404}) @@ -525,14 +549,14 @@ async def __call_method(self, path, arguments, result={}): result['error'] = e.__str__() def update_converter_config(self, converter_name, config): - log.debug('Received remote converter configuration update for %s with configuration %s', converter_name, - config) + self.__log.debug('Received remote converter configuration update for %s with configuration %s', converter_name, + config) for device in self.__device_nodes: if device.converter.__class__.__name__ == converter_name: device.config.update(config) device.load_values() - log.info('Updated converter configuration for: %s with configuration %s', - converter_name, device.config) + self.__log.info('Updated converter configuration for: %s with configuration %s', + converter_name, device.config) for node_config in self.__server_conf['mapping']: if node_config['deviceNodePattern'] == device.config['deviceNodePattern']: @@ -540,24 +564,12 @@ def update_converter_config(self, converter_name, config): self.__gateway.update_connector_config_file(self.name, {'server': self.__server_conf}) + class SubHandler: - def __init__(self, queue): + def __init__(self, queue, logger): + self.__log = logger self.__queue = queue def datachange_notification(self, node, _, data): - log.debug("New data change event %s %s", node, data) + self.__log.debug("New data change event %s %s", node, data) self.__queue.put((node, data)) - - @staticmethod - def event_notification(event): - try: - log.info("New event %s", event) - except Exception as e: - log.exception(e) - - @staticmethod - def status_change_notification(status): - try: - log.info("Status change %s", status) - except Exception as e: - log.exception(e) diff --git a/thingsboard_gateway/connectors/opcua_asyncio/opcua_converter.py b/thingsboard_gateway/connectors/opcua_asyncio/opcua_converter.py index 9d8cf6aa6..656a828b5 100644 --- a/thingsboard_gateway/connectors/opcua_asyncio/opcua_converter.py +++ b/thingsboard_gateway/connectors/opcua_asyncio/opcua_converter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import Converter, abstractmethod, log +from thingsboard_gateway.connectors.converter import Converter, abstractmethod class OpcUaConverter(Converter): diff --git a/thingsboard_gateway/connectors/opcua_asyncio/opcua_uplink_converter.py b/thingsboard_gateway/connectors/opcua_asyncio/opcua_uplink_converter.py index 99a3f8ad2..9e877d20b 100644 --- a/thingsboard_gateway/connectors/opcua_asyncio/opcua_uplink_converter.py +++ b/thingsboard_gateway/connectors/opcua_asyncio/opcua_uplink_converter.py @@ -15,7 +15,7 @@ from time import time from datetime import timezone -from thingsboard_gateway.connectors.opcua_asyncio.opcua_converter import OpcUaConverter, log +from thingsboard_gateway.connectors.opcua_asyncio.opcua_converter import OpcUaConverter from asyncua.ua.uatypes import LocalizedText, VariantType DATA_TYPES = { @@ -25,7 +25,8 @@ class OpcUaUplinkConverter(OpcUaConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config self.data = { 'deviceName': self.__config['device_name'], @@ -85,4 +86,4 @@ def convert(self, config, val): else: self.data[DATA_TYPES[config['section']]].append({config['key']: data}) - log.debug('Converted data: %s', self.data) + self._log.debug('Converted data: %s', self.data) diff --git a/thingsboard_gateway/connectors/request/json_request_downlink_converter.py b/thingsboard_gateway/connectors/request/json_request_downlink_converter.py index 755c05b38..31e30593f 100644 --- a/thingsboard_gateway/connectors/request/json_request_downlink_converter.py +++ b/thingsboard_gateway/connectors/request/json_request_downlink_converter.py @@ -14,13 +14,14 @@ from urllib.parse import quote -from thingsboard_gateway.connectors.request.request_converter import RequestConverter, log +from thingsboard_gateway.connectors.request.request_converter import RequestConverter from thingsboard_gateway.tb_utility.tb_utility import TBUtility from thingsboard_gateway.gateway.statistics_service import StatisticsService class JsonRequestDownlinkConverter(RequestConverter): - def __init__(self, config): + def __init__(self, config, logger): + self.__log = logger self.__config = config @StatisticsService.CollectStatistics(start_stat_type='allReceivedBytesFromTB', @@ -64,4 +65,4 @@ def convert(self, config, data): return result except Exception as e: - log.exception(e) + self.__log.exception(e) diff --git a/thingsboard_gateway/connectors/request/json_request_uplink_converter.py b/thingsboard_gateway/connectors/request/json_request_uplink_converter.py index d7e0c843b..75b0710ef 100644 --- a/thingsboard_gateway/connectors/request/json_request_uplink_converter.py +++ b/thingsboard_gateway/connectors/request/json_request_uplink_converter.py @@ -16,13 +16,14 @@ from simplejson import dumps, loads -from thingsboard_gateway.connectors.request.request_converter import RequestConverter, log +from thingsboard_gateway.connectors.request.request_converter import RequestConverter from thingsboard_gateway.tb_utility.tb_utility import TBUtility from thingsboard_gateway.gateway.statistics_service import StatisticsService class JsonRequestUplinkConverter(RequestConverter): - def __init__(self, config): + def __init__(self, config, logger): + self.__log = logger self.__config = config self.__datatypes = {"attributes": "attributes", "telemetry": "telemetry"} @@ -49,8 +50,8 @@ def convert(self, config, data): str(device_name_value)) \ if is_valid_key else device_name_tag else: - log.error("The expression for looking \"deviceName\" not found in config %s", - dumps(self.__config['converter'])) + self.__log.error("The expression for looking \"deviceName\" not found in config %s", + dumps(self.__config['converter'])) if self.__config['converter'].get("deviceTypeJsonExpression") is not None: device_type_tags = TBUtility.get_values(self.__config['converter'].get("deviceTypeJsonExpression"), data, @@ -66,10 +67,10 @@ def convert(self, config, data): str(device_type_value)) \ if is_valid_key else device_type_tag else: - log.error("The expression for looking \"deviceType\" not found in config %s", - dumps(self.__config['converter'])) + self.__log.error("The expression for looking \"deviceType\" not found in config %s", + dumps(self.__config['converter'])) except Exception as e: - log.exception(e) + self.__log.exception(e) try: for datatype in self.__datatypes: @@ -109,6 +110,6 @@ def convert(self, config, data): dict_result[self.__datatypes[datatype]].append({full_key: full_value}) except Exception as e: - log.exception(e) + self.__log.exception(e) return dict_result diff --git a/thingsboard_gateway/connectors/request/request_connector.py b/thingsboard_gateway/connectors/request/request_connector.py index 42921aad3..0e30d155b 100644 --- a/thingsboard_gateway/connectors/request/request_connector.py +++ b/thingsboard_gateway/connectors/request/request_connector.py @@ -20,10 +20,9 @@ from threading import Thread from time import sleep, time -import simplejson - from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader from thingsboard_gateway.tb_utility.tb_utility import TBUtility +from thingsboard_gateway.tb_utility.tb_logger import init_logger try: from requests import Timeout, request @@ -35,7 +34,7 @@ from requests.auth import HTTPBasicAuth from requests.exceptions import RequestException -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector from thingsboard_gateway.connectors.request.json_request_uplink_converter import JsonRequestUplinkConverter from thingsboard_gateway.connectors.request.json_request_downlink_converter import JsonRequestDownlinkConverter @@ -52,6 +51,8 @@ def __init__(self, gateway, config, connector_type): self.__config = config self._connector_type = connector_type self.__gateway = gateway + self.setName(self.__config.get("name", "".join(choice(ascii_lowercase) for _ in range(5)))) + self._log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO')) self.__security = HTTPBasicAuth(self.__config["security"]["username"], self.__config["security"]["password"]) if \ self.__config["security"]["type"] == "basic" else None self.__host = None @@ -61,7 +62,6 @@ def __init__(self, gateway, config, connector_type): else: self.__host = "http://" + self.__config["host"] self.__ssl_verify = self.__config.get("SSLVerify", False) - self.setName(self.__config.get("name", "".join(choice(ascii_lowercase) for _ in range(5)))) self.daemon = True self.__connected = False self.__stopped = False @@ -77,7 +77,7 @@ def run(self): if self.__requests_in_progress: for request in self.__requests_in_progress: if time() >= request["next_time"]: - thread = Thread(target=self.__send_request, args=(request, self.__convert_queue, log), + thread = Thread(target=self.__send_request, args=(request, self.__convert_queue, self._log), daemon=True, name="Request to endpoint \'%s\' Thread" % (request["config"].get("url"))) thread.start() @@ -97,17 +97,17 @@ def on_attributes_update(self, content): "request": request, "withResponse": True} attribute_update_request_thread = Thread(target=self.__send_request, - args=(request_dict, response_queue, log), + args=(request_dict, response_queue, self._log), daemon=True, name="Attribute request to %s" % (converted_data["url"])) attribute_update_request_thread.start() attribute_update_request_thread.join() if not response_queue.empty(): response = response_queue.get_nowait() - log.debug(response) + self._log.debug(response) del response_queue except Exception as e: - log.exception(e) + self._log.exception(e) def server_side_rpc_handler(self, content): try: @@ -121,7 +121,7 @@ def server_side_rpc_handler(self, content): "request": request, "withResponse": True} rpc_request_thread = Thread(target=self.__send_request, - args=(request_dict, response_queue, log), + args=(request_dict, response_queue, self._log), daemon=True, name="RPC request to %s" % (converted_data["url"])) rpc_request_thread.start() @@ -157,46 +157,48 @@ def server_side_rpc_handler(self, content): del response_queue except Exception as e: - log.exception(e) + self._log.exception(e) def __fill_requests(self): - log.debug(self.__config["mapping"]) + self._log.debug(self.__config["mapping"]) for endpoint in self.__config["mapping"]: try: - log.debug(endpoint) + self._log.debug(endpoint) converter = None if endpoint["converter"]["type"] == "custom": module = TBModuleLoader.import_module(self._connector_type, endpoint["converter"]["extension"]) if module is not None: - log.debug('Custom converter for url %s - found!', endpoint["url"]) - converter = module(endpoint) + self._log.debug('Custom converter for url %s - found!', endpoint["url"]) + converter = module(endpoint, self._log) else: - log.error("\n\nCannot find extension module for %s url.\nPlease check your configuration.\n", - endpoint["url"]) + self._log.error( + "\n\nCannot find extension module for %s url.\nPlease check your configuration.\n", + endpoint["url"]) else: - converter = JsonRequestUplinkConverter(endpoint) + converter = JsonRequestUplinkConverter(endpoint, self._log) self.__requests_in_progress.append({"config": endpoint, "converter": converter, "next_time": time(), "request": request}) except Exception as e: - log.exception(e) + self._log.exception(e) def __fill_attribute_updates(self): for attribute_request in self.__config.get("attributeUpdates", []): if attribute_request.get("converter") is not None: - converter = TBModuleLoader.import_module("request", attribute_request["converter"])(attribute_request) + converter = TBModuleLoader.import_module("request", attribute_request["converter"])(attribute_request, + self._log) else: - converter = JsonRequestDownlinkConverter(attribute_request) + converter = JsonRequestDownlinkConverter(attribute_request, self._log) attribute_request_dict = {**attribute_request, "converter": converter} self.__attribute_updates.append(attribute_request_dict) def __fill_rpc_requests(self): for rpc_request in self.__config.get("serverSideRpc", []): if rpc_request.get("converter") is not None: - converter = TBModuleLoader.import_module("request", rpc_request["converter"])(rpc_request) + converter = TBModuleLoader.import_module("request", rpc_request["converter"])(rpc_request, self._log) else: - converter = JsonRequestDownlinkConverter(rpc_request) + converter = JsonRequestDownlinkConverter(rpc_request, self._log) rpc_request_dict = {**rpc_request, "converter": converter} self.__rpc_requests.append(rpc_request_dict) @@ -281,7 +283,7 @@ def __convert_data(self, data): self.__convert_queue.put(data_to_send) except Exception as e: - log.exception(e) + self._log.exception(e) def __add_ts(self, data): if data.get("ts") is None: @@ -295,11 +297,14 @@ def __process_data(self): self.statistics["MessagesSent"] = self.statistics["MessagesSent"] + 1 except Exception as e: - log.exception(e) + self._log.exception(e) def get_name(self): return self.name + def get_type(self): + return self._connector_type + def is_connected(self): return self.__connected @@ -309,6 +314,7 @@ def open(self): def close(self): self.__stopped = True + self._log.reset() def get_config(self): return self.__config diff --git a/thingsboard_gateway/connectors/request/request_converter.py b/thingsboard_gateway/connectors/request/request_converter.py index 681a79b32..e9f68a634 100644 --- a/thingsboard_gateway/connectors/request/request_converter.py +++ b/thingsboard_gateway/connectors/request/request_converter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import Converter, abstractmethod, log +from thingsboard_gateway.connectors.converter import Converter, abstractmethod class RequestConverter(Converter): diff --git a/thingsboard_gateway/connectors/rest/json_rest_downlink_converter.py b/thingsboard_gateway/connectors/rest/json_rest_downlink_converter.py index e7cff62ec..8dc44d4c0 100644 --- a/thingsboard_gateway/connectors/rest/json_rest_downlink_converter.py +++ b/thingsboard_gateway/connectors/rest/json_rest_downlink_converter.py @@ -14,13 +14,14 @@ from urllib.parse import quote -from thingsboard_gateway.connectors.rest.rest_converter import RESTConverter, log +from thingsboard_gateway.connectors.rest.rest_converter import RESTConverter from thingsboard_gateway.tb_utility.tb_utility import TBUtility from thingsboard_gateway.gateway.statistics_service import StatisticsService class JsonRESTDownlinkConverter(RESTConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config @StatisticsService.CollectStatistics(start_stat_type='allReceivedBytesFromTB', @@ -65,4 +66,4 @@ def convert(self, config, data): return result except Exception as e: - log.exception(e) + self._log.exception(e) diff --git a/thingsboard_gateway/connectors/rest/json_rest_uplink_converter.py b/thingsboard_gateway/connectors/rest/json_rest_uplink_converter.py index 290c448bd..c609117c1 100644 --- a/thingsboard_gateway/connectors/rest/json_rest_uplink_converter.py +++ b/thingsboard_gateway/connectors/rest/json_rest_uplink_converter.py @@ -16,14 +16,15 @@ from simplejson import dumps -from thingsboard_gateway.connectors.rest.rest_converter import RESTConverter, log +from thingsboard_gateway.connectors.rest.rest_converter import RESTConverter from thingsboard_gateway.tb_utility.tb_utility import TBUtility from thingsboard_gateway.gateway.statistics_service import StatisticsService class JsonRESTUplinkConverter(RESTConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config @StatisticsService.CollectStatistics(start_stat_type='receivedBytesFromDevices', @@ -46,7 +47,8 @@ def convert(self, config, data): str(device_name_value)) \ if is_valid_key else device_name_tag else: - log.error("The expression for looking \"deviceName\" not found in config %s", dumps(self.__config)) + self._log.error("The expression for looking \"deviceName\" not found in config %s", + dumps(self.__config)) if self.__config.get("deviceTypeExpression") is not None: device_type_tags = TBUtility.get_values(self.__config.get("deviceTypeExpression"), data, get_tag=True) @@ -60,10 +62,11 @@ def convert(self, config, data): str(device_type_value)) \ if is_valid_key else device_type_tag else: - log.error("The expression for looking \"deviceType\" not found in config %s", dumps(self.__config)) + self._log.error("The expression for looking \"deviceType\" not found in config %s", + dumps(self.__config)) except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), data) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config), data, + e) try: for datatype in datatypes: @@ -102,7 +105,7 @@ def convert(self, config, data): else: dict_result[datatypes[datatype]].append({full_key: full_value}) except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), str(data)) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config), + str(data), e) return dict_result diff --git a/thingsboard_gateway/connectors/rest/rest_connector.py b/thingsboard_gateway/connectors/rest/rest_connector.py index d6f79ecdc..b6d88e20c 100644 --- a/thingsboard_gateway/connectors/rest/rest_connector.py +++ b/thingsboard_gateway/connectors/rest/rest_connector.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import json from queue import Queue from random import choice @@ -28,7 +29,8 @@ from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader from thingsboard_gateway.tb_utility.tb_utility import TBUtility -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector +from thingsboard_gateway.tb_utility.tb_logger import init_logger try: from requests import Timeout, request as regular_request @@ -50,7 +52,6 @@ class RESTConnector(Connector, Thread): def __init__(self, gateway, config, connector_type): super().__init__() - self.__log = log self._default_converters = { "uplink": "JsonRESTUplinkConverter", "downlink": "JsonRESTDownlinkConverter" @@ -63,13 +64,14 @@ def __init__(self, gateway, config, connector_type): self._connector_type = connector_type self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} + self.setName(config.get("name", 'REST Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))) self.__gateway = gateway + self.__log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO')) self._default_downlink_converter = TBModuleLoader.import_module(self._connector_type, self._default_converters['downlink']) self._default_uplink_converter = TBModuleLoader.import_module(self._connector_type, self._default_converters['uplink']) self.__USER_DATA = {} - self.setName(config.get("name", 'REST Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))) self._app = None self._connected = False self.__stopped = False @@ -120,11 +122,11 @@ def load_handlers(self): mapping['security']['password']) for http_method in mapping['HTTPMethods']: handler = data_handlers[security_type](self.collect_statistic_and_send, self.get_name(), - self.endpoints[mapping["endpoint"]], + self.endpoints[mapping["endpoint"]], self.__log, provider=self.__event_provider) handlers.append(web.route(http_method, mapping['endpoint'], handler)) except Exception as e: - log.error("Error on creating handlers - %s", str(e)) + self.__log.error("Error on creating handlers - %s", str(e)) self._app.add_routes(handlers) def open(self): @@ -153,8 +155,7 @@ def __run_server(self): cert = self.__config['security']['cert'] key = self.__config['security']['key'] except KeyError as e: - log.exception(e) - log.error('Provide certificate and key path!') + self.__log.error('Provide certificate and key path!\n %s', e) ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) ssl_context.load_cert_chain(cert, key) @@ -169,15 +170,19 @@ def run(self): try: self.__run_server() except Exception as e: - log.exception(e) + self.__log.exception(e) def close(self): self.__stopped = True self._connected = False + self.__log.reset() def get_name(self): return self.name + def get_type(self): + return self._connector_type + def is_connected(self): return self._connected @@ -195,14 +200,14 @@ def on_attributes_update(self, content): **converted_data}, "request": regular_request} attribute_update_request_thread = Thread(target=self.__send_request, - args=(request_dict, response_queue, log), + args=(request_dict, response_queue, self.__log), daemon=True, name="Attribute request to %s" % ( converted_data["url"])) attribute_update_request_thread.start() if not response_queue.empty(): response = response_queue.get_nowait() - log.debug(response) + self.__log.debug(response) del response_queue # ONLY if initialized "response" section for endpoint @@ -212,14 +217,27 @@ def on_attributes_update(self, content): if list(content['data'].keys())[0] == response_attribute: BaseDataHandler.responses_queue.put(content['data'][response_attribute]) except Exception as e: - log.exception(e) + self.__log.exception(e) def server_side_rpc_handler(self, content): try: + if content.get('data') is None: + content['data'] = {'params': content['params'], 'method': content['method']} + rpc_method = content['data']['method'] + # check if RPC type is connector RPC (can be only 'get' or 'set') + try: + (connector_type, rpc_method_name) = rpc_method.split('_') + if connector_type == self._connector_type: + rpc_method = rpc_method_name + content['device'] = content['params'].split(' ')[0].split('=')[-1] + except (IndexError, ValueError): + pass + # check if RPC method is reserved get/set if rpc_method == 'get' or rpc_method == 'set': + device = content.get('device') params = {} for param in content['data']['params'].split(';'): try: @@ -236,11 +254,11 @@ def server_side_rpc_handler(self, content): request_dict = {'config': {**params, **converted_data}, 'request': regular_request, 'converter': uplink_converter} - response = self.__send_request(request_dict, Queue(1), log, with_queue=False) + response = self.__send_request(request_dict, Queue(1), self.__log, with_queue=False) - log.debug('Response from RPC request: %s', response) - self.__gateway.send_rpc_reply(device=content["device"], - req_id=content["data"]["id"], + self.__log.debug('Response from RPC request: %s', response) + self.__gateway.send_rpc_reply(device=device, + req_id=content["data"].get('id'), content=response[2] if response and len( response) >= 3 else response) else: @@ -254,15 +272,15 @@ def server_side_rpc_handler(self, content): "request": regular_request} request_dict["converter"] = request_dict["config"].get("uplink_converter") - response = self.__send_request(request_dict, Queue(1), log, with_queue=False) + response = self.__send_request(request_dict, Queue(1), self.__log, with_queue=False) - log.debug('Response from RPC request: %s', response) + self.__log.debug('Response from RPC request: %s', response) self.__gateway.send_rpc_reply(device=content["device"], req_id=content["data"]["id"], content=response[2] if response and len( response) >= 3 else response) except Exception as e: - log.exception(e) + self.__log.exception(e) def __event_provider(self, event_type): try: @@ -292,12 +310,12 @@ def __fill_requests_from_TB(self): request_config_object.get("extension", self._default_converters[ "uplink"]))( - request_config_object) + request_config_object, self.__log) downlink_converter = TBModuleLoader.import_module(self._connector_type, request_config_object.get("extension", self._default_converters[ "downlink"]))( - request_config_object) + request_config_object, self.__log) request_dict = {**request_config_object, "uplink_converter": uplink_converter, "downlink_converter": downlink_converter, @@ -384,7 +402,8 @@ class BaseDataHandler: responses_queue = Queue() response_attribute_request = Queue() - def __init__(self, send_to_storage, name, endpoint, provider=None): + def __init__(self, send_to_storage, name, endpoint, logger, provider=None): + self.log = logger self.send_to_storage = send_to_storage self.__name = name self.__endpoint = endpoint @@ -511,18 +530,18 @@ async def __call__(self, request: web.Request): return result try: - log.info("CONVERTER CONFIG: %r", endpoint_config['converter']) - converter = self.endpoint['converter'](endpoint_config['converter']) + self.log.info("CONVERTER CONFIG: %r", endpoint_config['converter']) + converter = self.endpoint['converter'](endpoint_config['converter'], self.log) converted_data = converter.convert(config=endpoint_config['converter'], data=data) self.modify_data_for_remote_response(converted_data, self.response_expected) self.send_to_storage(self.name, converted_data) - log.info("CONVERTED_DATA: %r", converted_data) + self.log.info("CONVERTED_DATA: %r", converted_data) return self.get_response() except Exception as e: - log.exception("Error while post to anonymous handler: %s", e) + self.log.exception("Error while post to anonymous handler: %s", e) return web.Response(body=str(self.success_response) if self.success_response else None, status=500) @@ -560,18 +579,18 @@ async def __call__(self, request: web.Request): return result try: - log.info("CONVERTER CONFIG: %r", endpoint_config['converter']) - converter = self.endpoint['converter'](endpoint_config['converter']) + self.log.info("CONVERTER CONFIG: %r", endpoint_config['converter']) + converter = self.endpoint['converter'](endpoint_config['converter'], self.log) converted_data = converter.convert(config=endpoint_config['converter'], data=data) self.modify_data_for_remote_response(converted_data, self.response_expected) self.send_to_storage(self.name, converted_data) - log.info("CONVERTED_DATA: %r", converted_data) + self.log.info("CONVERTED_DATA: %r", converted_data) return self.get_response() except Exception as e: - log.exception("Error while post to basic handler: %s", e) + self.log.exception("Error while post to basic handler: %s", e) return web.Response(body=str(self.unsuccessful_response) if self.unsuccessful_response else None, status=500) diff --git a/thingsboard_gateway/connectors/rest/rest_converter.py b/thingsboard_gateway/connectors/rest/rest_converter.py index b88f04ed9..2060d0beb 100644 --- a/thingsboard_gateway/connectors/rest/rest_converter.py +++ b/thingsboard_gateway/connectors/rest/rest_converter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import Converter, log +from thingsboard_gateway.connectors.converter import Converter class RESTConverter(Converter): diff --git a/thingsboard_gateway/connectors/snmp/snmp_connector.py b/thingsboard_gateway/connectors/snmp/snmp_connector.py index 09fa46b15..96ed38bef 100644 --- a/thingsboard_gateway/connectors/snmp/snmp_connector.py +++ b/thingsboard_gateway/connectors/snmp/snmp_connector.py @@ -20,9 +20,10 @@ from threading import Thread from time import sleep, time -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader from thingsboard_gateway.tb_utility.tb_utility import TBUtility +from thingsboard_gateway.tb_utility.tb_logger import init_logger # Try import Pymodbus library or install it and import installation_required = False @@ -52,8 +53,9 @@ def __init__(self, gateway, config, connector_type): self.__stopped = False self._connector_type = connector_type self.__config = config - self.__devices = self.__config["devices"] self.setName(config.get("name", 'SNMP Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))) + self._log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO')) + self.__devices = self.__config["devices"] self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} self._default_converters = { @@ -76,7 +78,7 @@ def run(self): try: self.__loop.run_until_complete(self._run()) except Exception as e: - log.exception(e) + self._log.exception(e) async def _run(self): while not self.__stopped: @@ -87,7 +89,7 @@ async def _run(self): await self.__process_data(device) device["previous_poll_time"] = current_time except Exception as e: - log.exception(e) + self._log.exception(e) if self.__stopped: break else: @@ -100,6 +102,9 @@ def close(self): def get_name(self): return self.name + def get_type(self): + return self._connector_type + def is_connected(self): return self._connected @@ -120,26 +125,25 @@ async def __process_data(self, device): response = None method = datatype_config.get("method") if method is None: - log.error("Method not found in configuration: %r", datatype_config) + self._log.error("Method not found in configuration: %r", datatype_config) continue else: method = method.lower() if method not in self.__methods: - log.error("Unknown method: %s, configuration is: %r", method, datatype_config) + self._log.error("Unknown method: %s, configuration is: %r", method, datatype_config) response = await self.__process_methods(method, common_parameters, datatype_config) converted_data.update(**device["uplink_converter"].convert((datatype, datatype_config), response)) except SNMPTimeoutException: - log.error("Timeout exception on connection to device \"%s\" with ip: \"%s\"", device["deviceName"], + self._log.error("Timeout exception on connection to device \"%s\" with ip: \"%s\"", device["deviceName"], device["ip"]) return except Exception as e: - log.exception(e) + self._log.exception(e) if isinstance(converted_data, dict) and (converted_data.get("attributes") or converted_data.get("telemetry")): self.collect_statistic_and_send(self.get_name(), converted_data) - @staticmethod - async def __process_methods(method, common_parameters, datatype_config): + async def __process_methods(self, method, common_parameters, datatype_config): client = Client(ip=common_parameters['ip'], port=common_parameters['port'], credentials=credentials.V1(common_parameters['community'])) @@ -203,7 +207,7 @@ async def __process_methods(method, common_parameters, datatype_config): bulk_size = datatype_config.get("bulkSize", 10) response = await client.bulktable(oid=oid, bulk_size=bulk_size) else: - log.error("Method \"%s\" - Not found", str(method)) + self._log.error("Method \"%s\" - Not found", str(method)) return response def __fill_converters(self): @@ -211,12 +215,13 @@ def __fill_converters(self): for device in self.__devices: device["uplink_converter"] = TBModuleLoader.import_module("snmp", device.get('converter', self._default_converters[ - "uplink"]))(device) + "uplink"]))(device, + self._log) device["downlink_converter"] = TBModuleLoader.import_module("snmp", device.get('converter', self._default_converters[ "downlink"]))(device) except Exception as e: - log.exception(e) + self._log.exception(e) @staticmethod def __get_common_parameters(device): @@ -236,15 +241,15 @@ def on_attributes_update(self, content): common_parameters = self.__get_common_parameters(device) result = self.__process_methods(attribute_request_config["method"], common_parameters, {**attribute_request_config, "value": value}) - log.debug( + self._log.debug( "Received attribute update request for device \"%s\" " "with attribute \"%s\" and value \"%s\"", content["device"], attribute) - log.debug(result) - log.debug(content) + self._log.debug(result) + self._log.debug(content) except Exception as e: - log.exception(e) + self._log.exception(e) def server_side_rpc_handler(self, content): try: @@ -255,13 +260,13 @@ def server_side_rpc_handler(self, content): common_parameters = self.__get_common_parameters(device) result = self.__process_methods(rpc_request_config["method"], common_parameters, {**rpc_request_config, "value": content["data"]["params"]}) - log.debug("Received RPC request for device \"%s\" with command \"%s\" and value \"%s\"", + self._log.debug("Received RPC request for device \"%s\" with command \"%s\" and value \"%s\"", content["device"], content["data"]["method"]) - log.debug(result) - log.debug(content) + self._log.debug(result) + self._log.debug(content) self.__gateway.send_rpc_reply(device=content["device"], req_id=content["data"]["id"], content=result) except Exception as e: - log.exception(e) + self._log.exception(e) self.__gateway.send_rpc_reply(device=content["device"], req_id=content["data"]["id"], success_sent=False) diff --git a/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py b/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py index cb8d8bad3..77c502038 100644 --- a/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py +++ b/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py @@ -12,12 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import Converter, log +from thingsboard_gateway.connectors.converter import Converter from thingsboard_gateway.gateway.statistics_service import StatisticsService class SNMPUplinkConverter(Converter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config @StatisticsService.CollectStatistics(start_stat_type='receivedBytesFromDevices', @@ -46,7 +47,7 @@ def convert(self, config, data): result[config[0]].append({config[1]["key"]: data.decode("UTF-8")}) else: result[config[0]].append({config[1]["key"]: data}) - log.debug(result) + self._log.debug(result) except Exception as e: - log.exception(e) + self._log.exception(e) return result diff --git a/thingsboard_gateway/connectors/socket/bytes_socket_uplink_converter.py b/thingsboard_gateway/connectors/socket/bytes_socket_uplink_converter.py index 9d0cb9807..8ce6921d6 100644 --- a/thingsboard_gateway/connectors/socket/bytes_socket_uplink_converter.py +++ b/thingsboard_gateway/connectors/socket/bytes_socket_uplink_converter.py @@ -12,12 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.socket.socket_uplink_converter import SocketUplinkConverter, log +from thingsboard_gateway.connectors.socket.socket_uplink_converter import SocketUplinkConverter from thingsboard_gateway.gateway.statistics_service import StatisticsService class BytesSocketUplinkConverter(SocketUplinkConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config dict_result = { "deviceName": config['deviceName'], @@ -57,11 +58,11 @@ def convert(self, config, data): dict_result[section].append( {item['key']: converted_data}) else: - log.error('Key for %s not found in config: %s', config['type'], config['section_config']) + self._log.error('Key for %s not found in config: %s', config['type'], config['section_config']) except Exception as e: - log.exception(e) + self._log.exception(e) except Exception as e: - log.exception(e) + self._log.exception(e) - log.debug(dict_result) + self._log.debug(dict_result) return dict_result diff --git a/thingsboard_gateway/connectors/socket/socket_connector.py b/thingsboard_gateway/connectors/socket/socket_connector.py index ec6499c2f..5da9a6d8a 100644 --- a/thingsboard_gateway/connectors/socket/socket_connector.py +++ b/thingsboard_gateway/connectors/socket/socket_connector.py @@ -22,10 +22,11 @@ from simplejson import dumps -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader from thingsboard_gateway.gateway.statistics_service import StatisticsService from thingsboard_gateway.connectors.socket.socket_decorators import CustomCollectStatistics +from thingsboard_gateway.tb_utility.tb_logger import init_logger SOCKET_TYPE = { 'TCP': socket.SOCK_STREAM, @@ -37,13 +38,13 @@ class SocketConnector(Connector, Thread): def __init__(self, gateway, config, connector_type): super().__init__() - self.__log = log self.__config = config self._connector_type = connector_type self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} self.__gateway = gateway self.setName(config.get("name", 'TCP Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))) + self.__log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO')) self.daemon = True self.__stopped = False self._connected = False @@ -67,7 +68,7 @@ def __convert_devices_list(self): module = self.__load_converter(device) converter = module( {'deviceName': device['deviceName'], - 'deviceType': device.get('deviceType', 'default')}) if module else None + 'deviceType': device.get('deviceType', 'default')}, self.__log) if module else None device['converter'] = converter # validate attributeRequests requestExpression @@ -88,10 +89,10 @@ def __load_converter(self, device): module = TBModuleLoader.import_module(self._connector_type, converter_class_name) if module: - log.debug('Converter %s for device %s - found!', converter_class_name, self.name) + self.__log.debug('Converter %s for device %s - found!', converter_class_name, self.name) return module - log.error("Cannot find converter for %s device", self.name) + self.__log.error("Cannot find converter for %s device", self.name) return None def __validate_attr_requests(self, attr_requests): @@ -144,7 +145,7 @@ def run(self): try: self.__socket.bind((self.__socket_address, self.__socket_port)) except OSError: - log.error('Address already in use. Reconnecting...') + self.__log.error('Address already in use. Reconnecting...') sleep(3) else: self.__bind = True @@ -237,7 +238,7 @@ def __convert_data(self, device, data): if converted_data is not None: self.__gateway.send_to_storage(self.get_name(), converted_data) self.statistics['MessagesSent'] = self.statistics['MessagesSent'] + 1 - log.info('Data to ThingsBoard %s', converted_data) + self.__log.info('Data to ThingsBoard %s', converted_data) except Exception as e: self.__log.exception(e) @@ -286,10 +287,14 @@ def close(self): self.__stopped = True self._connected = False self.__connections = {} + self.__log.reset() def get_name(self): return self.name + def get_type(self): + return self._connector_type + def is_connected(self): return self._connected @@ -309,8 +314,7 @@ def __write_value_via_tcp(self, address, port, value): new_socket.close() return 'ok' except ConnectionRefusedError as e: - self.__log.error('Can\'t connect to %s:%s', address, port) - self.__log.exception(e) + self.__log.error('Can\'t connect to %s:%s\n %s', address, port, e) return e @staticmethod @@ -338,10 +342,24 @@ def on_attributes_update(self, content): @StatisticsService.CollectAllReceivedBytesStatistics(start_stat_type='allReceivedBytesFromTB') def server_side_rpc_handler(self, content): try: + if content.get('data') is None: + content['data'] = {'params': content['params'], 'method': content['method']} + + rpc_method = content['data']['method'] + + # check if RPC type is connector RPC (can be only 'set') + try: + (connector_type, rpc_method_name) = rpc_method.split('_') + if connector_type == self._connector_type: + rpc_method = rpc_method_name + content['device'] = content['params'].split(' ')[0].split('=')[-1] + except (IndexError, ValueError): + pass + device = tuple(filter(lambda item: item['deviceName'] == content['device'], self.__config['devices']))[0] # check if RPC method is reserved set - if content['data']['method'] == 'set': + if rpc_method == 'set': params = {} for param in content['data']['params'].split(';'): try: @@ -359,12 +377,13 @@ def server_side_rpc_handler(self, content): else: self.__write_value_via_udp(params['address'], int(params['port']), params['value']) except KeyError: - self.__gateway.send_rpc_reply(content['device'], content['data']['id'], 'Not enough params') + self.__gateway.send_rpc_reply(device=device, req_id=content['data'].get('id'), + content='Not enough params') except ValueError: - self.__gateway.send_rpc_reply(content['device'], content['data']['id'], - 'Param "port" have to be int type') + self.__gateway.send_rpc_reply(device=device, req_id=content['data']['id'], + content='Param "port" have to be int type') else: - self.__gateway.send_rpc_reply(content['device'], content['data']['id'], str(result)) + self.__gateway.send_rpc_reply(device=device, req_id=content['data'].get('id'), content=str(result)) else: for rpc_config in device['serverSideRpc']: for (key, value) in content['data'].items(): diff --git a/thingsboard_gateway/connectors/socket/socket_uplink_converter.py b/thingsboard_gateway/connectors/socket/socket_uplink_converter.py index 6c5e03ad7..d70b8a86f 100644 --- a/thingsboard_gateway/connectors/socket/socket_uplink_converter.py +++ b/thingsboard_gateway/connectors/socket/socket_uplink_converter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import Converter, abstractmethod, log +from thingsboard_gateway.connectors.converter import Converter, abstractmethod class SocketUplinkConverter(Converter): diff --git a/thingsboard_gateway/connectors/xmpp/xmpp_connector.py b/thingsboard_gateway/connectors/xmpp/xmpp_connector.py index 840234a89..b3824e69f 100644 --- a/thingsboard_gateway/connectors/xmpp/xmpp_connector.py +++ b/thingsboard_gateway/connectors/xmpp/xmpp_connector.py @@ -20,10 +20,11 @@ from threading import Thread from time import sleep -from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.connectors.connector import Connector from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader from thingsboard_gateway.tb_utility.tb_utility import TBUtility from thingsboard_gateway.gateway.statistics_service import StatisticsService +from thingsboard_gateway.tb_utility.tb_logger import init_logger try: from slixmpp import ClientXMPP @@ -44,7 +45,6 @@ class XMPPConnector(Connector, Thread): def __init__(self, gateway, config, connector_type): self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} - self.__log = log super().__init__() @@ -54,6 +54,8 @@ def __init__(self, gateway, config, connector_type): self.__config = config self._server_config = config['server'] self._devices_config = config.get('devices', []) + self.setName(config.get("name", 'XMPP Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))) + self.__log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO')) self._devices = {} self._reformat_devices_config() @@ -62,8 +64,6 @@ def __init__(self, gateway, config, connector_type): # {'deviceName': 'device_jid'} self._available_device = {} - self.setName(config.get("name", 'XMPP Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))) - self.__stopped = False self._connected = False self.daemon = True @@ -81,7 +81,7 @@ def _reformat_devices_config(self): continue self._devices[device_jid] = config - self._devices[device_jid]['converter'] = converter(config) + self._devices[device_jid]['converter'] = converter(config, self.__log) except KeyError as e: self.__log.error('Invalid configuration %s with key error %s', config, e) continue @@ -90,10 +90,10 @@ def _load_converter(self, converter_name): module = TBModuleLoader.import_module(self._connector_type, converter_name) if module: - log.debug('Converter %s for device %s - found!', converter_name, self.name) + self.__log.debug('Converter %s for device %s - found!', converter_name, self.name) return module - log.error("Cannot find converter for %s device", self.name) + self.__log.error("Cannot find converter for %s device", self.name) return None def open(self): @@ -192,10 +192,14 @@ def close(self): self.__stopped = True self._connected = False self.__log.info('%s has been stopped.', self.get_name()) + self.__log.reset() def get_name(self): return self.name + def get_type(self): + return self._connector_type + def is_connected(self): return self._connected diff --git a/thingsboard_gateway/connectors/xmpp/xmpp_converter.py b/thingsboard_gateway/connectors/xmpp/xmpp_converter.py index 83c60c57d..019a66dfc 100644 --- a/thingsboard_gateway/connectors/xmpp/xmpp_converter.py +++ b/thingsboard_gateway/connectors/xmpp/xmpp_converter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import Converter, abstractmethod, log +from thingsboard_gateway.connectors.converter import Converter, abstractmethod class XmppConverter(Converter): diff --git a/thingsboard_gateway/connectors/xmpp/xmpp_uplink_converter.py b/thingsboard_gateway/connectors/xmpp/xmpp_uplink_converter.py index 289d06907..335284e29 100644 --- a/thingsboard_gateway/connectors/xmpp/xmpp_uplink_converter.py +++ b/thingsboard_gateway/connectors/xmpp/xmpp_uplink_converter.py @@ -16,13 +16,14 @@ from re import findall from time import time -from thingsboard_gateway.connectors.xmpp.xmpp_converter import XmppConverter, log +from thingsboard_gateway.connectors.xmpp.xmpp_converter import XmppConverter from thingsboard_gateway.tb_utility.tb_utility import TBUtility from thingsboard_gateway.gateway.statistics_service import StatisticsService class XmppUplinkConverter(XmppConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config self._datatypes = {"attributes": "attributes", "timeseries": "telemetry"} @@ -98,8 +99,7 @@ def _convert_json(self, val): except json.decoder.JSONDecodeError: return None - @staticmethod - def _get_value(data, config, key): + def _get_value(self, data, config, key): expression_arr = findall(r'\[\S[\d:]*]', config[key]) if expression_arr: indexes = expression_arr[0][1:-1].split(':') @@ -111,7 +111,7 @@ def _get_value(data, config, key): index = int(indexes[0]) return data[index] except IndexError: - log.error('deviceName expression invalid (index out of range)') + self._log.error('deviceName expression invalid (index out of range)') return None else: return config[key] diff --git a/thingsboard_gateway/extensions/mqtt/custom_mqtt_uplink_converter.py b/thingsboard_gateway/extensions/mqtt/custom_mqtt_uplink_converter.py index 0ef9d5ad2..05f7dcb0b 100644 --- a/thingsboard_gateway/extensions/mqtt/custom_mqtt_uplink_converter.py +++ b/thingsboard_gateway/extensions/mqtt/custom_mqtt_uplink_converter.py @@ -14,11 +14,12 @@ from simplejson import dumps -from thingsboard_gateway.connectors.mqtt.mqtt_uplink_converter import MqttUplinkConverter, log +from thingsboard_gateway.connectors.mqtt.mqtt_uplink_converter import MqttUplinkConverter class CustomMqttUplinkConverter(MqttUplinkConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config.get('converter') @property @@ -46,5 +47,5 @@ def convert(self, topic, body): return dict_result except Exception as e: - log.exception('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), body) - log.exception(e) + self._log.exception('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), body) + self._log.exception(e) diff --git a/thingsboard_gateway/extensions/request/custom_request_uplink_converter.py b/thingsboard_gateway/extensions/request/custom_request_uplink_converter.py index e86dfc5c5..c2805c5b8 100644 --- a/thingsboard_gateway/extensions/request/custom_request_uplink_converter.py +++ b/thingsboard_gateway/extensions/request/custom_request_uplink_converter.py @@ -16,12 +16,13 @@ from simplejson import dumps -from thingsboard_gateway.connectors.request.request_converter import RequestConverter, log +from thingsboard_gateway.connectors.request.request_converter import RequestConverter from thingsboard_gateway.tb_utility.tb_utility import TBUtility class CustomRequestUplinkConverter(RequestConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config.get('converter') def convert(self, _, body): @@ -62,5 +63,5 @@ def convert(self, _, body): return dict_result except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), body) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), body) + self._log.exception(e) diff --git a/thingsboard_gateway/extensions/serial/custom_serial_connector.py b/thingsboard_gateway/extensions/serial/custom_serial_connector.py index 2dd081a12..ea0a21fa0 100644 --- a/thingsboard_gateway/extensions/serial/custom_serial_connector.py +++ b/thingsboard_gateway/extensions/serial/custom_serial_connector.py @@ -27,8 +27,9 @@ TBUtility.install_package("pyserial") import serial -from thingsboard_gateway.connectors.connector import Connector, log # Import base class for connector and logger +from thingsboard_gateway.connectors.connector import Connector # Import base class for connector and logger from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader +from thingsboard_gateway.tb_utility.tb_logger import init_logger class CustomSerialConnector(Thread, Connector): # Define a connector class, it should inherit from "Connector" class. @@ -36,20 +37,22 @@ def __init__(self, gateway, config, connector_type): super().__init__() # Initialize parents classes self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} # Dictionary, will save information about count received and sent messages. + self._connector_type = connector_type self.__config = config # Save configuration from the configuration file. self.__gateway = gateway # Save gateway object, we will use some gateway methods for adding devices and saving data from them. self.setName(self.__config.get("name", "Custom %s connector " % self.get_name() + ''.join( choice(ascii_lowercase) for _ in range(5)))) # get from the configuration or create name for logs. - log.info("Starting Custom %s connector", self.get_name()) # Send message to logger + self._log = init_logger(self.__gateway, self.name, level=self.__config.get('logLevel')) + self._log.info("Starting Custom %s connector", self.get_name()) # Send message to logger self.daemon = True # Set self thread as daemon self.stopped = True # Service variable for check state self.__connected = False # Service variable for check connection to device self.__devices = {} # Dictionary with devices, will contain devices configurations, converters for devices and serial port objects self.__load_converters(connector_type) # Call function to load converters and save it into devices dictionary self.__connect_to_devices() # Call function for connect to devices - log.info('Custom connector %s initialization success.', self.get_name()) # Message to logger - log.info("Devices in configuration file found: %s ", '\n'.join(device for device in self.__devices)) # Message to logger + self._log.info('Custom connector %s initialization success.', self.get_name()) # Message to logger + self._log.info("Devices in configuration file found: %s ", '\n'.join(device for device in self.__devices)) # Message to logger def __connect_to_devices(self): # Function for opening connection and connecting to devices for device in self.__devices: @@ -76,12 +79,12 @@ def __connect_to_devices(self): # Function for opening connection and connectin exclusive=device_config.get('exclusive', None)) time.sleep(.1) if time.time() - connection_start > 10: # Break connection try if it setting up for 10 seconds - log.error("Connection refused per timeout for device %s", device_config.get("name")) + self._log.error("Connection refused per timeout for device %s", device_config.get("name")) break except serial.serialutil.SerialException: - log.error("Port %s for device %s - not found", self.__devices[device]["device_config"].get('port', '/dev/ttyUSB0'), device) + self._log.error("Port %s for device %s - not found", self.__devices[device]["device_config"].get('port', '/dev/ttyUSB0'), device) except Exception as e: - log.exception(e) + self._log.exception(e) else: # if no exception handled - add device and change connection state self.__gateway.add_device(self.__devices[device]["device_config"]["name"], {"connector": self}, self.__devices[device]["device_config"]["type"]) self.__connected = True @@ -93,6 +96,9 @@ def open(self): # Function called by gateway on start def get_name(self): # Function used for logging, sending data and statistic return self.name + def get_type(self): + return self._connector_type + def is_connected(self): # Function for checking connection state return self.__connected @@ -103,15 +109,15 @@ def __load_converters(self, connector_type): # Function for search a converter for device_config in devices_config: if device_config.get('converter') is not None: converter = TBModuleLoader.import_module(connector_type, device_config['converter']) - self.__devices[device_config['name']] = {'converter': converter(device_config), + self.__devices[device_config['name']] = {'converter': converter(device_config, self._log), 'device_config': device_config} else: - log.error('Converter configuration for the custom connector %s -- not found, please check your configuration file.', self.get_name()) + self._log.error('Converter configuration for the custom connector %s -- not found, please check your configuration file.', self.get_name()) else: - log.error('Section "devices" in the configuration not found. A custom connector %s has being stopped.', self.get_name()) + self._log.error('Section "devices" in the configuration not found. A custom connector %s has being stopped.', self.get_name()) self.close() except Exception as e: - log.exception(e) + self._log.exception(e) def run(self): # Main loop of thread try: @@ -128,7 +134,7 @@ def run(self): # Main loop of thread self.__connect_to_devices() # if port not found - try to connect to it raise e except Exception as e: - log.exception(e) + self._log.exception(e) break else: data_from_device = data_from_device + received_character @@ -138,13 +144,13 @@ def run(self): # Main loop of thread self.__gateway.send_to_storage(self.get_name(), converted_data) time.sleep(.1) except Exception as e: - log.exception(e) + self._log.exception(e) self.close() raise e if not self.__connected: break except Exception as e: - log.exception(e) + self._log.exception(e) def close(self): # Close connect function, usually used if exception handled in gateway main loop or in connector main loop self.stopped = True @@ -152,25 +158,26 @@ def close(self): # Close connect function, usually used if exception handled in self.__gateway.del_device(self.__devices[device]["device_config"]["name"]) if self.__devices[device]['serial'].isOpen(): self.__devices[device]['serial'].close() + self._log.reset() def on_attributes_update(self, content): # Function used for processing attribute update requests from ThingsBoard - log.debug(content) + self._log.debug(content) if self.__devices.get(content["device"]) is not None: device_config = self.__devices[content["device"]].get("device_config") if device_config is not None and device_config.get("attributeUpdates") is not None: requests = device_config["attributeUpdates"] for request in requests: attribute = request.get("attributeOnThingsBoard") - log.debug(attribute) + self._log.debug(attribute) if attribute is not None and attribute in content["data"]: try: value = content["data"][attribute] str_to_send = str(request["stringToDevice"].replace("${" + attribute + "}", str(value))).encode("UTF-8") self.__devices[content["device"]]["serial"].write(str_to_send) - log.debug("Attribute update request to device %s : %s", content["device"], str_to_send) + self._log.debug("Attribute update request to device %s : %s", content["device"], str_to_send) time.sleep(.01) except Exception as e: - log.exception(e) + self._log.exception(e) def server_side_rpc_handler(self, content): pass diff --git a/thingsboard_gateway/extensions/serial/custom_serial_converter.py b/thingsboard_gateway/extensions/serial/custom_serial_converter.py index 6d5d85b8d..2b67c3ff1 100644 --- a/thingsboard_gateway/extensions/serial/custom_serial_converter.py +++ b/thingsboard_gateway/extensions/serial/custom_serial_converter.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.converter import Converter, log +from thingsboard_gateway.connectors.converter import Converter class CustomSerialUplinkConverter(Converter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config def convert(self, config, data: bytes): @@ -25,7 +26,7 @@ def convert(self, config, data: bytes): 'deviceType': self.__config.get('deviceType', 'default'), 'attributes': [], 'telemetry': [] - } + } keys = ['attributes', 'telemetry'] for key in keys: dict_result[key] = [] @@ -46,5 +47,5 @@ def convert(self, config, data: bytes): data_to_convert = data_to_convert[from_byte:] converted_data = {config_object['key']: data_to_convert.decode('UTF-8')} dict_result[key].append(converted_data) - log.debug("Converted data: %s", dict_result) + self._log.debug("Converted data: %s", dict_result) return dict_result diff --git a/thingsboard_gateway/gateway/statistics_service.py b/thingsboard_gateway/gateway/statistics_service.py index dca3694ab..f157be77e 100644 --- a/thingsboard_gateway/gateway/statistics_service.py +++ b/thingsboard_gateway/gateway/statistics_service.py @@ -2,6 +2,7 @@ import subprocess from threading import Thread from time import time, sleep +from platform import system as platform_system import simplejson @@ -61,8 +62,14 @@ def run(self) -> None: data_to_send = {} for attribute in self._config: try: - process = subprocess.run(attribute['command'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding='utf-8', timeout=attribute['timeout']) + if platform_system() == 'Windows': + process = subprocess.run(attribute['command'], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding='utf-8', timeout=attribute['timeout']) + else: + process = subprocess.run(['/bin/sh', '-c', attribute['command']], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding='utf-8', timeout=attribute['timeout']) except Exception as e: self._log.warning("Statistic parameter %s raise the exception: %s", attribute['attributeOnGateway'], e) @@ -72,12 +79,12 @@ def run(self) -> None: data_to_send[attribute['attributeOnGateway']] = value - self._gateway.tb_client.client.send_attributes(data_to_send) + self._gateway.tb_client.client.send_telemetry(data_to_send) if datetime.datetime.now() - self._last_streams_statistics_clear_time >= datetime.timedelta(days=1): self.clear_streams_statistics() - self._gateway.tb_client.client.send_attributes(StatisticsService.DATA_STREAMS_STATISTICS) + self._gateway.tb_client.client.send_telemetry(StatisticsService.DATA_STREAMS_STATISTICS) self._last_poll = time() diff --git a/thingsboard_gateway/gateway/tb_gateway_service.py b/thingsboard_gateway/gateway/tb_gateway_service.py index 6f8979195..e5a86c18b 100644 --- a/thingsboard_gateway/gateway/tb_gateway_service.py +++ b/thingsboard_gateway/gateway/tb_gateway_service.py @@ -1,4 +1,4 @@ -# Copyright 2022. ThingsBoard +# Copyright 2023. ThingsBoard # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ import logging.config import logging.handlers import multiprocessing.managers +import os.path +from signal import signal, SIGINT import subprocess from os import execv, listdir, path, pathsep, stat, system, environ from platform import system as platform_system @@ -27,7 +29,6 @@ from threading import RLock, Thread, main_thread, current_thread from time import sleep, time -import simplejson from simplejson import JSONDecodeError, dumps, load, loads from yaml import safe_load @@ -44,10 +45,11 @@ from thingsboard_gateway.storage.sqlite.sqlite_event_storage import SQLiteEventStorage from thingsboard_gateway.tb_utility.tb_gateway_remote_configurator import RemoteConfigurator from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader -from thingsboard_gateway.tb_utility.tb_logger import TBLoggerHandler +from thingsboard_gateway.tb_utility.tb_handler import TBLoggerHandler from thingsboard_gateway.tb_utility.tb_remote_shell import RemoteShell from thingsboard_gateway.tb_utility.tb_updater import TBUpdater from thingsboard_gateway.tb_utility.tb_utility import TBUtility +from thingsboard_gateway.tb_utility.tb_logger import TbLogger GRPC_LOADED = False try: @@ -58,7 +60,7 @@ except ImportError: print("Cannot load GRPC connector!") -log = logging.getLogger('service') +log = None main_handler = logging.handlers.MemoryHandler(-1) DEFAULT_CONNECTORS = { @@ -88,7 +90,7 @@ 'enable': False } -SECURITY_VAR = ('accessToken', 'caCert', 'privateKey', 'cert') +SECURITY_VAR = ('accessToken', 'caCert', 'privateKey', 'cert', 'clientId', 'username', 'password') def load_file(path_to_file): @@ -105,7 +107,10 @@ def get_env_variables(): 'accessToken': environ.get('accessToken'), 'caCert': environ.get('caCert'), 'privateKey': environ.get('privateKey'), - 'cert': environ.get('cert') + 'cert': environ.get('cert'), + 'clientId': environ.get('clientId'), + 'username': environ.get('username'), + 'password': environ.get('password') } converted_env_variables = {} @@ -135,7 +140,7 @@ def add_gateway(self, gateway): self.gateway = gateway def shutdown(self) -> None: - pass + super().shutdown() class TBGatewayService: @@ -162,23 +167,32 @@ def __init__(self, config_file=None): self.__async_device_actions_queue = SimpleQueue() self.__process_async_actions_thread = Thread(target=self.__process_async_device_actions, name="Async device actions processing thread", daemon=True) + + self._config_dir = path.dirname(path.abspath(config_file)) + path.sep if config_file is None: - config_file = path.dirname(path.dirname(path.abspath(__file__))) + '/config/tb_gateway.yaml'.replace('/', + config_file = path.dirname(path.dirname(path.abspath(__file__))) + '/config/tb_gateway.json'.replace('/', path.sep) - with open(config_file) as general_config: - self.__config = safe_load(general_config) - - # change main config if Gateway running with docker env variables - self.__modify_main_config() - self._config_dir = path.dirname(path.abspath(config_file)) + path.sep logging_error = None try: - logging.config.fileConfig(self._config_dir + "logs.conf", disable_existing_loggers=False) + with open(self._config_dir + 'logs.json', 'r') as file: + log_config = load(file) + logging.config.dictConfig(log_config) except Exception as e: logging_error = e + global log - log = logging.getLogger('service') + log = TbLogger('service', gateway=self, level='INFO') + global main_handler + self.main_handler = main_handler + log.addHandler(self.main_handler) + + # load general configuration YAML/JSON + self.__config = self.__load_general_config(config_file) + + # change main config if Gateway running with docker env variables + self.__modify_main_config() + log.info("Gateway starting...") self.__updater = TBUpdater() self.__updates_check_period_ms = 300000 @@ -207,10 +221,9 @@ def __init__(self, config_file=None): "LOGS": "Logging loading exception, logs.conf is wrong: %s" % (str(logging_error),)}}) TBLoggerHandler.set_default_handler() self.__rpc_reply_sent = False - global main_handler - self.main_handler = main_handler self.remote_handler = TBLoggerHandler(self) - self.main_handler.setTarget(self.remote_handler) + log.addHandler(self.remote_handler) + # self.main_handler.setTarget(self.remote_handler) self._default_connectors = DEFAULT_CONNECTORS self.__converted_data_queue = SimpleQueue() self.__save_converted_data_thread = Thread(name="Save converted data", daemon=True, @@ -231,11 +244,10 @@ def __init__(self, config_file=None): "device_renamed": self.__process_renamed_gateway_devices, "device_deleted": self.__process_deleted_gateway_devices, } + self.__remote_shell = None - if self.__config["thingsboard"].get("remoteShell"): - log.warning("Remote shell is enabled. Please be carefully with this feature.") - self.__remote_shell = RemoteShell(platform=self.__updater.get_platform(), - release=self.__updater.get_release()) + self.init_remote_shell(self.__config["thingsboard"].get("remoteShell")) + self.__rpc_remote_shell_command_in_progress = None self.__scheduled_rpc_calls = [] self.__rpc_processing_queue = SimpleQueue() @@ -250,16 +262,14 @@ def __init__(self, config_file=None): self.connectors_configs = {} self.__remote_configurator = None self.__request_config_after_connect = False - self.__init_remote_configuration() - self.__grpc_config = self.__config.get('grpc') + + self.__grpc_config = None + self.__grpc_connectors = None self.__grpc_manager = None - self.__grpc_connectors = {} - if GRPC_LOADED and self.__grpc_config is not None and self.__grpc_config.get("enabled"): - self.__process_async_actions_thread.start() - self.__grpc_manager = TBGRPCServerManager(self, self.__grpc_config) - self.__grpc_manager.set_gateway_read_callbacks(self.__register_connector, self.__unregister_connector) + self.init_grpc_service(self.__config.get('grpc')) + self._load_connectors() - self._connect_with_connectors() + self.__connect_with_connectors() self.__load_persistent_devices() self.__devices_idle_checker = self.__config['thingsboard'].get('checkingDeviceActivity', {}) @@ -269,13 +279,9 @@ def __init__(self, config_file=None): thread.start() log.info('Start checking devices idle time') - self.__statistics = self.__config['thingsboard'].get('statistics', DEFAULT_STATISTIC) + self.__statistics = None self.__statistics_service = None - if self.__statistics['enable']: - self.__statistics_service = StatisticsService(self.__statistics['statsSendPeriodInSeconds'], self, log, - config_path=self._config_dir + self.__statistics[ - 'configuration'] if self.__statistics.get( - 'configuration') else None) + self.init_statistics_service(self.__config['thingsboard'].get('statistics', DEFAULT_STATISTIC)) self._published_events = SimpleQueue() @@ -287,14 +293,15 @@ def __init__(self, config_file=None): name="Send data to Thingsboard Thread") self._send_thread.start() - self.__device_filter_config = self.__config['thingsboard'].get('deviceFiltering', DEFAULT_DEVICE_FILTER) + self.__device_filter_config = None self.__device_filter = None - if self.__device_filter_config['enable']: - self.__device_filter = DeviceFilter(config_path=self._config_dir + self.__device_filter_config[ - 'filterFile'] if self.__device_filter_config.get('filterFile') else None) + self.__grpc_manager = None + self.init_device_filtering(self.__config['thingsboard'].get('deviceFiltering', DEFAULT_DEVICE_FILTER)) self.__duplicate_detector = DuplicateDetector(self.available_connectors) + self.__init_remote_configuration() + log.info("Gateway started.") self._watchers_thread = Thread(target=self._watchers, name='Watchers', daemon=True) @@ -318,6 +325,84 @@ def __init__(self, config_file=None): self.server = self.manager.get_server() self.server.serve_forever() + @staticmethod + def __load_general_config(config_file): + file_extension = config_file.split('.')[-1] + if file_extension == 'json' and os.path.exists(config_file): + with open(config_file) as general_config: + try: + return load(general_config) + except Exception as e: + log.exception('Failed to load configuration file:\n %s', e) + else: + log.warning('YAML configuration is deprecated. ' + 'Please, use JSON configuration instead.') + log.warning( + 'See default configuration on ' + 'https://thingsboard.io/docs/iot-gateway/configuration/?storageConfig=sqlite#storage-configuration') + + config = {} + try: + with open(''.join(config_file.split('.')[:-1]) + '.yaml') as general_config: + config = safe_load(general_config) + + with open(config_file, 'w') as file: + file.writelines(dumps(config)) + except Exception as e: + log.exception('Failed to load configuration file:\n %s', e) + + return config + + def init_grpc_service(self, config): + self.__grpc_config = config + self.__grpc_connectors = {} + if GRPC_LOADED and self.__grpc_config is not None and self.__grpc_config.get("enabled"): + self.__process_async_actions_thread.start() + self.__grpc_manager = TBGRPCServerManager(self, self.__grpc_config) + self.__grpc_manager.set_gateway_read_callbacks(self.__register_connector, self.__unregister_connector) + + def init_statistics_service(self, config): + self.__statistics = config + if self.__statistics['enable']: + if isinstance(self.__statistics_service, StatisticsService): + self.__statistics_service.stop() + self.__statistics_service = StatisticsService(self.__statistics['statsSendPeriodInSeconds'], self, log, + config_path=self._config_dir + self.__statistics[ + 'configuration'] if self.__statistics.get( + 'configuration') else None) + else: + self.__statistics_service = None + + def init_device_filtering(self, config): + self.__device_filter_config = config + self.__device_filter = None + if self.__device_filter_config['enable']: + self.__device_filter = DeviceFilter(config_path=self._config_dir + self.__device_filter_config[ + 'filterFile'] if self.__device_filter_config.get('filterFile') else None) + else: + self.__device_filter = None + + def init_remote_shell(self, enable): + self.__remote_shell = None + if enable: + log.warning("Remote shell is enabled. Please be carefully with this feature.") + self.__remote_shell = RemoteShell(platform=self.__updater.get_platform(), + release=self.__updater.get_release()) + else: + self.__remote_shell = None + + @property + def event_storage_types(self): + return self._event_storage_types + + @property + def config(self): + return self.__config + + @config.setter + def config(self, config): + self.__config.update(config) + def get_gateway(self): if self.manager.has_gateway(): return self.manager.gateway @@ -424,7 +509,7 @@ def __stop_gateway(self): self.__updater.stop() log.info("Stopping...") - if self.__statistics_service: + if hasattr(self, '_TBGatewayService__statistics_service'): self.__statistics_service.stop() if self.__grpc_manager is not None: @@ -469,12 +554,12 @@ def _attributes_parse(self, content, *args): def __process_attribute_update(self, content): self.__process_remote_logging_update(content.get("RemoteLoggingLevel")) - self.__process_remote_configuration(content.get("configuration")) + self.__process_remote_configuration(content) self.__process_remote_converter_configuration_update(content) def __process_attributes_response(self, shared_attributes, client_attributes): self.__process_remote_logging_update(shared_attributes.get('RemoteLoggingLevel')) - self.__process_remote_configuration(shared_attributes.get("configuration")) + self.__process_remote_configuration(shared_attributes) def __process_remote_logging_update(self, remote_logging_level): if remote_logging_level == 'NONE': @@ -496,11 +581,8 @@ def __process_remote_converter_configuration_update(self, content: dict): raise ValueError self.available_connectors[connector_name].update_converter_config(converter_name, content[key]) - except (ValueError, AttributeError) as e: - log.error('Remote converter configuration update failed with exception:') - log.exception(e) - except IndexError: - log.debug('Received unknown request with content: %s', content) + except (ValueError, AttributeError, IndexError) as e: + log.debug('Failed to process remote converter update: %s', e) def update_connector_config_file(self, connector_name, config): for connector in self.__config['connectors']: @@ -548,8 +630,7 @@ def __process_renamed_gateway_devices(self, renamed_device: dict): def __process_remote_configuration(self, new_configuration): if new_configuration is not None and self.__remote_configurator is not None: try: - self.__remote_configurator.process_configuration(new_configuration) - self.__remote_configurator.send_current_configuration() + self.__remote_configurator.process_config_request(new_configuration) except Exception as e: log.exception(e) @@ -612,11 +693,20 @@ def _generate_persistent_key(connector, connectors_persistent_keys): return connector_persistent_key - def _load_connectors(self): + def load_connectors(self, config=None): + self._load_connectors(config=config) + + def _load_connectors(self, config=None): self.connectors_configs = {} connectors_persistent_keys = self.__load_persistent_connector_keys() - if self.__config.get("connectors"): - for connector in self.__config['connectors']: + + if config: + configuration = config.get('connectors') + else: + configuration = self.__config.get('connectors') + + if configuration: + for connector in configuration: try: connector_persistent_key = None if connector['type'] == "grpc" and self.__grpc_manager is None: @@ -626,7 +716,7 @@ def _load_connectors(self): if connector['type'] != "grpc": connector_class = None - if connector.get('useGRPC', True): + if connector.get('useGRPC', False): module_name = f'Grpc{self._default_connectors.get(connector["type"], connector.get("class"))}' connector_class = TBModuleLoader.import_module(connector['type'], module_name) @@ -683,26 +773,35 @@ def _load_connectors(self): self.__init_remote_configuration(force=True) log.info("Remote configuration is enabled forcibly!") - def _connect_with_connectors(self): + def connect_with_connectors(self): + self.__connect_with_connectors() + + def __connect_with_connectors(self): for connector_type in self.connectors_configs: for connector_config in self.connectors_configs[connector_type]: if self._implemented_connectors.get(connector_type.lower()) is not None: if connector_type.lower() != 'grpc' and 'Grpc' not in self._implemented_connectors[connector_type.lower()].__name__: for config in connector_config["config"]: connector = None + connector_name = None try: if connector_config["config"][config] is not None: - connector = self._implemented_connectors[connector_type](self, - connector_config["config"][ - config], - connector_type) - connector.name = connector_config["name"] - self.available_connectors[connector.get_name()] = connector - connector.open() + connector_name = connector_config["name"] + + if not self.available_connectors.get(connector_name): + connector = self._implemented_connectors[connector_type](self, + connector_config["config"][ + config], + connector_type) + connector.setName(connector_name) + self.available_connectors[connector.get_name()] = connector + connector.open() + else: + break else: log.info("Config not found for %s", connector_type) except Exception as e: - log.exception(e) + log.exception(e, attr_name=connector_name) if connector is not None: connector.close() else: @@ -711,7 +810,7 @@ def _connect_with_connectors(self): connector_dir_abs = "/".join(self._config_dir.split("/")[:-2]) connector_file_name = f'{connector_type}_connector.py' connector_abs_path = f'{connector_dir_abs}/grpc_connectors/{connector_type}/{connector_file_name}' - connector_config_json = simplejson.dumps({ + connector_config_json = dumps({ **connector_config, 'gateway': { 'host': 'localhost', @@ -741,7 +840,18 @@ def check_connector_configuration_updates(self): if configuration_changed: self.__close_connectors() self._load_connectors() - self._connect_with_connectors() + self.__connect_with_connectors() + + # Updating global self.__config['connectors'] configuration for states syncing + for connector_type in self.connectors_configs: + for connector_config in self.connectors_configs[connector_type]: + for (index, connector) in enumerate(self.__config['connectors']): + if connector_config['config'].get(connector['configuration']): + self.__config['connectors'][index]['configurationJson'] = connector_config['config'][ + connector['configuration']] + + if self.__remote_configurator is not None: + self.__remote_configurator.send_current_configuration() def send_to_storage(self, connector_name, data): try: @@ -872,7 +982,12 @@ def __convert_telemetry_to_ts(data): if item.get("ts") is None: telemetry = {**telemetry, **item} else: - telemetry_with_ts.append({"ts": item["ts"], "values": {**item["values"]}}) + if isinstance(item['ts'], int): + telemetry_with_ts.append({"ts": item["ts"], "values": {**item["values"]}}) + else: + log.warning('Data has invalid TS (timestamp) format! Using generated TS instead.') + telemetry_with_ts.append({"ts": int(time() * 1000), "values": {**item["values"]}}) + if telemetry_with_ts: data["telemetry"] = telemetry_with_ts elif len(data['telemetry']) > 0: @@ -1187,7 +1302,7 @@ def _attribute_update_callback(self, content, *args): def __form_statistics(self): summary_messages = {"eventsProduced": 0, "eventsSent": 0} for connector in self.available_connectors: - connector_camel_case = connector.lower().replace(' ', '') + connector_camel_case = connector.replace(' ', '') telemetry = { (connector_camel_case + ' EventsProduced').replace(' ', ''): self.available_connectors[ connector].statistics.get('MessagesReceived', 0), @@ -1214,7 +1329,12 @@ def add_device(self, device_name, content, device_type=None, reconnect=False): self.__connected_devices[device_name] = {**content, "device_type": device_type} self.__saved_devices[device_name] = {**content, "device_type": device_type} self.__save_persistent_devices() + device_details = { + 'connectorType': content['connector'].get_type(), + 'connectorName': content['connector'].get_name() + } self.tb_client.client.gw_connect_device(device_name, device_type) + self.tb_client.client.gw_send_attributes(device_name, device_details) def update_device(self, device_name, event, content): if event == 'connector' and self.__connected_devices[device_name].get(event) != content: diff --git a/thingsboard_gateway/grpc_connectors/modbus/bytes_modbus_uplink_converter.py b/thingsboard_gateway/grpc_connectors/modbus/bytes_modbus_uplink_converter.py index c1b43b5cc..24ac6e979 100644 --- a/thingsboard_gateway/grpc_connectors/modbus/bytes_modbus_uplink_converter.py +++ b/thingsboard_gateway/grpc_connectors/modbus/bytes_modbus_uplink_converter.py @@ -18,11 +18,12 @@ from pymodbus.pdu import ExceptionResponse from thingsboard_gateway.connectors.modbus.bytes_modbus_uplink_converter import BytesModbusUplinkConverter -from thingsboard_gateway.connectors.modbus.modbus_converter import ModbusConverter, log +from thingsboard_gateway.connectors.modbus.modbus_converter import ModbusConverter class GrpcBytesModbusUplinkConverter(ModbusConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__datatypes = { "timeseries": "telemetry", "attributes": "attributes" @@ -68,7 +69,7 @@ def convert(self, config, data): elif configuration["functionCode"] in [2, 3, 4]: decoder = None registers = response.registers - log.debug("Tag: %s Config: %s registers: %s", tag, str(configuration), str(registers)) + self._log.debug("Tag: %s Config: %s registers: %s", tag, str(configuration), str(registers)) try: decoder = BinaryPayloadDecoder.fromRegisters(registers, byteorder=endian_order, wordorder=word_endian_order) @@ -83,15 +84,15 @@ def convert(self, config, data): if configuration.get("multiplier"): decoded_data = decoded_data * configuration["multiplier"] else: - log.exception(response) + self._log.exception(response) decoded_data = None if config_data == "rpc": return decoded_data - log.debug("datatype: %s \t key: %s \t value: %s", self.__datatypes[config_data], tag, + self._log.debug("datatype: %s \t key: %s \t value: %s", self.__datatypes[config_data], tag, str(decoded_data)) if decoded_data is not None: self.__result[self.__datatypes[config_data]][tag] = decoded_data except Exception as e: - log.exception(e) - log.debug(self.__result) + self._log.exception(e) + self._log.debug(self.__result) return self.__result diff --git a/thingsboard_gateway/grpc_connectors/modbus/modbus_connector.py b/thingsboard_gateway/grpc_connectors/modbus/modbus_connector.py index 4fce58b91..ab23364a1 100644 --- a/thingsboard_gateway/grpc_connectors/modbus/modbus_connector.py +++ b/thingsboard_gateway/grpc_connectors/modbus/modbus_connector.py @@ -172,7 +172,7 @@ def __modify_main_config(self): def __load_slaves(self): self.__slaves = [ Slave(**{**device, 'connector': self, 'connector_key': self._connector_key, - 'callback': GrpcModbusConnector.callback}) + 'callback': GrpcModbusConnector.callback, 'logger': log}) for device in self.__config.get('master', {'slaves': []}).get('slaves', [])] @classmethod @@ -190,6 +190,9 @@ def stop(self): def get_name(self): return self.name + def get_type(self): + return self._connector_type + def __process_slaves(self): device = GrpcModbusConnector.process_requests.get() diff --git a/thingsboard_gateway/grpc_connectors/modbus/slave.py b/thingsboard_gateway/grpc_connectors/modbus/slave.py index 3df983c65..b3d137126 100644 --- a/thingsboard_gateway/grpc_connectors/modbus/slave.py +++ b/thingsboard_gateway/grpc_connectors/modbus/slave.py @@ -19,7 +19,6 @@ from thingsboard_gateway.connectors.modbus.bytes_modbus_downlink_converter import BytesModbusDownlinkConverter from thingsboard_gateway.connectors.modbus.constants import * -from thingsboard_gateway.connectors.connector import log from thingsboard_gateway.grpc_connectors.gw_grpc_msg_creator import GrpcMsgCreator from thingsboard_gateway.grpc_connectors.modbus.bytes_modbus_uplink_converter import GrpcBytesModbusUplinkConverter from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader @@ -28,6 +27,7 @@ class Slave(Thread): def __init__(self, **kwargs): super().__init__() + self._log = kwargs['logger'] self.timeout = kwargs.get('timeout') self.name = kwargs['deviceName'] self.poll_period = kwargs['pollPeriod'] / 1000 @@ -112,7 +112,7 @@ def __load_converters(self, connector, connector_key): self.config[UPLINK_PREFIX + CONVERTER_PARAMETER] = converter self.config[DOWNLINK_PREFIX + CONVERTER_PARAMETER] = downlink_converter except Exception as e: - log.exception(e) + self._log.exception(e) def __str__(self): return f'{self.name}' diff --git a/thingsboard_gateway/grpc_connectors/mqtt/bytes_mqtt_uplink_converter.py b/thingsboard_gateway/grpc_connectors/mqtt/bytes_mqtt_uplink_converter.py index 267bbf493..cf7ee9e1b 100644 --- a/thingsboard_gateway/grpc_connectors/mqtt/bytes_mqtt_uplink_converter.py +++ b/thingsboard_gateway/grpc_connectors/mqtt/bytes_mqtt_uplink_converter.py @@ -33,8 +33,8 @@ def convert(self, topic, data): else: dict_result[datatypes[datatype]].append(value_item) except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), str(data)) - log.exception(e) + log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config), str(data), + e) log.debug('Converted data: %s', dict_result) return dict_result diff --git a/thingsboard_gateway/grpc_connectors/mqtt/json_mqtt_uplink_converter.py b/thingsboard_gateway/grpc_connectors/mqtt/json_mqtt_uplink_converter.py index 31397012a..b060c3383 100644 --- a/thingsboard_gateway/grpc_connectors/mqtt/json_mqtt_uplink_converter.py +++ b/thingsboard_gateway/grpc_connectors/mqtt/json_mqtt_uplink_converter.py @@ -17,12 +17,13 @@ from simplejson import dumps -from thingsboard_gateway.connectors.mqtt.mqtt_uplink_converter import MqttUplinkConverter, log +from thingsboard_gateway.connectors.mqtt.mqtt_uplink_converter import MqttUplinkConverter from thingsboard_gateway.tb_utility.tb_utility import TBUtility class JsonGrpcMqttUplinkConverter(MqttUplinkConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config.get('converter') def convert(self, config, data): @@ -50,13 +51,13 @@ def convert(self, config, data): if search_result is not None: dict_result["deviceName"] = search_result.group(0) else: - log.debug( + self._log.debug( "Regular expression result is None. deviceNameTopicExpression parameter will be interpreted " "as a deviceName\n Topic: %s\nRegex: %s", config, self.__config.get("deviceNameTopicExpression")) dict_result["deviceName"] = self.__config.get("deviceNameTopicExpression") else: - log.error("The expression for looking \"deviceName\" not found in config %s", dumps(self.__config)) + self._log.error("The expression for looking \"deviceName\" not found in config %s", dumps(self.__config)) if self.__config.get("deviceTypeJsonExpression") is not None: device_type_tags = TBUtility.get_values(self.__config.get("deviceTypeJsonExpression"), data, @@ -77,17 +78,18 @@ def convert(self, config, data): if search_result is not None: dict_result["deviceType"] = search_result.group(0) else: - log.debug( + self._log.debug( "Regular expression result is None. deviceTypeTopicExpression will be interpreted as " "a deviceType\n Topic: %s\nRegex: %s", config, self.__config.get("deviceTypeTopicExpression")) dict_result["deviceType"] = self.__config.get("deviceTypeTopicExpression") else: - log.error("The expression for looking \"deviceType\" not found in config %s", dumps(self.__config)) + self._log.error("The expression for looking \"deviceType\" not found in config %s", + dumps(self.__config)) except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), data) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config), data, + e) try: for datatype in datatypes: @@ -128,6 +130,7 @@ def convert(self, config, data): dict_result[datatypes[datatype]] = {} dict_result[datatypes[datatype]][full_key] = full_value except Exception as e: - log.error('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), str(data)) - log.exception(e) + self._log.error('Error in converter, for config: \n%s\n and message: \n%s\n %s', dumps(self.__config), + str(data), e) + return dict_result diff --git a/thingsboard_gateway/grpc_connectors/mqtt/mqtt_connector.py b/thingsboard_gateway/grpc_connectors/mqtt/mqtt_connector.py index 3ec0aba11..b2bcf2665 100644 --- a/thingsboard_gateway/grpc_connectors/mqtt/mqtt_connector.py +++ b/thingsboard_gateway/grpc_connectors/mqtt/mqtt_connector.py @@ -130,9 +130,8 @@ def __init__(self, connector_config: str, config_dir_path: str): ciphers=None) except Exception as e: log.error("Cannot setup connection to broker %s using SSL. " - "Please check your configuration.\nError: ", - self.get_name()) - log.exception(e) + "Please check your configuration.\nError: %s", + self.get_name(), e) if self.__broker["security"].get("insecure", False): self._client.tls_insecure_set(True) else: @@ -623,8 +622,7 @@ def notify_attribute(self, content): self._client.publish(topic, data, retain=retain).wait_for_publish() return except (AttributeError, IndexError) as e: - log.error('Error when processing attribute response') - log.exception(e) + log.error('Error when processing attribute response\n %s', e) return def on_attributes_update(self, content): diff --git a/thingsboard_gateway/grpc_connectors/opcua/opcua_connector.py b/thingsboard_gateway/grpc_connectors/opcua/opcua_connector.py index 54960f8ac..86c27f88a 100644 --- a/thingsboard_gateway/grpc_connectors/opcua/opcua_connector.py +++ b/thingsboard_gateway/grpc_connectors/opcua/opcua_connector.py @@ -114,8 +114,7 @@ def __connect(self): try: self.client.load_type_definitions() except Exception as e: - log.error("Error on loading type definitions:") - log.error(e) + log.error("Error on loading type definitions:\n %s", e) log.debug(self.client.get_namespace_array()[-1]) log.debug(self.client.get_namespace_index(self.client.get_namespace_array()[-1])) @@ -133,8 +132,7 @@ def __connect(self): log.error("Connection refused on connection to OPC-UA server with url %s", self.__server_conf.get("url")) time.sleep(10) except Exception as e: - log.debug("error on connection to OPC-UA server.") - log.error(e) + log.debug("error on connection to OPC-UA server.\n %s", e) time.sleep(10) def run(self): @@ -179,9 +177,8 @@ def run(self): except FuturesTimeoutError: self.__check_connection() except Exception as e: - log.error("Connection failed on connection to OPC-UA server with url %s", - self.__server_conf.get("url")) - log.exception(e) + log.error("Connection failed on connection to OPC-UA server with url %s.\n %s", + self.__server_conf.get("url"), e) time.sleep(10) @@ -243,6 +240,9 @@ def stop(self): def get_name(self): return self.name + def get_type(self): + return self._connector_type + def on_attributes_update(self, content): log.debug(content) try: diff --git a/thingsboard_gateway/grpc_connectors/opcua/opcua_uplink_converter.py b/thingsboard_gateway/grpc_connectors/opcua/opcua_uplink_converter.py index 1d1ec7521..a0e9bb8ca 100644 --- a/thingsboard_gateway/grpc_connectors/opcua/opcua_uplink_converter.py +++ b/thingsboard_gateway/grpc_connectors/opcua/opcua_uplink_converter.py @@ -16,12 +16,13 @@ from time import time from datetime import timezone -from thingsboard_gateway.connectors.opcua.opcua_converter import OpcUaConverter, log +from thingsboard_gateway.connectors.opcua.opcua_converter import OpcUaConverter from thingsboard_gateway.tb_utility.tb_utility import TBUtility class GrpcOpcUaUplinkConverter(OpcUaConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config def convert(self, config, val, data=None): @@ -68,4 +69,4 @@ def convert(self, config, val, data=None): result[information_types[information_type]][full_key] = full_value return result except Exception as e: - log.exception(e) + self._log.exception(e) diff --git a/thingsboard_gateway/grpc_connectors/socket/bytes_socket_uplink_converter.py b/thingsboard_gateway/grpc_connectors/socket/bytes_socket_uplink_converter.py index d2e2e286f..b5eef5f11 100644 --- a/thingsboard_gateway/grpc_connectors/socket/bytes_socket_uplink_converter.py +++ b/thingsboard_gateway/grpc_connectors/socket/bytes_socket_uplink_converter.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from thingsboard_gateway.connectors.socket.socket_uplink_converter import SocketUplinkConverter, log +from thingsboard_gateway.connectors.socket.socket_uplink_converter import SocketUplinkConverter class BytesGrpcSocketUplinkConverter(SocketUplinkConverter): - def __init__(self, config): + def __init__(self, config, logger): + self._log = logger self.__config = config def convert(self, config, data): @@ -46,11 +47,12 @@ def convert(self, config, data): if item.get('key') is not None: dict_result[section][item['key']] = converted_data else: - log.error('Key for %s not found in config: %s', config['type'], config['section_config']) + self._log.error('Key for %s not found in config: %s', config['type'], + config['section_config']) except Exception as e: - log.exception(e) + self._log.exception(e) except Exception as e: - log.exception(e) + self._log.exception(e) - log.debug(dict_result) + self._log.debug(dict_result) return dict_result diff --git a/thingsboard_gateway/grpc_connectors/socket/socket_connector.py b/thingsboard_gateway/grpc_connectors/socket/socket_connector.py index 24b76e7e8..9db18ef70 100644 --- a/thingsboard_gateway/grpc_connectors/socket/socket_connector.py +++ b/thingsboard_gateway/grpc_connectors/socket/socket_connector.py @@ -272,6 +272,12 @@ def stop(self): self.__socket.shutdown(socket.SHUT_RDWR) self.__socket.close() + def get_name(self): + return self.name + + def get_type(self): + return self._connector_type + if __name__ == '__main__': import sys diff --git a/thingsboard_gateway/tb_gateway.py b/thingsboard_gateway/tb_gateway.py index cd24efb24..d9fcc79b6 100644 --- a/thingsboard_gateway/tb_gateway.py +++ b/thingsboard_gateway/tb_gateway.py @@ -30,11 +30,11 @@ def main(): if hot_reload: HotReloader(TBGatewayService) else: - TBGatewayService(path.dirname(path.abspath(__file__)) + '/config/tb_gateway.yaml'.replace('/', path.sep)) + TBGatewayService(path.dirname(path.abspath(__file__)) + '/config/tb_gateway.json'.replace('/', path.sep)) def daemon(): - TBGatewayService("/etc/thingsboard-gateway/config/tb_gateway.yaml".replace('/', path.sep)) + TBGatewayService("/etc/thingsboard-gateway/config/tb_gateway.json".replace('/', path.sep)) if __name__ == '__main__': diff --git a/thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py b/thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py index f0d02729e..aff55ff16 100644 --- a/thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py +++ b/thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py @@ -1,4 +1,4 @@ -# Copyright 2022. ThingsBoard +# Copyright 2023. ThingsBoard # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,288 +12,569 @@ # See the License for the specific language governing permissions and # limitations under the License. -from base64 import b64decode, b64encode -from configparser import ConfigParser +import os.path from logging import getLogger -from logging.config import fileConfig -from os import linesep, remove, stat -from os.path import dirname, exists -from re import findall from time import sleep, time +from logging.config import dictConfig -from simplejson import dump, dumps, loads -from yaml import safe_dump +from regex import fullmatch +from simplejson import dumps, load from thingsboard_gateway.gateway.tb_client import TBClient -from thingsboard_gateway.tb_utility.tb_loader import TBModuleLoader -from thingsboard_gateway.tb_utility.tb_logger import TBLoggerHandler +from thingsboard_gateway.tb_utility.tb_handler import TBLoggerHandler -# pylint: disable=protected-access LOG = getLogger("service") class RemoteConfigurator: + DEFAULT_STATISTICS = { + 'enable': True, + 'statsSendPeriodInSeconds': 3600 + } + def __init__(self, gateway, config): - self.__gateway = gateway - self.__new_configuration = None - self.__old_configuration = None - self.__apply_timeout = 10 - self.__old_tb_client = None - self.__old_logs_configuration = self.__get_current_logs_configuration() - self.__new_logs_configuration = None - self.__old_connectors_configs = {} - self.__new_connectors_configs = {} - self.__old_general_configuration_file = config - self.__new_general_configuration_file = {} - self.__old_event_storage = None - self.__new_event_storage = None + self._gateway = gateway + self._config = config + self._load_connectors_configuration() + self._logs_configuration = self._load_logs_configuration() self.in_process = False + self._active_connectors = [] + self._handlers = { + 'general_configuration': self._handle_general_configuration_update, + 'storage_configuration': self._handle_storage_configuration_update, + 'grpc_configuration': self._handle_grpc_configuration_update, + 'logs_configuration': self._handle_logs_configuration_update, + 'active_connectors': self._handle_active_connectors_update, + 'RemoteLoggingLevel': self._handle_remote_logging_level_update, + r'(?=\D*\d?).*': self._handle_connector_configuration_update, + } + self._modifiable_static_attrs = { + 'logs_configuration': 'logs.json' + } + LOG.info('Remote Configurator started') - def process_configuration(self, configuration): - try: - if not self.in_process: - self.in_process = True - # while not self.__gateway._published_events.empty(): - # LOG.debug("Waiting for end of the data processing...") - # sleep(1) - decoded_configuration = b64decode(configuration) - self.__new_configuration = loads(decoded_configuration) - self.__old_connectors_configs = self.__gateway.connectors_configs - self.__new_general_configuration_file = self.__new_configuration.get("thingsboard") - - # To maintain RemoteShell status - if not isinstance(self.__old_configuration, dict): - self.__old_configuration = loads(b64decode(self.__old_configuration)) - if self.__old_configuration.get("thingsboard").get("thingsboard").get("remoteShell"): - self.__new_configuration["thingsboard"]["thingsboard"]["remoteShell"] = True - - self.__new_logs_configuration = b64decode(self.__new_general_configuration_file.pop("logs")).decode('UTF-8').replace('}}', '\n') - if self.__old_configuration != decoded_configuration: - LOG.info("Remote configuration received: \n %s", decoded_configuration) - result = self.__process_connectors_configuration() - self.in_process = False - if result: - self.__old_configuration = self.__new_configuration - return True - else: - return False - else: - LOG.info("Remote configuration is the same.") - else: - LOG.error("Remote configuration is already in processing") - return False - except Exception as e: - self.in_process = False - LOG.exception(e) + @property + def general_configuration(self): + return self._config.get('thingsboard', {}) + + @general_configuration.setter + def general_configuration(self, config): + self._config['thingsboard'].update(config) + + @property + def storage_configuration(self): + return self._config.get('storage', {}) + + @storage_configuration.setter + def storage_configuration(self, config): + self._config['storage'].update(config) + + @property + def grpc_configuration(self): + return self._config.get('grpc', {}) + + @grpc_configuration.setter + def grpc_configuration(self, config): + self._config.get('grpc', {}).update(config) + + @property + def connectors_configuration(self): + connectors = self._config.get('connectors', []) + for connector in connectors: + connector.pop('config_updated', None) + connector.pop('config_file_path', None) + return connectors + + def _get_active_connectors(self): + return [connector['name'] for connector in self.connectors_configuration] + + def _get_general_config_in_local_format(self): + """ + Method returns general configuration in format that should be used only for local files + !!!Don't use it for sending data to TB (use `_get_general_config_in_remote_format` instead)!!! + """ + + connectors_config = [ + {'type': connector['type'], 'name': connector['name'], 'configuration': connector['configuration']} for + connector in self.connectors_configuration] + + return { + 'thingsboard': self.general_configuration, + 'storage': self.storage_configuration, + 'grpc': self.grpc_configuration, + 'connectors': connectors_config + } + + def _get_general_config_in_remote_format(self): + """ + Method returns general configuration in format that should be used only for sending configuration to the server. + !!!Don't use it for saving data to conf files (use `_get_general_config_in_local_format`)!!! + """ + + stat_conf_path = self.general_configuration['statistics'].get('configuration') + commands = [] + if stat_conf_path: + with open(self._gateway.get_config_path() + stat_conf_path, 'r') as file: + commands = load(file) + config = self.general_configuration + config.update( + { + 'statistics': { + 'enable': self.general_configuration['statistics']['enable'], + 'statsSendPeriodInSeconds': self.general_configuration['statistics']['statsSendPeriodInSeconds'], + 'configuration': self.general_configuration['statistics'].get('configuration'), + 'commands': commands + } + } + ) + return config def send_current_configuration(self): + """ + Calling manually only on RemoteConfigurator init (TbGatewayService.__init_remote_configuration method) + When Gateway started, sending all configs for states synchronizing + """ + + LOG.debug('Sending all configurations (init)') + self._gateway.tb_client.client.send_attributes( + {'general_configuration': self._get_general_config_in_remote_format()}) + self._gateway.tb_client.client.send_attributes({'storage_configuration': self.storage_configuration}) + self._gateway.tb_client.client.send_attributes({'grpc_configuration': self.grpc_configuration}) + self._gateway.tb_client.client.send_attributes( + {'logs_configuration': {**self._logs_configuration, 'ts': int(time() * 1000)}}) + self._gateway.tb_client.client.send_attributes({'active_connectors': self._get_active_connectors()}) + self._gateway.tb_client.client.send_attributes({'Version': self._gateway.version.get('current_version', 0.0)}) + for connector in self.connectors_configuration: + self._gateway.tb_client.client.send_attributes( + {connector['name']: {**connector, 'logLevel': connector['configurationJson']['logLevel'], + 'ts': int(time() * 1000)}}) + + def _load_connectors_configuration(self): + for (_, connector_list) in self._gateway.connectors_configs.items(): + for connector in connector_list: + for general_connector_config in self.connectors_configuration: + if general_connector_config['name'] == connector['name']: + config = connector.pop('config')[general_connector_config['configuration']] + general_connector_config.update(connector) + general_connector_config['configurationJson'] = config + + def _load_logs_configuration(self): + """ + Calling only on RemoteConfigurator init (__init__ method) + """ + try: - current_configuration = {} - for connector in self.__gateway.connectors_configs: - if current_configuration.get(connector) is None: - current_configuration[connector] = [] - for config in self.__gateway.connectors_configs[connector]: - for config_file in config['config']: - current_configuration[connector].append({'name': config['name'], 'config': config['config'][config_file]}) - current_configuration["thingsboard"] = self.__old_general_configuration_file - current_configuration["thingsboard"]["logs"] = b64encode(self.__old_logs_configuration.replace('\n', '}}').encode("UTF-8")) - json_current_configuration = dumps(current_configuration) - encoded_current_configuration = b64encode(json_current_configuration.encode()) - self.__old_configuration = encoded_current_configuration - self.__gateway.tb_client.client.send_attributes( - {"current_configuration": encoded_current_configuration.decode("UTF-8")}) - LOG.debug('Current configuration has been sent to ThingsBoard: %s', json_current_configuration) + with open(self._gateway.get_config_path() + 'logs.json', 'r') as logs: + return load(logs) except Exception as e: LOG.exception(e) + return {} - def __process_connectors_configuration(self): - LOG.info("Processing remote connectors configuration...") - if self.__apply_new_connectors_configuration(): - self.__write_new_configuration_files() - self.__apply_storage_configuration() - if self.__safe_apply_connection_configuration(): - LOG.info("Remote configuration has been applied.") - with open(self.__gateway.get_config_path() + "tb_gateway.yaml", "w", encoding="UTF-8") as general_configuration_file: - safe_dump(self.__new_general_configuration_file, general_configuration_file) - self.__old_connectors_configs = {} - self.__new_connectors_configs = {} - self.__old_general_configuration_file = self.__new_general_configuration_file - self.__old_logs_configuration = self.__new_logs_configuration - self.__update_logs_configuration() - self.__new_logs_configuration = None - self.__new_general_configuration_file = {} - return True + def process_config_request(self, config): + if not self.in_process: + LOG.debug('Got config update request: %s', config) + + self.in_process = True + + try: + for attr_name in config.keys(): + if 'deleted' in attr_name: + continue + + request_config = config[attr_name] + if not self._is_modified(attr_name, request_config): + continue + + for (name, func) in self._handlers.items(): + if fullmatch(name, attr_name): + func(request_config) + break + except (KeyError, AttributeError): + LOG.error('Unknown attribute update name (Available: %s)', ', '.join(self._handlers.keys())) + finally: + self.in_process = False else: - self.__update_logs_configuration() - self.__old_general_configuration_file.pop("logs") - with open(self.__gateway.get_config_path() + "tb_gateway.yaml", "w", encoding="UTF-8") as general_configuration_file: - safe_dump(self.__old_general_configuration_file, general_configuration_file) - LOG.error("A remote general configuration applying has been failed.") - self.__old_connectors_configs = {} - self.__new_connectors_configs = {} - self.__new_logs_configuration = None - self.__new_general_configuration_file = {} - return False + LOG.error("Remote configuration is already in processing") + + # HANDLERS --------------------------------------------------------------------------------------------------------- + def _handle_general_configuration_update(self, config): + """ + General configuration update handling in 5 steps: + 1. Checking if connection changed (host, port, QoS, security or provisioning sections); + 2. Checking if statistics collecting changed; + 3. Checking if device filtering changed; + 4. Checking if Remote Shell on/off state changed; + 5. Updating other params (regardless of whether they have changed): + a. maxPayloadSizeBytes; + b. minPackSendDelayMS; + c. minPackSizeToSend; + d. checkConnectorsConfigurationInSeconds; + f. handleDeviceRenaming. + + If config from steps 1-4 changed: + True -> applying new config with related objects creating (old objects will remove). + """ + + LOG.info('Processing general configuration update') + + LOG.info('--- Checking connection configuration changes...') + if config['host'] != self.general_configuration['host'] or config['port'] != self.general_configuration[ + 'port'] or config['security'] != self.general_configuration['security'] or config.get('provisioning', + {}) != self.general_configuration.get( + 'provisioning', {}) or config['qos'] != self.general_configuration['qos']: + LOG.info('---- Connection configuration changed. Processing...') + success = self._apply_connection_config(config) + if not success: + config.update(self.general_configuration) + else: + LOG.info('--- Connection configuration not changed.') + + LOG.info('--- Checking statistics configuration changes...') + changed = self._check_statistics_configuration_changes(config.get('statistics', self.DEFAULT_STATISTICS)) + if changed: + LOG.info('---- Statistics configuration changed. Processing...') + success = self._apply_statistics_config(config.get('statistics', self.DEFAULT_STATISTICS)) + if not success: + config['statistics'].update(self.general_configuration['statistics']) + else: + LOG.info('--- Statistics configuration not changed.') + + LOG.info('--- Checking device filtering configuration changes...') + if config.get('deviceFiltering') != self.general_configuration.get('deviceFiltering'): + LOG.info('---- Device filtering configuration changed. Processing...') + success = self._apply_device_filtering_config(config) + if not success: + config['deviceFiltering'].update(self.general_configuration['deviceFiltering']) + else: + LOG.info('--- Device filtering configuration not changed.') + + LOG.info('--- Checking Remote Shell configuration changes...') + if config.get('remoteShell') != self.general_configuration.get('remoteShell'): + LOG.info('---- Remote Shell configuration changed. Processing...') + success = self._apply_remote_shell_config(config) + if not success: + config['remoteShell'].update(self.general_configuration['remoteShell']) + else: + LOG.info('--- Remote Shell configuration not changed.') + + LOG.info('--- Checking other configuration parameters changes...') + self._apply_other_params_config(config) + + LOG.info('--- Saving new general configuration...') + self.general_configuration = config + self._gateway.tb_client.client.send_attributes({'general_configuration': self.general_configuration}) + self._cleanup() + with open(self._gateway.get_config_path() + "tb_gateway.json", "w", + encoding="UTF-8") as file: + file.writelines(dumps(self._get_general_config_in_local_format(), indent=' ')) - def __prepare_connectors_configuration(self, input_connector_config): + def _handle_storage_configuration_update(self, config): + LOG.debug('Processing storage configuration update...') + + old_event_storage = self._gateway._event_storage try: - self.__gateway.connectors_configs = {} - for connector in input_connector_config['thingsboard']['connectors']: - for input_connector in input_connector_config[connector['type']]: - if input_connector['name'] == connector['name']: - if not self.__gateway.connectors_configs.get(connector['type']): - self.__gateway.connectors_configs[connector['type']] = [] - config_file_path = self.__gateway.get_config_path() + connector['configuration'] - # Create the configuration json file if not exists - open(config_file_path, 'w') - self.__gateway.connectors_configs[connector['type']].append( - {"name": connector["name"], - "config": {connector['configuration']: input_connector["config"]}, - "config_updated": stat(config_file_path), - "config_file_path": config_file_path}) - connector_class = TBModuleLoader.import_module(connector["type"], - self.__gateway._default_connectors.get(connector["type"], connector.get("class"))) - self.__gateway._implemented_connectors[connector["type"]] = connector_class + storage_class = self._gateway.event_storage_types[config["type"]] + self._gateway._event_storage = storage_class(config) except Exception as e: + LOG.error('Something went wrong with applying the new storage configuration. Reverting...') LOG.exception(e) + self._gateway._event_storage = old_event_storage + else: + self.storage_configuration = config + with open(self._gateway.get_config_path() + "tb_gateway.json", "w", encoding="UTF-8") as file: + file.writelines(dumps(self._get_general_config_in_local_format(), indent=' ')) + self._gateway.tb_client.client.send_attributes({'storage_configuration': self.storage_configuration}) + + LOG.info('Processed storage configuration update successfully') + + def _handle_grpc_configuration_update(self, config): + LOG.debug('Processing GRPC configuration update...') + if config != self.grpc_configuration: + try: + self._gateway.init_grpc_service(config) + for connector_name in self._gateway.available_connectors: + self._gateway.available_connectors[connector_name].close() + self._gateway.load_connectors(self._get_general_config_in_local_format()) + self._gateway.connect_with_connectors() + except Exception as e: + LOG.error('Something went wrong with applying the new GRPC configuration. Reverting...') + LOG.exception(e) + self._gateway.init_grpc_service(self.grpc_configuration) + for connector_name in self._gateway.available_connectors: + self._gateway.available_connectors[connector_name].close() + self._gateway.load_connectors(self._get_general_config_in_local_format()) + self._gateway.connect_with_connectors() + else: + self.grpc_configuration = config + with open(self._gateway.get_config_path() + "tb_gateway.json", "w", encoding="UTF-8") as file: + file.writelines(dumps(self._get_general_config_in_local_format(), indent=' ')) + self._gateway.tb_client.client.send_attributes({'grpc_configuration': self.grpc_configuration}) - def __apply_new_connectors_configuration(self): + LOG.info('Processed GRPC configuration update successfully') + + def _handle_logs_configuration_update(self, config): + global LOG + LOG.debug('Processing logs configuration update...') try: - self.__prepare_connectors_configuration(self.__new_configuration) - for connector_name in self.__gateway.available_connectors: + LOG = getLogger('service') + logs_conf_file_path = self._gateway.get_config_path() + 'logs.json' + + dictConfig(config) + LOG = getLogger('service') + self._gateway.remote_handler = TBLoggerHandler(self._gateway) + self._gateway.remote_handler.activate(self._gateway.main_handler.level) + self._gateway.main_handler.setTarget(self._gateway.remote_handler) + LOG.addHandler(self._gateway.remote_handler) + + with open(logs_conf_file_path, 'w') as logs: + logs.write(dumps(config, indent=' ')) + + LOG.debug("Logs configuration has been updated.") + self._gateway.tb_client.client.send_attributes({'logs_configuration': config}) + except Exception as e: + LOG.error("Remote logging configuration is wrong!") + LOG.exception(e) + + def _handle_active_connectors_update(self, config): + LOG.debug('Processing active connectors configuration update...') + + has_changed = False + for_deletion = [] + for active_connector_name in self._gateway.available_connectors: + if active_connector_name not in config: try: - self.__gateway.available_connectors[connector_name].close() + self._gateway.available_connectors[active_connector_name].close() + for_deletion.append(active_connector_name) + has_changed = True except Exception as e: LOG.exception(e) - self.__gateway._connect_with_connectors() - LOG.debug("New connectors configuration has been applied") - self.__old_connectors_configs = {} - return True - except Exception as e: - self.__gateway.connectors_configs = self.__old_connectors_configs - for connector_name in self.__gateway.available_connectors: - self.__gateway.available_connectors[connector_name].close() - self.__gateway._load_connectors(self.__old_general_configuration_file) - self.__gateway._connect_with_connectors() - LOG.exception(e) - return False - def __write_new_configuration_files(self): + if has_changed: + for name in for_deletion: + self._gateway.available_connectors.pop(name) + + self._delete_connectors_from_config(config) + with open(self._gateway.get_config_path() + 'tb_gateway.json', 'w') as file: + file.writelines(dumps(self._get_general_config_in_local_format(), indent=' ')) + self._active_connectors = config + + self._gateway.tb_client.client.send_attributes({'active_connectors': config}) + + def _handle_connector_configuration_update(self, config): + """ + Expected the following data structure: + { + "name": "Mqtt Broker Connector", + "type": "mqtt", + "configuration": "mqtt.json", + "logLevel": "INFO", + "key?type=>grpc": "auto", + "class?type=>custom": "", + "configurationJson": { + ... + } + } + """ + + LOG.debug('Processing connectors configuration update...') + try: - self.__new_connectors_configs = self.__new_connectors_configs if self.__new_connectors_configs else self.__gateway.connectors_configs - new_connectors_files = [] - for connector_type in self.__new_connectors_configs: - for connector_config_section in self.__new_connectors_configs[connector_type]: - for connector_file in connector_config_section["config"]: - connector_config = connector_config_section["config"][connector_file] - with open(self.__gateway.get_config_path() + connector_file, "w", encoding="UTF-8") as config_file: - dump(connector_config, config_file, sort_keys=True, indent=2) - new_connectors_files.append(connector_file) - LOG.debug("Saving new configuration for \"%s\" connector to file \"%s\"", connector_type, - connector_file) - break - self.__old_general_configuration_file["connectors"] = self.__new_general_configuration_file["connectors"] - for old_connector_type in self.__old_connectors_configs: - for old_connector_config_section in self.__old_connectors_configs[old_connector_type]: - for old_connector_file in old_connector_config_section["config"]: - if old_connector_file not in new_connectors_files: - remove(self.__gateway.get_config_path() + old_connector_file) - LOG.debug("Remove old configuration file \"%s\" for \"%s\" connector ", old_connector_file, - old_connector_type) + configuration = config['configuration'] + + found_connectors = list(filter(lambda item: item['name'] == config['name'], self.connectors_configuration)) + if not found_connectors: + connector_configuration = {'name': config['name'], 'type': config['type'], + 'configuration': configuration} + if config.get('key'): + connector_configuration['key'] = config['key'] + + if config.get('class'): + connector_configuration['class'] = config['class'] + + with open(self._gateway.get_config_path() + configuration, 'w') as file: + config['configurationJson'].update({'logLevel': config['logLevel'], 'name': config['name']}) + file.writelines(dumps(config['configurationJson'], indent=' ')) + + self.connectors_configuration.append(connector_configuration) + with open(self._gateway.get_config_path() + 'tb_gateway.json', 'w') as file: + file.writelines(dumps(self._get_general_config_in_local_format(), indent=' ')) + + self._gateway.load_connectors(self._get_general_config_in_local_format()) + self._gateway.connect_with_connectors() + else: + found_connector = found_connectors[0] + changed = False + + config_path = self._gateway.get_config_path() + configuration + if os.path.exists(config_path): + with open(config_path, 'r') as file: + conf = load(file) + config_hash = hash(str(conf)) + + if config_hash != hash(str(config['configurationJson'])): + changed = True + + connector_configuration = None + if found_connector.get('name') != config['name'] or found_connector.get('type') != config[ + 'type'] or found_connector.get('class') != config.get('class') or found_connector.get( + 'key') != config.get('key') or found_connector.get('configurationJson', {}).get( + 'logLevel') != config.get('logLevel'): + changed = True + connector_configuration = {'name': config['name'], 'type': config['type'], + 'configuration': configuration} + + if config.get('key'): + connector_configuration['key'] = config['key'] + + if config.get('class'): + connector_configuration['class'] = config['class'] + + found_connector.update(connector_configuration) + + if changed: + with open(self._gateway.get_config_path() + configuration, 'w') as file: + config['configurationJson'].update({'logLevel': config['logLevel'], 'name': config['name']}) + file.writelines(dumps(config['configurationJson'], indent=' ')) + + if connector_configuration is None: + connector_configuration = found_connector + + self._gateway.available_connectors[connector_configuration['name']].close() + self._gateway.load_connectors(self._get_general_config_in_local_format()) + self._gateway.connect_with_connectors() + + self._gateway.tb_client.client.send_attributes({config['name']: config}) except Exception as e: LOG.exception(e) - def __safe_apply_connection_configuration(self): + def _handle_remote_logging_level_update(self, config): + self._gateway.tb_client.client.send_attributes({'RemoteLoggingLevel': config}) + + # HANDLERS SUPPORT METHODS ----------------------------------------------------------------------------------------- + def _apply_connection_config(self, config) -> bool: apply_start = time() * 1000 - self.__old_tb_client = self.__gateway.tb_client + old_tb_client = self._gateway.tb_client try: - self.__old_tb_client.unsubscribe('*') - self.__old_tb_client.stop() - self.__old_tb_client.disconnect() - self.__gateway.tb_client = TBClient(self.__new_general_configuration_file["thingsboard"], self.__old_tb_client.get_config_folder_path()) - self.__gateway.tb_client.connect() + old_tb_client.disconnect() + + new_tb_client = TBClient(config, old_tb_client.get_config_folder_path()) + connection_state = False - while time() * 1000 - apply_start < self.__apply_timeout * 1000 and not connection_state: - connection_state = self.__gateway.tb_client.is_connected() - sleep(.1) - if not connection_state: - self.__revert_configuration() - LOG.info("The gateway cannot connect to the ThingsBoard server with a new configuration.") - return False - else: - self.__old_tb_client.stop() - self.__gateway.subscribe_to_required_topics() - return True + while not connection_state: + for client in (new_tb_client, old_tb_client): + client.connect() + while time() * 1000 - apply_start >= 1000 and not connection_state: + connection_state = client.is_connected() + sleep(.1) + + if connection_state: + self._gateway.tb_client = client + self._gateway.subscribe_to_required_topics() + return True except Exception as e: LOG.exception(e) - self.__revert_configuration() + self._revert_connection() return False - def __apply_storage_configuration(self): - if self.__old_general_configuration_file["storage"] != self.__new_general_configuration_file["storage"]: - self.__old_event_storage = self.__gateway._event_storage - try: - storage_class = self.__gateway._event_storage_types[self.__new_general_configuration_file["storage"]["type"]] - self.__gateway._event_storage = storage_class(self.__new_general_configuration_file["storage"]) - self.__old_event_storage = None - except Exception as e: - LOG.exception(e) - self.__gateway._event_storage = self.__old_event_storage - - def __revert_configuration(self): + def _revert_connection(self): try: LOG.info("Remote general configuration will be restored.") - self.__new_general_configuration_file = self.__old_general_configuration_file - self.__gateway.tb_client.disconnect() - self.__gateway.tb_client.stop() - self.__gateway.tb_client = TBClient(self.__old_general_configuration_file["thingsboard"]) - self.__gateway.tb_client.connect() - self.__gateway.subscribe_to_required_topics() - LOG.debug("%s connection has been restored", str(self.__gateway.tb_client.client._client)) + self._gateway.tb_client.disconnect() + self._gateway.tb_client.stop() + self._gateway.tb_client = TBClient(self.general_configuration, self._gateway.get_config_path()) + self._gateway.tb_client.connect() + self._gateway.subscribe_to_required_topics() + LOG.debug("%s connection has been restored", str(self._gateway.tb_client.client)) except Exception as e: LOG.exception("Exception on reverting configuration occurred:") LOG.exception(e) - def __get_current_logs_configuration(self): + def _apply_statistics_config(self, config) -> bool: try: - with open(self.__gateway.get_config_path() + 'logs.conf', 'r', encoding="UTF-8") as logs: - current_logs_configuration = logs.read() - return current_logs_configuration + commands = config.get('commands', []) + if commands: + statistics_conf_file_name = self.general_configuration['statistics'].get('configuration', + 'statistics.json') + if statistics_conf_file_name is None: + statistics_conf_file_name = 'statistics.json' + + with open(self._gateway.get_config_path() + statistics_conf_file_name, 'w') as file: + file.writelines(dumps(commands, indent=' ')) + config['configuration'] = statistics_conf_file_name + + self._gateway.init_statistics_service(config) + self.general_configuration['statistics'] = config + return True except Exception as e: + LOG.error('Something went wrong with applying the new statistics configuration. Reverting...') LOG.exception(e) + self._gateway.init_statistics_service( + self.general_configuration.get('statistics', {'enable': True, 'statsSendPeriodInSeconds': 3600})) + return False - def __update_logs_configuration(self): - global LOG + def _apply_device_filtering_config(self, config): try: - LOG = getLogger('service') - logs_conf_file_path = self.__gateway.get_config_path() + 'logs.conf' - new_logging_level = findall(r'level=(.*)', self.__new_logs_configuration.replace("NONE", "NOTSET"))[-1] - new_logging_config = self.__new_logs_configuration.replace("NONE", "NOTSET").replace("\r\n", linesep) - logs_config = ConfigParser(allow_no_value=True) - logs_config.read_string(new_logging_config) - for section in logs_config: - if "handler_" in section and section != "handler_consoleHandler": - args = tuple(logs_config[section]["args"] - .replace('(', '') - .replace(')', '') - .split(', ')) - path = args[0][1:-1] - LOG.debug("Checking %s...", path) - if not exists(dirname(path)): - raise FileNotFoundError - with open(logs_conf_file_path, 'w', encoding="UTF-8") as logs: - logs.write(self.__new_logs_configuration.replace("NONE", "NOTSET") + "\r\n") - fileConfig(logs_config) - LOG = getLogger('service') - # self.__gateway.remote_handler.deactivate() - self.__gateway.remote_handler = TBLoggerHandler(self.__gateway) - self.__gateway.main_handler.setLevel(new_logging_level) - self.__gateway.main_handler.setTarget(self.__gateway.remote_handler) - if new_logging_level == "NOTSET": - self.__gateway.remote_handler.deactivate() - else: - self.__gateway.remote_handler.activate(new_logging_level) - LOG.debug("Logs configuration has been updated.") + self._gateway.init_device_filtering(config.get('deviceFiltering', {'enable': False})) + return True except Exception as e: - LOG.error("Remote logging configuration is wrong!") + LOG.error('Something went wrong with applying the new device filtering configuration. Reverting...') LOG.exception(e) + self._gateway.init_device_filtering( + self.general_configuration.get('deviceFiltering', {'enable': False})) + return False + + def _apply_remote_shell_config(self, config): + try: + self._gateway.init_remote_shell(config.get('remoteShell')) + return True + except Exception as e: + LOG.error('Something went wrong with applying the new Remote Shell configuration. Reverting...') + LOG.exception(e) + self._gateway.init_remote_shell(self.general_configuration.get('remoteShell')) + return False + + def _apply_other_params_config(self, config): + self._gateway.config['thingsboard'].update(config) + + def _delete_connectors_from_config(self, connector_list): + self._config['connectors'] = list( + filter(lambda connector: connector['name'] in connector_list, self.connectors_configuration)) + + def _check_statistics_configuration_changes(self, config): + general_statistics_config = self.general_configuration.get('statistics', self.DEFAULT_STATISTICS) + if config['enable'] != general_statistics_config['enable'] or config['statsSendPeriodInSeconds'] != \ + general_statistics_config['statsSendPeriodInSeconds']: + return True + + commands = [] + if general_statistics_config.get('configuration'): + with open(self._gateway.get_config_path() + general_statistics_config['configuration'], 'r') as file: + commands = load(file) + + if config.get('commands', []) != commands: + return True + + return False + + def _cleanup(self): + self.general_configuration['statistics'].pop('commands') + + def _is_modified(self, attr_name, config): + try: + file_path = config.get('configuration') or self._modifiable_static_attrs.get(attr_name) + except AttributeError: + file_path = None + + # if there is no file path that means that it is RemoteLoggingLevel or active_connectors attribute update + # in this case, we have to update the configuration without TS compare + if file_path is None: + return True + + try: + file_path = self._gateway.get_config_path() + file_path + if config.get('ts', 0) <= int(os.path.getmtime(file_path) * 1000): + return False + except OSError: + LOG.warning('File %s not exist', file_path) + + return True diff --git a/thingsboard_gateway/tb_utility/tb_handler.py b/thingsboard_gateway/tb_utility/tb_handler.py new file mode 100644 index 000000000..02061a33b --- /dev/null +++ b/thingsboard_gateway/tb_utility/tb_handler.py @@ -0,0 +1,108 @@ +# Copyright 2023. ThingsBoard +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import logging.handlers +from sys import stdout +from time import time +from os import environ + +from thingsboard_gateway.tb_utility.tb_logger import TbLogger + + +class TBLoggerHandler(logging.Handler): + LOGGER_NAME_TO_ATTRIBUTE_NAME = { + 'service': 'SERVICE_LOGS', + 'extension': 'EXTENSION_LOGS', + 'tb_connection': 'CONNECTION_LOGS', + 'storage': 'STORAGE_LOGS', + } + + def __init__(self, gateway): + self.current_log_level = 'INFO' + super().__init__(logging.getLevelName(self.current_log_level)) + self.setLevel(logging.getLevelName('DEBUG')) + self.__gateway = gateway + self.activated = False + self.setFormatter(logging.Formatter('%(asctime)s - |%(levelname)s| - [%(filename)s] - %(module)s - %(lineno)d - %(message)s')) + self.loggers = ['service', + 'extension', + 'tb_connection', + 'storage' + ] + for logger in self.loggers: + log = TbLogger(name=logger, gateway=gateway) + log.addHandler(self.__gateway.main_handler) + log.debug("Added remote handler to log %s", logger) + + def add_logger(self, name): + log = TbLogger(name) + log.addHandler(self.__gateway.main_handler) + log.debug("Added remote handler to log %s", name) + + def activate(self, log_level=None): + try: + for logger in self.loggers: + if log_level is not None and logging.getLevelName(log_level) is not None: + if logger == 'tb_connection' and log_level == 'DEBUG': + log = TbLogger(logger, gateway=self.__gateway) + log.setLevel(logging.getLevelName('INFO')) + else: + log = TbLogger(logger, gateway=self.__gateway) + self.current_log_level = log_level + log.setLevel(logging.getLevelName(log_level)) + except Exception as e: + log = TbLogger('service') + log.exception(e) + self.activated = True + + def handle(self, record): + if self.activated and not self.__gateway.stopped: + name = record.name + record = self.formatter.format(record) + try: + telemetry_key = self.LOGGER_NAME_TO_ATTRIBUTE_NAME[name] + except KeyError: + telemetry_key = name + '_LOGS' + + self.__gateway.tb_client.client.send_telemetry( + {'ts': int(time() * 1000), 'values': {telemetry_key: record, 'LOGS': record}}) + + def deactivate(self): + self.activated = False + + @staticmethod + def set_default_handler(): + logger_names = [ + 'service', + 'storage', + 'extension', + 'tb_connection' + ] + for logger_name in logger_names: + logger = TbLogger(logger_name) + handler = logging.StreamHandler(stdout) + handler.setFormatter(logging.Formatter('[STREAM ONLY] %(asctime)s - %(levelname)s - [%(filename)s] - %(module)s - %(lineno)d - %(message)s')) + logger.addHandler(handler) + + +class TimedRotatingFileHandler(logging.handlers.TimedRotatingFileHandler): + def __init__(self, filename, when='h', interval=1, backupCount=0, + encoding=None, delay=False, utc=False): + config_path = environ.get('logs') + if config_path: + filename = config_path + '/' + filename.split('/')[-1] + + super().__init__(filename, when=when, interval=interval, backupCount=backupCount, + encoding=encoding, delay=delay, utc=utc) diff --git a/thingsboard_gateway/tb_utility/tb_logger.py b/thingsboard_gateway/tb_utility/tb_logger.py index 98eabea2f..4fb2e56ad 100644 --- a/thingsboard_gateway/tb_utility/tb_logger.py +++ b/thingsboard_gateway/tb_utility/tb_logger.py @@ -1,4 +1,4 @@ -# Copyright 2022. ThingsBoard +# Copyright 2023. ThingsBoard # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,78 +13,102 @@ # limitations under the License. import logging -import logging.handlers -from sys import stdout -from time import time -from os import environ - - -class TBLoggerHandler(logging.Handler): - def __init__(self, gateway): - self.current_log_level = 'INFO' - super().__init__(logging.getLevelName(self.current_log_level)) - self.setLevel(logging.getLevelName('DEBUG')) - self.__gateway = gateway - self.activated = False - self.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - [%(filename)s] - %(module)s - %(lineno)d - %(message)s')) - self.loggers = ['service', - 'extension', - 'converter', - 'connector', - 'tb_connection' - ] - for logger in self.loggers: - log = logging.getLogger(logger) - log.addHandler(self.__gateway.main_handler) - log.debug("Added remote handler to log %s", logger) - - def activate(self, log_level=None): - try: - for logger in self.loggers: - if log_level is not None and logging.getLevelName(log_level) is not None: - if logger == 'tb_connection' and log_level == 'DEBUG': - log = logging.getLogger(logger) - log.setLevel(logging.getLevelName('INFO')) - else: - log = logging.getLogger(logger) - self.current_log_level = log_level - log.setLevel(logging.getLevelName(log_level)) - except Exception as e: - log = logging.getLogger('service') - log.exception(e) - self.activated = True - - def handle(self, record): - if self.activated and not self.__gateway.stopped: - record = self.formatter.format(record) - self.__gateway.send_to_storage(self.__gateway.name, {"deviceName": self.__gateway.name, "telemetry": [{"ts": int(time()*1000), "values":{'LOGS': record}}]}) - - def deactivate(self): - self.activated = False - - @staticmethod - def set_default_handler(): - logger_names = [ - 'service', - 'storage', - 'extension', - 'converter', - 'connector', - 'tb_connection' - ] - for logger_name in logger_names: - logger = logging.getLogger(logger_name) - handler = logging.StreamHandler(stdout) - handler.setFormatter(logging.Formatter('[STREAM ONLY] %(asctime)s - %(levelname)s - [%(filename)s] - %(module)s - %(lineno)d - %(message)s')) - logger.addHandler(handler) - - -class TimedRotatingFileHandler(logging.handlers.TimedRotatingFileHandler): - def __init__(self, filename, when='h', interval=1, backupCount=0, - encoding=None, delay=False, utc=False): - config_path = environ.get('logs') - if config_path: - filename = config_path + '/' + filename.split('/')[-1] - - super().__init__(filename, when=when, interval=interval, backupCount=backupCount, - encoding=encoding, delay=delay, utc=utc) +from time import sleep +from threading import Thread + + +def init_logger(gateway, name, level): + """ + For creating a Logger with all config automatically + Create a Logger manually only if you know what you are doing! + """ + log = TbLogger(name=name, gateway=gateway) + + if hasattr(gateway, 'remote_handler'): + log.addHandler(gateway.remote_handler) + log.setLevel(gateway.main_handler.level) + gateway.remote_handler.add_logger(name) + + if hasattr(gateway, 'main_handler'): + log.addHandler(gateway.main_handler) + log.setLevel(gateway.remote_handler.level) + + log_level_conf = level + if log_level_conf: + log_level = logging.getLevelName(log_level_conf) + log.setLevel(log_level) + + return log + + +class TbLogger(logging.Logger): + ALL_ERRORS_COUNT = 0 + IS_ALL_ERRORS_COUNT_RESET = False + + def __init__(self, name, gateway=None, level=logging.NOTSET): + super(TbLogger, self).__init__(name=name, level=level) + self.propagate = True + self.parent = self.root + self._gateway = gateway + self.errors = 0 + self.attr_name = self.name + '_ERRORS_COUNT' + self._is_on_init_state = True + self._errors_sender_thread = Thread(name='Log Errors Sender', daemon=True, target=self._send_errors) + self._errors_sender_thread.start() + + def reset(self): + """ + !!!Need to be called manually in the connector 'close' method!!! + """ + TbLogger.ALL_ERRORS_COUNT = TbLogger.ALL_ERRORS_COUNT - self.errors + + @property + def gateway(self): + return self._gateway + + @gateway.setter + def gateway(self, gateway): + self._gateway = gateway + + def _send_errors(self): + is_tb_client = False + + while not self._gateway: + sleep(1) + + while not is_tb_client: + is_tb_client = hasattr(self._gateway, 'tb_client') + sleep(1) + + if not TbLogger.IS_ALL_ERRORS_COUNT_RESET and self._gateway.tb_client.is_connected(): + self._gateway.tb_client.client.send_telemetry( + {self.attr_name: 0, 'ALL_ERRORS_COUNT': 0}, quality_of_service=0) + TbLogger.IS_ALL_ERRORS_COUNT_RESET = True + self._is_on_init_state = False + + def error(self, msg, *args, **kwargs): + kwargs['stacklevel'] = 2 + super(TbLogger, self).error(msg, *args, **kwargs) + self._send_error_count() + + def exception(self, msg, *args, **kwargs) -> None: + attr_name = kwargs.pop('attr_name', None) + kwargs['stacklevel'] = 2 + super(TbLogger, self).exception(msg, *args, **kwargs) + self._send_error_count(error_attr_name=attr_name) + + def _send_error_count(self, error_attr_name=None): + TbLogger.ALL_ERRORS_COUNT += 1 + self.errors += 1 + + while self._is_on_init_state: + sleep(.2) + + if self._gateway and hasattr(self._gateway, 'tb_client'): + if error_attr_name: + error_attr_name = error_attr_name + '_ERRORS_COUNT' + else: + error_attr_name = self.attr_name + + self._gateway.tb_client.client.send_telemetry( + {error_attr_name: self.errors, 'ALL_ERRORS_COUNT': TbLogger.ALL_ERRORS_COUNT})