From 818c702a791f983506eb7d2c038e2f930b0b6299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Mon, 9 Sep 2024 11:59:43 +0000 Subject: [PATCH 1/5] feat(qix performance): Add fine-grained performance monitoring for app objects Implements #320 --- src/config/production_template.yaml | 82 +++++ src/globals.js | 1 - src/lib/config-file-schema.js | 63 ++++ src/lib/config-file-verify.js | 1 + src/lib/post-to-influxdb.js | 90 ++++-- src/lib/post-to-new-relic.js | 2 +- src/lib/udp_handlers_log_events.js | 473 +++++++++++++++++++++++++++- 7 files changed, 684 insertions(+), 28 deletions(-) diff --git a/src/config/production_template.yaml b/src/config/production_template.yaml index 82e67130..4d61e47e 100644 --- a/src/config/production_template.yaml +++ b/src/config/production_template.yaml @@ -205,6 +205,88 @@ Butler-SOS: category: - name: qs_log_category value: unknown + appPerformanceMonitor: # Detailed app performance data extraction from log events + enable: false # Should app performance data be extracted from log events? + appNameLookup: # Should app names be looked up based on app IDs? + enable: true + monitorFilter: # What objects should be monitored? Entire apps or just specific object(s) within some specific app(s)? + # Two kinds of monitoring can be done: + # 1) Monitor all apps, except those listed for exclusion. This is defined in the allApps section. + # 2) Monitor only specific apps. This is defined in the appSpecific section. + # An event will be accepted if it matches any of the rules in the allApps section OR any of the rules in the appSpecific section. + allApps: + enable: false # Should all apps be monitored? + appExclude: # What apps should be excluded from monitoring? + # If both appId and appName are specified, both must match the event's data for it to be considered a match. + - appId: 5b817efe-472d-43ce-8a31-6cce34af7de9 + - appName: Sales forecast + - appId: f42d6b16-8faf-45ca-a783-59f9da47db6e + appName: Inventory analysis + objectType: + allObjectTypes: true # Should all object types be monitored? + allObjectTypesExclude: # If allObjectTypes is set to true, the object types in this array are excluded from monitoring. + # someObjectTypesInclude (below) is ignored in that case. + - LoadModelList + - + - linechart + - map + someObjectTypesInclude: # What object types should be included in monitoring? + # Only applicable if allObjectTypes is set to false. + - LoadModelList + - sheet + - barchart + method: + allMethods: true # Should all methods be monitored? + allMethodsExclude: # If allMethods is set to true, the methods in this array are excluded from monitoring. + # someMethodsInclude (below) is ignored in that case. + - Global::OpenApp + - Doc::GetAppLayout + - Doc::CreateSessionObject + someMethodsInclude: # What methods should be included in monitoring? + # Only applicable if allMethods is set to false. + - GenericObject::GetLayout + - GenericObject::GetHyperCubeContinuousData + appSpecific: + enable: false # Should app specific monitoring be done? + app: + - include: # What apps should be monitored? + # If both appId and appName are specified, both must match the event's data for it to be considered a match. + - appId: d7cf16f9-6a95-462a-9ff1-a6d413326de4 + - appName: Budget 2025 + - appId: 6931136d-c234-4358-a40c-e37153aba7c9 + appName: Sales basket analysis + objectType: + allObjectTypes: true # Should all object types be monitored? + allObjectTypesExclude: # If allObjectTypes is set to true, the object types in this array are excluded from monitoring. + # someObjectTypesInclude (below) is ignored in that case. + - table + - map + someObjectTypesInclude: # What object types should be included in monitoring? + # Only applicable if allObjectTypes is set to false. + - sheet + - barchart + - linechart + - map + appObject: + allAppObjects: true # Should all app objects be monitored? + allAppObjectsExclude: # If allAppObjects is set to true, the app objects in this array are excluded from monitoring. + # someAppObjectsInclude (below) is ignored in that case. + - objectId: AaBbCc + - objectId: DdEeFf + someAppObjectsInclude: # What app objects should be included in monitoring? + # Only applicable if allAppObjects is set to false. + - objectId: YJEpPT + method: + allMethods: true # Should all methods be monitored? + allMethodsExclude: # If allMethods is set to true, the methods in this array are excluded from monitoring. + # someMethodsInclude (below) is ignored in that case. + - Global::OpenApp + - Doc::GetAppLayout + - Doc::CreateSessionObject + someMethodsInclude: # What methods should be included in monitoring? + # Only applicable if allMethods is set to false. + - GenericObject::GetLayout + - GenericObject::GetHyperCubeContinuousData sendToMQTT: enable: false # Should log events be sent as MQTT messages? baseTopic: qliksense/logevent # What topic should log events be forwarded to? diff --git a/src/globals.js b/src/globals.js index 800139c6..10977339 100755 --- a/src/globals.js +++ b/src/globals.js @@ -694,7 +694,6 @@ class Settings { this.logger.verbose('GLOBALS: Init done'); - // eslint-disable-next-line no-constructor-return return instance; } diff --git a/src/lib/config-file-schema.js b/src/lib/config-file-schema.js index 82203cf9..6604c945 100755 --- a/src/lib/config-file-schema.js +++ b/src/lib/config-file-schema.js @@ -173,6 +173,69 @@ export const confifgFileSchema = { ], }, }, + appPerformanceMonitor: { + enable: 'boolean', + appNameLookup: { + enable: 'boolean', + }, + monitorFilter: { + allApps: { + enable: 'boolean', + 'appExclude?': [ + { + 'appId?': 'string', + 'appName?': 'string', + }, + ], + objectType: { + allObjectTypes: 'boolean', + 'allObjectTypesExclude?': [], + 'someObjectTypesInclude?': [], + }, + method: { + allMethods: 'boolean', + 'allMethodsExclude?': [], + 'someMethodsInclude?': [], + }, + }, + appSpecific: { + enable: 'boolean', + app: [ + { + 'include?': [ + { + 'appId?': 'string', + 'appName?': 'string', + }, + ], + objectType: { + allObjectTypes: 'boolean', + 'allObjectTypesExclude?': [], + 'someObjectTypesInclude?': [], + }, + appObject: { + allAppObjects: 'boolean', + 'allAppObjectsExclude?': [ + { + objectId: 'string', + }, + ], + 'someAppObjectsInclude?': [ + { + objectId: 'string', + }, + ], + }, + method: { + allMethods: 'boolean', + 'allMethodsExclude?': [], + 'someMethodsInclude?': [], + }, + }, + ], + }, + }, + }, sendToMQTT: { enable: 'boolean', baseTopic: 'string', diff --git a/src/lib/config-file-verify.js b/src/lib/config-file-verify.js index 04267d25..e9fea22b 100755 --- a/src/lib/config-file-verify.js +++ b/src/lib/config-file-verify.js @@ -42,6 +42,7 @@ export async function verifyConfigFile() { process.exit(1); } + // ------------------------------ // Verify values of specific config entries // If InfluxDB is enabled, check if the version is valid diff --git a/src/lib/post-to-influxdb.js b/src/lib/post-to-influxdb.js index c0cf49a8..981c8b98 100755 --- a/src/lib/post-to-influxdb.js +++ b/src/lib/post-to-influxdb.js @@ -1,6 +1,3 @@ -/* eslint-disable prefer-destructuring */ -/* eslint-disable no-unused-vars */ - import { Point } from '@influxdata/influxdb-client'; import globals from '../globals.js'; @@ -72,7 +69,6 @@ export async function postHealthMetricsToInfluxdb(serverName, host, body, server const sessionAppNamesActive = []; const storeActivedDoc = function storeActivedDoc(docID) { - // eslint-disable-next-line no-unused-vars return new Promise((resolve, _reject) => { if (docID.substring(0, sessionAppPrefix.length) === sessionAppPrefix) { // Session app @@ -95,10 +91,8 @@ export async function postHealthMetricsToInfluxdb(serverName, host, body, server }); }; - // eslint-disable-next-line no-unused-vars const promisesActive = body.apps.active_docs.map( (docID, _idx) => - // eslint-disable-next-line no-unused-vars new Promise(async (resolve, _reject) => { await storeActivedDoc(docID); @@ -117,7 +111,6 @@ export async function postHealthMetricsToInfluxdb(serverName, host, body, server const sessionAppNamesLoaded = []; const storeLoadedDoc = function storeLoadedDoc(docID) { - // eslint-disable-next-line no-unused-vars return new Promise((resolve, _reject) => { if (docID.substring(0, sessionAppPrefix.length) === sessionAppPrefix) { // Session app @@ -140,10 +133,8 @@ export async function postHealthMetricsToInfluxdb(serverName, host, body, server }); }; - // eslint-disable-next-line no-unused-vars const promisesLoaded = body.apps.loaded_docs.map( (docID, _idx) => - // eslint-disable-next-line no-unused-vars new Promise(async (resolve, _reject) => { await storeLoadedDoc(docID); @@ -162,7 +153,6 @@ export async function postHealthMetricsToInfluxdb(serverName, host, body, server const sessionAppNamesInMemory = []; const storeInMemoryDoc = function storeInMemoryDoc(docID) { - // eslint-disable-next-line no-unused-vars return new Promise((resolve, _reject) => { if (docID.substring(0, sessionAppPrefix.length) === sessionAppPrefix) { // Session app @@ -189,10 +179,8 @@ export async function postHealthMetricsToInfluxdb(serverName, host, body, server }); }; - // eslint-disable-next-line no-unused-vars const promisesInMemory = body.apps.in_memory_docs.map( (docID, _idx) => - // eslint-disable-next-line no-unused-vars new Promise(async (resolve, _reject) => { await storeInMemoryDoc(docID); @@ -723,7 +711,6 @@ export async function postUserEventToInfluxdb(msg) { globals.config.get('Butler-SOS.userEvents.tags').length > 0 ) { const configTags = globals.config.get('Butler-SOS.userEvents.tags'); - // eslint-disable-next-line no-restricted-syntax for (const item of configTags) { tags[item.tag] = item.value; } @@ -836,7 +823,6 @@ export async function postUserEventToInfluxdb(msg) { globals.config.get('Butler-SOS.userEvents.tags').length > 0 ) { const configTags = globals.config.get('Butler-SOS.userEvents.tags'); - // eslint-disable-next-line no-restricted-syntax for (const item of configTags) { point.tag(item.tag, item.value); } @@ -898,7 +884,8 @@ export async function postLogEventToInfluxdb(msg) { msg.source === 'qseow-engine' || msg.source === 'qseow-proxy' || msg.source === 'qseow-scheduler' || - msg.source === 'qseow-repository' + msg.source === 'qseow-repository' || + msg.source === 'qseow-qix-perf' ) { if (msg.source === 'qseow-engine') { tags = { @@ -908,6 +895,7 @@ export async function postLogEventToInfluxdb(msg) { log_row: msg.log_row, subsystem: msg.subsystem, }; + // Tags that are empty in some cases. Only add if they are non-empty if (msg?.user_full?.length > 0) tags.user_full = msg.user_full; if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory; @@ -939,6 +927,7 @@ export async function postLogEventToInfluxdb(msg) { log_row: msg.log_row, subsystem: msg.subsystem, }; + // Tags that are empty in some cases. Only add if they are non-empty if (msg?.user_full?.length > 0) tags.user_full = msg.user_full; if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory; @@ -962,6 +951,7 @@ export async function postLogEventToInfluxdb(msg) { log_row: msg.log_row, subsystem: msg.subsystem, }; + // Tags that are empty in some cases. Only add if they are non-empty if (msg?.user_full?.length > 0) tags.user_full = msg.user_full; if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory; @@ -985,6 +975,7 @@ export async function postLogEventToInfluxdb(msg) { log_row: msg.log_row, subsystem: msg.subsystem, }; + // Tags that are empty in some cases. Only add if they are non-empty if (msg?.user_full?.length > 0) tags.user_full = msg.user_full; if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory; @@ -1000,6 +991,39 @@ export async function postLogEventToInfluxdb(msg) { context: msg.context, raw_event: JSON.stringify(msg), }; + } else if (msg.source === 'qseow-qix-perf') { + tags = { + host: msg.host, + level: msg.level, + source: msg.source, + log_row: msg.log_row, + subsystem: msg.subsystem, + method: msg.method, + object_type: msg.object_type, + proxy_session_id: msg.proxy_session_id, + session_id: msg.session_id, + }; + + // Tags that are empty in some cases. Only add if they are non-empty + if (msg?.user_full?.length > 0) tags.user_full = msg.user_full; + if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory; + if (msg?.user_id?.length > 0) tags.user_id = msg.user_id; + if (msg?.app_id?.length > 0) tags.app_id = msg.app_id; + if (msg?.app_name?.length > 0) tags.app_name = msg.app_name; + if (msg?.object_id?.length > 0) tags.object_id = msg.object_id; + + fields = { + app_id: msg.app_id, + process_time: msg.process_time, + work_time: msg.work_time, + lock_time: msg.lock_time, + validate_time: msg.validate_time, + traverse_time: msg.traverse_time, + handle: msg.handle, + net_ram: msg.net_ram, + peak_ram: msg.peak_ram, + raw_event: JSON.stringify(msg), + }; } // Add log event categories to tags if available @@ -1017,7 +1041,6 @@ export async function postLogEventToInfluxdb(msg) { globals.config.get('Butler-SOS.logEvents.tags').length > 0 ) { const configTags = globals.config.get('Butler-SOS.logEvents.tags'); - // eslint-disable-next-line no-restricted-syntax for (const item of configTags) { tags[item.tag] = item.value; } @@ -1056,7 +1079,8 @@ export async function postLogEventToInfluxdb(msg) { msg.source === 'qseow-engine' || msg.source === 'qseow-proxy' || msg.source === 'qseow-scheduler' || - msg.source === 'qseow-repository' + msg.source === 'qseow-repository' || + msg.source === 'qseow-qix-perf' ) { // Create new write API object // Advanced write options @@ -1205,6 +1229,37 @@ export async function postLogEventToInfluxdb(msg) { point.tag('user_directory', msg.user_directory); if (msg?.user_id?.length > 0) point.tag('user_id', msg.user_id); if (msg?.result_code?.length > 0) point.tag('result_code', msg.result_code); + } else if (msg.source === 'qseow-qix-perf') { + // Create a new point with the data to be written to InfluxDB + point = new Point('log_event') + .tag('host', msg.host) + .tag('level', msg.level) + .tag('source', msg.source) + .tag('log_row', msg.log_row) + .tag('subsystem', msg.subsystem) + .tag('method', msg.method) + .tag('object_type', msg.object_type) + .tag('proxy_session_id', msg.proxy_session_id) + .tag('session_id', msg.session_id) + .stringField('app_id', msg.app_id) + .floatField('process_time', parseFloat(msg.process_time)) + .floatField('work_time', parseFloat(msg.work_time)) + .floatField('lock_time', parseFloat(msg.lock_time)) + .floatField('validate_time', parseFloat(msg.validate_time)) + .floatField('traverse_time', parseFloat(msg.traverse_time)) + .stringField('handle', msg.handle) + .intField('net_ram', parseInt(msg.net_ram)) + .intField('peak_ram', parseInt(msg.peak_ram)) + .stringField('raw_event', JSON.stringify(msg)); + + // Tags that are empty in some cases. Only add if they are non-empty + if (msg?.user_full?.length > 0) point.tag('user_full', msg.user_full); + if (msg?.user_directory?.length > 0) + point.tag('user_directory', msg.user_directory); + if (msg?.user_id?.length > 0) point.tag('user_id', msg.user_id); + if (msg?.app_id?.length > 0) point.tag('app_id', msg.app_id); + if (msg?.app_name?.length > 0) point.tag('app_name', msg.app_name); + if (msg?.object_id?.length > 0) point.tag('object_id', msg.object_id); } // Add log event categories to tags if available @@ -1222,7 +1277,6 @@ export async function postLogEventToInfluxdb(msg) { globals.config.get('Butler-SOS.logEvents.tags').length > 0 ) { const configTags = globals.config.get('Butler-SOS.logEvents.tags'); - // eslint-disable-next-line no-restricted-syntax for (const item of configTags) { point.tag(item.tag, item.value); } diff --git a/src/lib/post-to-new-relic.js b/src/lib/post-to-new-relic.js index 1cd2d703..6074e283 100755 --- a/src/lib/post-to-new-relic.js +++ b/src/lib/post-to-new-relic.js @@ -930,7 +930,7 @@ export async function postLogEventToNewRelic(msg) { globals.logger.debug(`LOG EVENT NEW RELIC: ${msg})`); try { - // Only send log events that are enabled in the confif file + // Only send log events that are enabled in the config file if (sendNRLogEventYesNo(msg.source, msg.level) === true) { // First prepare attributes relating to the actual log event, then add attributes defined in the config file // The config file attributes can for example be used to separate data from DEV/TEST/PROD environments diff --git a/src/lib/udp_handlers_log_events.js b/src/lib/udp_handlers_log_events.js index 2e3a9417..e3effd50 100644 --- a/src/lib/udp_handlers_log_events.js +++ b/src/lib/udp_handlers_log_events.js @@ -1,5 +1,3 @@ -/* eslint-disable no-unused-vars */ - // Load global variables and functions import globals from '../globals.js'; import { postLogEventToInfluxdb } from './post-to-influxdb.js'; @@ -82,13 +80,37 @@ export function udpInitLogEventServer() { // 16: App ID associated with the event. Ex: e7af59a0-c243-480d-9571-08727551a66f // 17: Execution ID associated with the event. Ex: 4831c6a5-34f6-45bb-9d40-73a6e6992670 + // >> Message parts for log messages with Qix performance information + // 0: Message type. Always /qseow-qix-perf/ + // 1: Row number. Ex: 14 + // 2: ISO8601 formatted timestamp. Example: 20211109T193744.331+0100 + // 3: Local timezone timestamp. Example: 2021-11-09 19:37:44,331 + // 4: Log level. Possible values are: WARN, ERROR, FATAL + // 5: Hostname where the log event occured + // 6: QSEoW subsystem where log event occured. Example: System.Scheduler.Scheduler.Slave.Tasks.ReloadTask + // 7: Windows username running the originating QSEoW service. Ex: COMPANYNAME\qlikservice + // 8: Proxy session ID. Ex: 3b3b3b3b-3b3b-3b3b-3b3b-3b3b3b3b3b3b + // 9: User directory of the user associated with the event. Ex: LAB + // 10: User ID of the user associated with the event. Ex: goran + // 11: Engine timestamp. Example: 2021-11-09T19:37:44.331+01:00 + // 12: Session ID. Ex: 3b3b3b3b-3b3b-3b3b-3b3b-3b3b3b3b3b3b + // 13: Document ID (=app ID). Ex: 3b3b3b3b-3b3b-3b3b-3b3b-3b3b3b3b3b3b + // 14: Request ID. Ex: 3b3b3b3b-3b3b-3b3b-3b3b-3b3b3b3b3b3b + // 15: Method. Ex: Global::OpenApp, Doc::GetAppLayout, GenericObject::GetLayout + // 16: Process time in milliseconds. Ex: 123 + // 17: Work time in milliseconds. Ex: 123 + // 18: Lock time in milliseconds. Ex: 123 + // 19: Validate time in milliseconds. Ex: 123 + // 20: Traverse time in milliseconds. Ex: 123 + // 21: Handle. Ex: -1, 123 + // 22: Object ID. Ex: df68e14d-1ed0-47c9-bcb6-b37a900441d8, , rwPjBk + // 23: Net RAM. Ex: 123456 bytes + // 24: Peak RAM. Ex: 123456 byets + // 25: Object type. Ex: , AppPropsList, SheetList, StoryList, VariableList, linechart, barchart, map, listbox, CurrentSelection + const msg = message.toString().split(';'); globals.logger.debug(`LOG EVENT (raw): ${message.toString()}`); - globals.logger.verbose( - `LOG EVENT: ${msg[0]}:${msg[5]}:${msg[4]}, ${msg[6]}: ${msg[8]}` - ); - // Check if the message is a log event message we recognise // If not, log a warning and return // Take into account that msg[0] may be undefined, so check for that first @@ -97,7 +119,8 @@ export function udpInitLogEventServer() { (msg[0].toLowerCase() !== '/qseow-engine/' && msg[0].toLowerCase() !== '/qseow-proxy/' && msg[0].toLowerCase() !== '/qseow-repository/' && - msg[0].toLowerCase() !== '/qseow-scheduler/') + msg[0].toLowerCase() !== '/qseow-scheduler/' && + msg[0].toLowerCase() !== '/qseow-qix-perf/') ) { // Show warning, include first 512 characters of the message const msgShort = message.toString().substring(0, 512); @@ -116,7 +139,9 @@ export function udpInitLogEventServer() { (globals.config.get('Butler-SOS.logEvents.source.repository.enable') === true && msg[0].toLowerCase() === '/qseow-repository/') || (globals.config.get('Butler-SOS.logEvents.source.scheduler.enable') === true && - msg[0].toLowerCase() === '/qseow-scheduler/') + msg[0].toLowerCase() === '/qseow-scheduler/') || + (globals.config.get('Butler-SOS.logEvents.source.qixPerf.enable') === true && + msg[0].toLowerCase() === '/qseow-qix-perf/') ) { // Clean up the first message field (=message source) // Remove leading and trailing / @@ -134,6 +159,10 @@ export function udpInitLogEventServer() { const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; if (msg[0] === 'qseow-engine') { + globals.logger.verbose( + `LOG EVENT: ${msg[0]}:${msg[5]}:${msg[4]}, ${msg[6]}, Msg: ${msg[8]}` + ); + // log_row: numeric // ts_iso: ISO8601 date // ts_local: ISO8601 date @@ -187,6 +216,10 @@ export function udpInitLogEventServer() { msgObj.user_full = ''; } } else if (msg[0] === 'qseow-proxy') { + globals.logger.verbose( + `LOG EVENT: ${msg[0]}:${msg[5]}:${msg[4]}, ${msg[6]}, Msg: ${msg[8]}` + ); + msgObj = { source: msg[0], log_row: Number.isInteger(parseInt(msg[1], 10)) ? parseInt(msg[1], 10) : -1, @@ -216,6 +249,10 @@ export function udpInitLogEventServer() { msgObj.user_full = ''; } } else if (msg[0] === 'qseow-scheduler') { + globals.logger.verbose( + `LOG EVENT: ${msg[0]}:${msg[5]}:${msg[4]}, ${msg[6]}, Msg: ${msg[8]}` + ); + msgObj = { source: msg[0], log_row: Number.isInteger(parseInt(msg[1], 10)) ? parseInt(msg[1], 10) : -1, @@ -258,6 +295,10 @@ export function udpInitLogEventServer() { msgObj.user_id = msgObj.user_full.split('\\')[1]; } } else if (msg[0] === 'qseow-repository') { + globals.logger.verbose( + `LOG EVENT: ${msg[0]}:${msg[5]}:${msg[4]}, ${msg[6]}, Msg: ${msg[8]}` + ); + msgObj = { source: msg[0], log_row: Number.isInteger(parseInt(msg[1], 10)) ? parseInt(msg[1], 10) : -1, @@ -277,6 +318,422 @@ export function udpInitLogEventServer() { context: msg[15], }; + // Different log events deliver QSEoW user directory/user differently. + // Create fields that are consistent across all log events + if (msgObj.user_directory !== '' && msgObj.user_id !== '') { + // User directory and user id available in separate fields. + // Combine them into a single field + msgObj.user_full = `${msgObj.user_directory}\\${msgObj.user_id}`; + } else { + msgObj.user_full = ''; + } + } else if (msg[0] === 'qseow-qix-perf') { + globals.logger.verbose( + `LOG EVENT: ${msg[0]}:${msg[5]}:${msg[4]}, ${msg[6]}, ${msg[9]}\\${msg[10]}, ${msg[13]}, ${msg[15]}, Object type: ${msg[25]}` + ); + + // Determine if the message should be handled, based on settings in the config file + if ( + globals.config.get('Butler-SOS.logEvents.appPerformanceMonitor.enable') === + false + ) { + globals.logger.debug( + 'LOG EVENT: Qix performance monitoring is disabled in the configuration. Skipping event.' + ); + return; + } + + // Does event match filters in the config file? + // + // There are two types of filters: + // - "All-apps" filters. Here we start with all apps and then exclude some based on filters + // - App specific filters. Here we start with an empty list of apps and then include some based on filters + // + // If an event matches any of the filters, it is accepted. + + // Get the app performance monitor filter configuration from the config file, + // so we don't have to read it every time we need some part of it + const monitorFilterConfig = globals.config.get( + 'Butler-SOS.logEvents.appPerformanceMonitor.monitorFilter' + ); + + let acceptEvent = false; + let acceptEventAppSpecific = true; + + const eventAppId = msg[13]; + let eventAppName = ''; + const eventObjectId = msg[22]; + const eventObjectType = msg[25]; + const eventMethod = msg[15]; + + // Should we get app name from the app ID? + if ( + globals.config.get( + 'Butler-SOS.logEvents.appPerformanceMonitor.appNameLookup.enable' + ) === true + ) { + // Get app name from app ID + const eventApp = globals.appNames.find((app) => app.id === eventAppId); + + if (eventApp?.name) { + eventAppName = eventApp.name; + } else { + eventAppName = ''; + } + } + + // -------------------------------------------------------- + // Check if data in event matches the app specific filters in the config file + // -------------------------------------------------------- + // monitorFilterConfig.appSpecific is an array of objects, each with following properties: + // - enable: boolean + // - app: Array of objects, each with following properties: + // - include: Array of objects, each of which has one or more of the following properties: + // - appId: string + // - appName: string + // - objectType: Object with following properties: + // - allObjectTypes: boolean + // - allObjectTypesExclude: array of strings + // - someObjectTypesInclude: array of strings + // - appObject: Object with following properties: + // - allAppObjects: boolean + // - allAppObjectsExclude: array of objects, each of which has one or more of the following properties: + // - objectId: string + // - someAppObjectsInclude: array of objects, each of which has one or more of the following properties: + // - objectId: string + // - method: Objects with following properties: + // - allMethods: boolean + // - allMethodsExclude: array of strings + // - someMethodsInclude: array of strings + // + // If the include array is empty, no apps will be accepted. + // + // If allObjectTypes is true, all object types are included, unless they are in allObjectTypesExclude. + // someObjectTypesInclude is ignored in this case. + // If allObjectTypes is false, only events matching the object types in someObjectTypesInclude are accepted. + // + // If allAppObjects is true, all objects are included, unless they are in allAppObjectsExclude. + // someAppObjectsInclude is ignored in this case. + // If allAppObjects is false, only events matching the objects in someAppObjectsInclude are accepted. + // + // If allMethods is true, all methods are included, unless they are in allMethodsExclude. + // someMethodsInclude is ignored in this case. + // If allMethods is false, only events matching the methods in someMethodsInclude are accepted. + // + + // Is app specific monitoring enabled? + if (monitorFilterConfig.appSpecific.enable === false) { + globals.logger.debug( + 'LOG EVENT: App specific monitoring is disabled in the configuration. Skipping app specific filters for this event.' + ); + acceptEventAppSpecific = false; + } + + if (acceptEventAppSpecific === true) { + // Process all app specific filters + // If one or more filter matches, the event is accepted + // If no filter matches, the event is rejected + for (const appFilter of monitorFilterConfig.appSpecific.app) { + // Check if the app ID is in the list of apps to monitor + // If not, skip the event + // The include array consists of objects with one or more of the following properties: + // - appId: string + // - appName: string + // If both appId and appName are present, both must match for the app to be included + const monitoredAppConfig = appFilter?.include?.find( + (appInclude) => + (appInclude?.appId === undefined || + appInclude.appId === eventAppId) && + (appInclude?.appName === undefined || + appInclude.appName === eventAppName) + ); + + if (monitoredAppConfig === undefined) { + // Event app ID does not match any app specific INCLUDE filters in the config file + acceptEventAppSpecific = false; + } else { + // App ID matches an app in the config file + if (appFilter.objectType.allObjectTypes === true) { + // Check if data in event matches the EXCLUDE object type filters in the config file + const excludedObjectType = + appFilter.objectType?.allObjectTypesExclude?.find( + (objectTypeExclude) => + objectTypeExclude === eventObjectType + ); + if (excludedObjectType !== undefined) { + // Object type matches an EXCLUDE object type in the config file + acceptEventAppSpecific = false; + } + } else { + // Check if data in event matches the INCLUDE object type filters in the config file + const monitoredObjectType = + appFilter.objectType?.someObjectTypesInclude?.find( + (objectTypeInclude) => + objectTypeInclude === eventObjectType + ); + if (monitoredObjectType === undefined) { + // Object type does not match an INCLUDE object type in the config file + acceptEventAppSpecific = false; + } else { + // Object type matches an INCLUDE object type in the config file + globals.logger.debug( + 'LOG EVENT: Qix performance event matches object type filters in the configuration' + ); + } + } + + // Only check object ID if the event has not been rejected so far + if (acceptEventAppSpecific === true) { + if (appFilter.appObject.allAppObjects === true) { + // Check if data in event matches the EXCLUDE object ID filters in the config file + const excludedAppObject = + appFilter.appObject?.allAppObjectsExclude?.find( + (appObjectExclude) => + appObjectExclude?.objectId === eventObjectId + ); + if (excludedAppObject !== undefined) { + // Object ID matches an EXCLUDE object ID in the config file + acceptEventAppSpecific = false; + } + } else { + // Check if data in event matches the INCLUDE object ID filters in the config file + const monitoredAppObject = + appFilter.appObject?.someAppObjectsInclude?.find( + (appObjectInclude) => + appObjectInclude?.objectId === eventObjectId + ); + if (monitoredAppObject === undefined) { + // Object ID does not match an INCLUDE object ID in the config file + acceptEventAppSpecific = false; + } + } + } + + // Only check methods if the event has not been rejected so far + if (acceptEventAppSpecific === true) { + if (appFilter.method.allMethods === true) { + // Check if data in event matches the EXCLUDE method filters in the config file + const excludedMethod = + appFilter.method?.allMethodsExclude?.find( + (methodExclude) => methodExclude === eventMethod + ); + if (excludedMethod !== undefined) { + // Method matches an EXCLUDE method in the config file + acceptEventAppSpecific = false; + } + } else { + // Check if data in event matches the INCLUDE method filters in the config file + const monitoredMethod = + appFilter.method?.someMethodsInclude?.find( + (methodInclude) => methodInclude === eventMethod + ); + if (monitoredMethod === undefined) { + // Method does not match an INCLUDE method in the config file + acceptEventAppSpecific = false; + } else { + // Method matches an INCLUDE method in the config file + globals.logger.debug( + 'LOG EVENT: Qix performance event matches method filters in the configuration' + ); + } + } + } + } + } + + // Done checking if event matches app-specific filters in the config file + + // Match on app specific filters? + if (acceptEventAppSpecific === true) { + globals.logger.debug( + 'LOG EVENT: Qix performance event matches app-specific filters in the configuration' + ); + } + } else { + acceptEventAppSpecific = false; + globals.logger.debug( + 'LOG EVENT: Qix performance event does not match app-specific filters in the configuration. Skipping app specific filters for this event.' + ); + } + + // -------------------------------------------------------- + // Check if data in event matches the "all apps" filters in the config file + // Only do this if the event has not been accepted so far + // -------------------------------------------------------- + // monitorFilterConfig.allApps is an array of objects, each with following properties: + // - enable: boolean + // - appExclude: array of objects, each of which has one or more of the following properties: + // - appId: string + // - appName: string + // - objectType: Objects with following properties: + // - allObjectTypes: boolean + // - allObjectTypesExclude: array of strings + // - someObjectTypesInclude: array of strings + // - method: Objects with following properties: + // - allMethods: boolean + // - allMethodsExclude: array of strings + // - someMethodsInclude: array of strings + // + // If appExclude is empty, all apps are included. + // If appExclude has objects, only apps not in appExclude are included. + // Matching is inclusive, i.e. if an object in the appExclude array has both appId and appName, both must match for the app to be excluded. + // + // If objectType.allObjectTypes is true, all object types are included, unless they are in allObjectTypesExclude. + // someObjectTypesInclude is ignored in this case. + // If objectType.allObjectTypes is false, only object types in someObjectTypesInclude are included. + // + // If method.allMethods is true, all methods are included, unless they are in allMethodsExclude. + // someMethodsInclude is ignored in this case. + // If method.allMethods is false, only methods in someMethodsInclude are included. + // + + let acceptEventAllApps = true; + + // Check if data in event matches the all-app filters in the config file + if (monitorFilterConfig.allApps.enable === false) { + globals.logger.debug( + 'LOG EVENT: All-apps monitoring is disabled in the configuration. Skipping all-app filters for this event.' + ); + acceptEventAllApps = false; + } else if ( + acceptEventAppSpecific === false && + monitorFilterConfig.allApps.enable === true + ) { + // Check if data in event matches the EXCLUDE app filters in the config file + if (monitorFilterConfig.allApps.appExclude?.length > 0) { + // Any matching appExclude object will cause the event to be rejected + // The appExclude array consists of objects with one or more of the following properties: + // - appId: string + // - appName: string + // If both appId and appName are present, both must match for the app to be excluded + const excludedApp = monitorFilterConfig.allApps?.appExclude.find( + (appExclude) => + (appExclude?.appId === undefined || + appExclude.appId === eventAppId) && + (appExclude?.appName === undefined || + appExclude.appName === eventAppName) + ); + if (excludedApp !== undefined) { + // App ID matches an app in the config file + acceptEventAllApps = false; + } + } + + // Only check object type if the event has not been rejected so far + if (acceptEventAllApps === true) { + if (monitorFilterConfig.allApps.objectType.allObjectTypes === true) { + // Check if data in event matches the EXCLUDE object type filters in the config file + const excludedObjectType = + monitorFilterConfig.allApps.objectType?.allObjectTypesExclude?.find( + (objectTypeExclude) => objectTypeExclude === eventObjectType + ); + if (excludedObjectType !== undefined) { + // Object type matches an object type in the config file + acceptEventAllApps = false; + } + } else { + // Check if data in event matches the INCLUDE object type filters in the config file + const monitoredObjectType = + monitorFilterConfig.allApps.objectType?.someObjectTypesInclude?.find( + (objectTypeInclude) => objectTypeInclude === eventObjectType + ); + if (monitoredObjectType === undefined) { + // Object type does not match an object type in the config file + acceptEventAllApps = false; + } + } + } + + // Only check methods if the event has not been rejected so far + if (acceptEventAllApps === true) { + if (monitorFilterConfig.allApps.method.allMethods === true) { + // Check if data in event matches the EXCLUDE method filters in the config file + const excludedMethod = + monitorFilterConfig.allApps.method?.allMethodsExclude?.find( + (methodExclude) => methodExclude === eventMethod + ); + if (excludedMethod !== undefined) { + // Method matches a method in the config file + acceptEventAllApps = false; + } + } else { + // Check if data in event matches the INCLUDE method filters in the config file + const monitoredMethod = + monitorFilterConfig.allApps.method?.someMethodsInclude?.find( + (methodInclude) => methodInclude === eventMethod + ); + if (monitoredMethod === undefined) { + // Method does not match a method in the config file + acceptEventAllApps = false; + } + } + } + + // Match on all-app filters? + if (acceptEventAllApps === true) { + acceptEvent = true; // Event matches global filters + globals.logger.debug( + 'LOG EVENT: Qix performance event matches global filters in the configuration' + ); + } + } + + // Was event accepted? + if (acceptEventAppSpecific === false && acceptEventAllApps === false) { + acceptEvent === false; + globals.logger.debug( + 'LOG EVENT: Qix performance event does not match filters in the configuration. Skipping event.' + ); + return; + } else { + acceptEvent = true; + } + + // Event matches filters in the configuration. Continue. + // Build the event object + msgObj = { + source: msg[0], + log_row: Number.isInteger(parseInt(msg[1], 10)) ? parseInt(msg[1], 10) : -1, + ts_iso: msg[2], + ts_local: msg[3], + // ts_iso: isoDateRegex.test(msg[2]) ? msg[2] : '', + // ts_local: isoDateRegex.test(msg[3]) ? msg[3] : '', + level: msg[4], + host: msg[5], + subsystem: msg[6], + windows_user: msg[7], + proxy_session_id: uuidRegex.test(msg[8]) ? msg[8] : '', + user_directory: msg[9], + user_id: msg[10], + engine_ts: msg[11], + session_id: uuidRegex.test(msg[12]) ? msg[12] : '', + app_id: uuidRegex.test(msg[13]) ? msg[13] : '', + app_name: eventAppName, + request_id: msg[14], // Request ID is an integer >= 0, set to -99 otherwise + method: msg[15], + // Processtime in float milliseconds + process_time: parseFloat(msg[16]), + work_time: parseFloat(msg[17]), + lock_time: parseFloat(msg[18]), + validate_time: parseFloat(msg[19]), + traverse_time: parseFloat(msg[20]), + // Handle is either -1 or a number. Set to -99 if not a number + handle: Number.isInteger(parseInt(msg[21], 10)) + ? parseInt(msg[21], 10) + : -99, + object_id: msg[22], + // Positive integer, set to -1 if not am integer >= 0 + net_ram: + Number.isInteger(parseInt(msg[23], 10)) && parseInt(msg[23], 10) >= 0 + ? parseInt(msg[23], 10) + : -1, + peak_ram: + Number.isInteger(parseInt(msg[24], 10)) && parseInt(msg[24], 10) >= 0 + ? parseInt(msg[24], 10) + : -1, + object_type: msg[25], + }; + // Different log events deliver QSEoW user directory/user differently. // Create fields that are consistent across all log events if (msgObj.user_directory !== '' && msgObj.user_id !== '') { From e8d8a13e37ecafe3aa9a72b369619a62c3e26e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Wed, 11 Sep 2024 15:18:22 +0000 Subject: [PATCH 2/5] feat(qs-events): Add counters for incoming Qlik Sense events Implements #884 --- package-lock.json | 10 + package.json | 1 + src/butler-sos.js | 9 +- src/config/production_template.yaml | 34 ++- src/globals.js | 28 +- src/lib/config-file-schema.js | 35 ++- src/lib/post-to-influxdb.js | 413 ++++++++++++++++++++++++++ src/lib/udp-event.js | 280 +++++++++++++++++ src/lib/udp_handlers_log_events.js | 60 +++- src/lib/udp_handlers_user_activity.js | 30 ++ 10 files changed, 880 insertions(+), 20 deletions(-) create mode 100644 src/lib/udp-event.js diff --git a/package-lock.json b/package-lock.json index 8def5460..a4600369 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@fastify/static": "^7.0.4", "@influxdata/influxdb-client": "^1.35.0", "@influxdata/influxdb-client-apis": "^1.35.0", + "async-mutex": "^0.5.0", "axios": "^1.7.5", "commander": "^12.1.0", "config": "^3.3.12", @@ -1175,6 +1176,15 @@ "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" }, + "node_modules/async-mutex": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.5.0.tgz", + "integrity": "sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", diff --git a/package.json b/package.json index 0b7bb2d7..9ecfe2d4 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "@fastify/static": "^7.0.4", "@influxdata/influxdb-client": "^1.35.0", "@influxdata/influxdb-client-apis": "^1.35.0", + "async-mutex": "^0.5.0", "axios": "^1.7.5", "commander": "^12.1.0", "config": "^3.3.12", diff --git a/src/butler-sos.js b/src/butler-sos.js index 70b2aaa5..56028fca 100755 --- a/src/butler-sos.js +++ b/src/butler-sos.js @@ -25,6 +25,7 @@ import { setupAnonUsageReportTimer } from './lib/telemetry.js'; import { setupPromClient } from './lib/prom-client.js'; import { verifyConfigFile } from './lib/config-file-verify.js'; import { setupConfigVisServer } from './lib/config-visualise.js'; +import { setupUdpEventsStorage } from './lib/udp-event.js'; // Suppress experimental warnings // https://stackoverflow.com/questions/55778283/how-to-disable-warnings-when-node-is-launched-via-a-global-shell-script @@ -51,7 +52,6 @@ process.emit = function (name, data, ...args) { }; async function sleep(ms) { - // eslint-disable-next-line no-promise-executor-return return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -81,7 +81,6 @@ async function mainScript() { // Sleep 5 seconds otherwise to llow globals to be initialised function sleepLocal(ms) { - // eslint-disable-next-line no-promise-executor-return return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -285,6 +284,12 @@ async function mainScript() { if (globals.config.get('Butler-SOS.configVisualisation.enable') === true) { await setupConfigVisServer(); } + + // Set up rejected user/log events storage, if enabled + if (globals.config.get('Butler-SOS.qlikSenseEvents.rejectedEventCount.enable') === true) { + globals.logger.verbose('MAIN: Rejected events storage enabled'); + await setupUdpEventsStorage(); + } } mainScript(); diff --git a/src/config/production_template.yaml b/src/config/production_template.yaml index 4d61e47e..b56a5f36 100644 --- a/src/config/production_template.yaml +++ b/src/config/production_template.yaml @@ -77,6 +77,27 @@ Butler-SOS: # insertApiKey: # accountId: + # Shared settings for user and log events (see below) + qlikSenseEvents: # Shared settings for user and log events (see below) + influxdb: + enable: true # Should summary (counter) of user and log events be stored in InfluxDB? + writeFrequency: 20000 # How often (milliseconds) should rejected event count be written to InfluxDB? + eventCount: # Track how many events are received from Sense. + # Some events are valid, some are not. Of the valid events, some are rejected by Butler SOS + # based on the configuration in this file. + enable: true # Should event count be stored in InfluxDB? + influxdb: + measurementName: event_count # Name of the InfluxDB measurement where event count is stored + tags: # Tags are added to the data before it's stored in InfluxDB + # - tag: env + # value: DEV + # - tag: foo + # value: bar + rejectedEventCount: + enable: true # Should rejected events be counted and stored in InfluxDB? + influxdb: + measurementName: rejected_event_count # Name of the InfluxDB measurement where rejected event count is stored + # Track individual users opening/closing apps and starting/stopping sessions. # Requires log appender XML file(s) to be added to Sense server(s). userEvents: @@ -205,10 +226,17 @@ Butler-SOS: category: - name: qs_log_category value: unknown - appPerformanceMonitor: # Detailed app performance data extraction from log events - enable: false # Should app performance data be extracted from log events? + enginePerformanceMonitor: # Detailed app performance data extraction from log events + enable: false # Should app performance data be extracted from log events? appNameLookup: # Should app names be looked up based on app IDs? - enable: true + enable: false + trackRejectedEvents: + enable: false # Should events that are rejected by the app performance monitor be tracked? + tags: # Tags are added to the data before it's stored in InfluxDB + # - tag: env + # value: DEV + # - tag: foo + # value: bar monitorFilter: # What objects should be monitored? Entire apps or just specific object(s) within some specific app(s)? # Two kinds of monitoring can be done: # 1) Monitor all apps, except those listed for exclusion. This is defined in the allApps section. diff --git a/src/globals.js b/src/globals.js index 10977339..a73238d2 100755 --- a/src/globals.js +++ b/src/globals.js @@ -13,9 +13,10 @@ import Influx from 'influx'; import { Command, Option } from 'commander'; import { InfluxDB, HttpError, DEFAULT_WriteOptions } from '@influxdata/influxdb-client'; import { OrgsAPI, BucketsAPI } from '@influxdata/influxdb-client-apis'; +import { fileURLToPath } from 'url'; import { getServerTags } from './lib/servertags.js'; -import { fileURLToPath } from 'url'; +import { UdpEvents } from './lib/udp-event.js'; let instance = null; @@ -122,7 +123,6 @@ class Settings { configFileBasename = upath.basename(this.configFile, configFileExtension); if (configFileExtension.toLowerCase() !== '.yaml') { - // eslint-disable-next-line no-console console.log('Error: Config file extension must be yaml'); process.exit(1); } @@ -131,7 +131,6 @@ class Settings { process.env.NODE_CONFIG_DIR = configFilePath; process.env.NODE_ENV = configFileBasename; } else { - // eslint-disable-next-line no-console console.log('Error: Specified config file does not exist'); process.exit(1); } @@ -192,24 +191,19 @@ class Settings { // Are we in a packaged app? if (this.isPkg) { - // eslint-disable-next-line no-console console.log(`Running in packaged app. Executable path: ${this.execPath}`); } else { - // eslint-disable-next-line no-console console.log( `Running in non-packaged environment. Executable path: ${this.execPath}` ); } - // eslint-disable-next-line no-console console.log( `Log file directory: ${upath.join(this.execPath, this.config.get('Butler-SOS.logDirectory'))}` ); - // eslint-disable-next-line no-console console.log(`upath.dirname(process.execPath): ${upath.dirname(process.execPath)}`); - // eslint-disable-next-line no-console console.log(`process.cwd(): ${process.cwd()}`); } @@ -338,6 +332,22 @@ class Settings { this.logger.error(`CONFIG: Setting up UDP log events listener: ${err}`); } + // ------------------------------------ + // Track user events and log events + if (this.config.get('Butler-SOS.qlikSenseEvents.eventCount.enable') === true) { + this.udpEvents = new UdpEvents(this.logger); + } else { + this.udpEvents = null; + } + + // ------------------------------------ + // Track rejected user and log events + if (this.config.get('Butler-SOS.qlikSenseEvents.rejectedEventCount.enable') === true) { + this.rejectedEvents = new UdpEvents(this.logger); + } else { + this.rejectedEvents = null; + } + // ------------------------------------ // Get info on what servers to monitor this.serverList = this.config.get('Butler-SOS.serversToMonitor.servers'); @@ -358,7 +368,6 @@ class Settings { // the pool will emit an error on behalf of any idle clients // it contains if a backend error or network partition happens - // eslint-disable-next-line no-unused-vars this.pgPool.on('error', (err, client) => { this.logger.error(`CONFIG: Unexpected error on idle client: ${err}`); // process.exit(-1); @@ -710,7 +719,6 @@ class Settings { // Static sleep function static sleep(ms) { - // eslint-disable-next-line no-promise-executor-return return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/src/lib/config-file-schema.js b/src/lib/config-file-schema.js index 6604c945..36c77aba 100755 --- a/src/lib/config-file-schema.js +++ b/src/lib/config-file-schema.js @@ -65,6 +65,30 @@ export const confifgFileSchema = { }, ], }, + + qlikSenseEvents: { + influxdb: { + enable: 'boolean', + writeFrequency: 'number', + }, + eventCount: { + enable: 'boolean', + influxdb: { + measurementName: 'string', + "tags?": [ + "tag": 'string', + value: 'string', + ], + }, + }, + rejectedEventCount: { + enable: 'boolean', + influxdb: { + measurementName: 'string', + }, + }, + }, + userEvents: { enable: 'boolean', 'excludeUser?': [ @@ -173,11 +197,20 @@ export const confifgFileSchema = { ], }, }, - appPerformanceMonitor: { + enginePerformanceMonitor: { enable: 'boolean', appNameLookup: { enable: 'boolean', }, + trackRejectedEvents: { + enable: 'boolean', + "tags?": [ + { + tag: 'string', + value: 'string', + }, + ], + }, monitorFilter: { allApps: { enable: 'boolean', diff --git a/src/lib/post-to-influxdb.js b/src/lib/post-to-influxdb.js index 981c8b98..6e036f4a 100755 --- a/src/lib/post-to-influxdb.js +++ b/src/lib/post-to-influxdb.js @@ -1312,3 +1312,416 @@ export async function postLogEventToInfluxdb(msg) { globals.logger.error(`LOG EVENT INFLUXDB 2: Error saving log event to InfluxDB! ${err}`); } } + +// Store event count for all kinds of events in InfluxDB +export async function storeEventCountInfluxDB() { + // Get array of log events + const logEvents = await globals.udpEvents.getLogEvents(); + const userEvents = await globals.udpEvents.getUserEvents(); + + // InfluxDB 1.x + if (globals.config.get('Butler-SOS.influxdbConfig.version') === 1) { + const points = []; + + // Get measurement name to use for event counts + const measurementName = globals.config.get( + 'Butler-SOS.qlikSenseEvents.eventCount.influxdb.measurementName' + ); + + // Loop through data in log events and create datapoints. + // Add the created data points to the points array + for (const event of logEvents) { + const point = { + measurement: measurementName, + tags: { + event_type: 'log', + event_name: event.eventName, + host: event.host, + subsystem: event.subsystem, + }, + fields: { + counter: event.counter, + }, + }; + + // Add static tags from config file + if ( + globals.config.has('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') && + globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') !== + null && + globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags').length > 0 + ) { + const configTags = globals.config.get( + 'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags' + ); + for (const item of configTags) { + point.tags[item.tag] = item.value; + } + } + + points.push(point); + } + + // Loop through data in user events and create datapoints. + // Add the created data points to the points array + for (const event of userEvents) { + const point = { + measurement: measurementName, + tags: { + event_type: 'user', + event_name: event.eventName, + host: event.host, + subsystem: event.subsystem, + }, + fields: { + counter: event.counter, + }, + }; + + // Add static tags from config file + if ( + globals.config.has('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') && + globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') !== + null && + globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags').length > 0 + ) { + const configTags = globals.config.get( + 'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags' + ); + for (const item of configTags) { + point.tags[item.tag] = item.value; + } + } + + points.push(point); + } + + try { + globals.influx.writePoints(points); + } catch (err) { + globals.logger.error(`EVENT COUNT INFLUXDB: Error saving data to InfluxDB v1! ${err}`); + } + } else if (globals.config.get('Butler-SOS.influxdbConfig.version') === 2) { + // Create new write API object + // Advanced write options + const writeOptions = { + /* the maximum points/lines to send in a single batch to InfluxDB server */ + // batchSize: flushBatchSize + 1, // don't let automatically flush data + + /* maximum time in millis to keep points in an unflushed batch, 0 means don't periodically flush */ + flushInterval: 5000, + + /* maximum size of the retry buffer - it contains items that could not be sent for the first time */ + // maxBufferLines: 30_000, + + /* the count of internally-scheduled retries upon write failure, the delays between write attempts follow an exponential backoff strategy if there is no Retry-After HTTP header */ + maxRetries: 2, // do not retry writes + + // ... there are more write options that can be customized, see + // https://influxdata.github.io/influxdb-client-js/influxdb-client.writeoptions.html and + // https://influxdata.github.io/influxdb-client-js/influxdb-client.writeretryoptions.html + }; + + // Create new datapoints object + const points = []; + + try { + const org = globals.config.get('Butler-SOS.influxdbConfig.v2Config.org'); + const bucketName = globals.config.get('Butler-SOS.influxdbConfig.v2Config.bucket'); + + const writeApi = globals.influx.getWriteApi(org, bucketName, 'ns', writeOptions); + + // Ensure that the writeApi object was found + if (!writeApi) { + globals.logger.warn( + `EVENT COUNT INFLUXDB: Influxdb write API object not found. Data will not be sent to InfluxDB` + ); + return; + } + + // Get measurement name to use for event counts + const measurementName = globals.config.get( + 'Butler-SOS.qlikSenseEvents.eventCount.influxdb.measurementName' + ); + + // Loop through data in log events and create datapoints. + // Add the created data points to the points array + for (const event of logEvents) { + const point = new Point(measurementName) + .tag('event_type', 'log') + .tag('event_name', event.eventName) + .tag('host', event.host) + .tag('subsystem', event.subsystem) + .intField('counter', event.counter); + + // Add static tags from config file + if ( + globals.config.has('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') && + globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') !== + null && + globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') + .length > 0 + ) { + const configTags = globals.config.get( + 'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags' + ); + for (const item of configTags) { + point.tag(item.tag, item.value); + } + } + + points.push(point); + } + + // Loop through data in user events and create datapoints. + // Add the created data points to the points array + for (const event of userEvents) { + const point = new Point(measurementName) + .tag('event_type', 'user') + .tag('event_name', event.eventName) + .tag('host', event.host) + .tag('subsystem', event.subsystem) + .intField('counter', event.counter); + + // Add static tags from config file + if ( + globals.config.has('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') && + globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') !== + null && + globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags') + .length > 0 + ) { + const configTags = globals.config.get( + 'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags' + ); + for (const item of configTags) { + point.tag(item.tag, item.value); + } + } + + points.push(point); + } + + try { + const res = await writeApi.writePoints(points); + globals.logger.debug(`EVENT COUNT INFLUXDB: Wrote data to InfluxDB v2`); + } catch (err) { + globals.logger.error( + `EVENT COUNT INFLUXDB: Error saving health data to InfluxDB v2! ${err.stack}` + ); + } + + globals.logger.verbose( + 'EVENT COUNT INFLUXDB: Sent Butler SOS event count data to InfluxDB' + ); + } catch (err) { + globals.logger.error(`EVENT COUNT INFLUXDB: Error getting write API: ${err}`); + } + } +} + +// Store rejected event count in InfluxDB +export async function storeRejectedEventCountInfluxDB() { + // Get array of rejected log events + const rejectedLogEvents = await globals.rejectedEvents.getRejectedLogEvents(); + + // InfluxDB 1.x + if (globals.config.get('Butler-SOS.influxdbConfig.version') === 1) { + const points = []; + + // Get measurement name to use for rejected events + const measurementName = globals.config.get( + 'Butler-SOS.qlikSenseEvents.rejectedEventCount.influxdb.measurementName' + ); + + // Loop through data in rejected log events and create datapoints. + // Add the created data points to the points array + // + // Use counter and process_time as fields + for (const event of rejectedLogEvents) { + if (event.eventName === 'qseow-qix-perf') { + // For each unique combination of eventName, appId, appName, .method and objectType, + // write the counter and processTime properties to InfluxDB + // + // Use eventName, appId,appName, method and objectType as tags + + const tags = { + event_name: event.eventName, + app_id: event.appId, + method: event.method, + object_type: event.objectType, + }; + + // Tags that are empty in some cases. Only add if they are non-empty + if (msg?.app_name?.length > 0) { + tags.app_name = msg.app_name; + tags.app_name_set = 'true'; + } else { + tags.app_name_set = 'false'; + } + + // Add static tags from config file + if ( + globals.config.has( + 'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags' + ) && + globals.config.get( + 'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags' + ) !== null && + globals.config.get( + 'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags' + ).length > 0 + ) { + const configTags = globals.config.get( + 'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags' + ); + for (const item of configTags) { + tags[item.tag] = item.value; + } + } + + const fields = { + counter: event.counter, + process_time: event.processTime, + }; + + const point = { + measurement: measurementName, + tags, + fields, + }; + + points.push(point); + } else { + const point = { + measurement: measurementName, + tags: { + event_name: event.eventName, + }, + fields: { + counter: event.counter, + }, + }; + + points.push(point); + } + } + + try { + globals.influx.writePoints(points); + } catch (err) { + globals.logger.error( + `REJECT LOG EVENT INFLUXDB: Error saving data to InfluxDB v1! ${err}` + ); + } + } else if (globals.config.get('Butler-SOS.influxdbConfig.version') === 2) { + // Create new write API object + // Advanced write options + const writeOptions = { + /* the maximum points/lines to send in a single batch to InfluxDB server */ + // batchSize: flushBatchSize + 1, // don't let automatically flush data + + /* maximum time in millis to keep points in an unflushed batch, 0 means don't periodically flush */ + flushInterval: 5000, + + /* maximum size of the retry buffer - it contains items that could not be sent for the first time */ + // maxBufferLines: 30_000, + + /* the count of internally-scheduled retries upon write failure, the delays between write attempts follow an exponential backoff strategy if there is no Retry-After HTTP header */ + maxRetries: 2, // do not retry writes + + // ... there are more write options that can be customized, see + // https://influxdata.github.io/influxdb-client-js/influxdb-client.writeoptions.html and + // https://influxdata.github.io/influxdb-client-js/influxdb-client.writeretryoptions.html + }; + + // Create new datapoints object + const points = []; + + try { + const org = globals.config.get('Butler-SOS.influxdbConfig.v2Config.org'); + const bucketName = globals.config.get('Butler-SOS.influxdbConfig.v2Config.bucket'); + + const writeApi = globals.influx.getWriteApi(org, bucketName, 'ns', writeOptions); + + // Ensure that the writeApi object was found + if (!writeApi) { + globals.logger.warn( + `LOG EVENT INFLUXDB: Influxdb write API object not found. Data will not be sent to InfluxDB` + ); + return; + } + + // Get measurement name to use for rejected events + const measurementName = globals.config.get( + 'Butler-SOS.qlikSenseEvents.rejectedEventCount.influxdb.measurementName' + ); + + // Loop through data in rejected log events and create datapoints. + // Add the created data points to the points array + // + // Use counter and process_time as fields + for (const event of rejectedLogEvents) { + if (event.eventName === 'qseow-qix-perf') { + // For each unique combination of eventName, appId, appName, .method and objectType, + // write the counter and processTime properties to InfluxDB + // + // Use eventName, appId,appName, method and objectType as tags + let point = new Point(measurementName) + .tag('event_name', event.eventName) + .tag('app_id', event.appId) + .tag('method', event.method) + .tag('object_type', event.objectType) + .intField('counter', event.counter) + .floatField('process_time', event.processTime); + + if (event?.appName?.length > 0) { + point.tag('app_name', event.appName).tag('app_name_set', 'true'); + } else { + point.tag('app_name_set', 'false'); + } + + // Add static tags from config file + if ( + globals.config.has( + 'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags' + ) && + globals.config.get( + 'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags' + ) !== null && + globals.config.get( + 'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags' + ).length > 0 + ) { + const configTags = globals.config.get( + 'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags' + ); + for (const item of configTags) { + point.tag(item.tag, item.value); + } + } + + points.push(point); + } else { + let point = new Point(measurementName) + .tag('event_name', event.eventName) + .intField('counter', event.counter); + + points.push(point); + } + } + + // Write to InfluxDB + try { + const res = await writeApi.writePoints(points); + globals.logger.debug(`REJECT LOG EVENT INFLUXDB: Wrote data to InfluxDB v2`); + } catch (err) { + globals.logger.error( + `REJECTED LOG EVENT INFLUXDB: Error saving data to InfluxDB v2! ${err.stack}` + ); + } + } catch (err) { + globals.logger.error(`REJECTED LOG EVENT INFLUXDB: Error getting write API: ${err}`); + } + } +} diff --git a/src/lib/udp-event.js b/src/lib/udp-event.js new file mode 100644 index 00000000..d79cd3c6 --- /dev/null +++ b/src/lib/udp-event.js @@ -0,0 +1,280 @@ +import { Mutex } from 'async-mutex'; + +import globals from '../globals.js'; +import { storeRejectedEventCountInfluxDB, storeEventCountInfluxDB } from './post-to-influxdb.js'; + +// Class for counting rejected events +export class UdpEvents { + constructor(logger) { + this.logger = logger; + + // Array of objects with log events + // Each object has properties: + // - eventName: string + // - subsystem: string + // - counter: integer + this.logEvents = []; + + // Array of objects with user events + // Each object has properties: + // - eventName: string + // - counter: integer + this.userEvents = []; + + // Array of objects with rejected log events + // Each object has a counter and dimension properties to track app IDs, methods and object types + this.rejectedLogEvents = []; + + // Mutexes for synchronizing access to the arrays + this.logMutex = new Mutex(); + this.userMutex = new Mutex(); + this.rejectedLogMutex = new Mutex(); + } + + // Add a log event of any type + async addLogEvent(event) { + // Ensure the passed event is an object with properties: + // - eventName: string + // - host: string + // - subsystem: string + if (!event.eventName || !event.subsystem || !event.host) { + this.logger.error( + `LOG EVENT TRACKER: Log event object must have properties "eventName", "subsystem" and "host": ${JSON.stringify( + event + )}` + ); + return; + } + + const release = await this.logMutex.acquire(); + + try { + const found = this.logEvents.find((element) => { + return ( + element.eventName === event.eventName && + element.subsystem === event.subsystem && + element.host === event.host + ); + }); + + if (found) { + found.counter += 1; + this.logger.debug( + `LOG EVENT TRACKER: Adding another log event: ${JSON.stringify(event)}, new counter value: ${found.counter}` + ); + } else { + this.logger.debug( + `LOG EVENT TRACKER: Adding first log event: ${JSON.stringify(event)}` + ); + + this.logEvents.push({ + eventName: event.eventName, + host: event.host, + subsystem: event.subsystem, + counter: 1, + }); + } + } finally { + release(); + } + } + + // Add a user event + async addUserEvent(event) { + // Ensure the passed event is an object with properties: + // - eventName: string + // - host: string + // - subsystem: string + if (!event.eventName || !event.subsystem || !event.host) { + this.logger.error( + `USER EVENT TRACKER: User event object must have properties "eventName", "subsystem" and "host": ${JSON.stringify( + event + )}` + ); + return; + } + + const release = await this.userMutex.acquire(); + + try { + const found = this.userEvents.find((element) => { + return ( + element.eventName === event.eventName && + element.subsystem === event.subsystem && + element.host === event.host + ); + }); + + if (found) { + found.counter += 1; + this.logger.debug( + `USER EVENT TRACKER: Adding another user event: ${JSON.stringify(event)}, new counter value: ${found.counter}` + ); + } else { + this.logger.debug( + `USER EVENT TRACKER: Adding first user event: ${JSON.stringify(event)}` + ); + + this.userEvents.push({ + eventName: event.eventName, + host: event.host, + subsystem: event.subsystem, + counter: 1, + }); + } + } finally { + release(); + } + } + + // Get log events + async getLogEvents() { + const release = await this.logMutex.acquire(); + + try { + return this.logEvents; + } finally { + release(); + } + } + + // Get user events + async getUserEvents() { + const release = await this.userMutex.acquire(); + + try { + return this.userEvents; + } finally { + release(); + } + } + + // Add rejected log event + // "Rejected log events" are events that are correctly formatted but are rejected + // Butler SOS due to some reason, e.g. matching the exclude filter criteria in the config file. + async addRejectedLogEvent(event) { + // Ensure the passed event is an object with properties: + // - eventName: string + // + // Pertformance log events also have these properties: + // - appId: string + // - method: string) + // - objectType: string) + // - processTime: float) + if (!event.eventName) { + this.logger.error( + `REJECTED EVENT: Log event object must have property "eventName": ${JSON.stringify( + event + )}` + ); + return; + } + + const release = await this.rejectedLogMutex.acquire(); + // Is this a performance log event? + if (event.eventName === 'qseow-qix-perf') { + try { + const found = this.rejectedLogEvents.find((element) => { + return ( + element.eventName === event.eventName && + element.appId === event.appId && + element.appName === event.appName && + element.method === event.method && + element.objectType === event.objectType + ); + }); + + if (found) { + found.counter += 1; + found.processTime += event.processTime; + this.logger.debug( + `REJECTED EVENT: Adding another log event: ${JSON.stringify(event)}, new counter value: ${found.counter}` + ); + } else { + this.logger.debug( + `REJECTED EVENT: Adding first log event: ${JSON.stringify(event)}` + ); + + this.rejectedLogEvents.push({ + eventName: event.eventName, + appId: event.appId, + appName: event.appName, + method: event.method, + objectType: event.objectType, + counter: 1, + processTime: event.processTime, + }); + } + } finally { + release(); + } + } else { + try { + const found = this.rejectedLogEvents.find((element) => { + return element.eventName === event.eventName; + }); + + if (found) { + found.counter += 1; + this.logger.debug( + `REJECTED EVENT: Adding another log event: ${JSON.stringify(event)}, new counter value: ${found.counter}` + ); + } else { + this.logger.debug( + `REJECTED EVENT: Adding first log event: ${JSON.stringify(event)}` + ); + + this.rejectedLogEvents.push({ + eventName: event.eventName, + counter: 1, + }); + } + } finally { + release(); + } + } + } + + // Get rejected log events + async getRejectedLogEvents() { + const release = await this.rejectedLogMutex.acquire(); + try { + return this.rejectedLogEvents; + } finally { + release(); + } + } + + // Clear rejected events + async clearRejectedEvents() { + const releaseLog = await this.rejectedLogMutex.acquire(); + + try { + this.rejectedLogEvents = []; + + this.logger.debug('REJECTED EVENT: Cleared all rejected events'); + } finally { + releaseLog(); + } + } +} + +export function setupUdpEventsStorage() { + // Is storing event counts to InfluxDB enabled? + if (globals.config.get('Butler-SOS.qlikSenseEvents.influxdb.enable') !== true) { + globals.logger.verbose( + 'EVENT COUNTS: Feature is disabled in config file. Skipping setup of timer for storing event counts to InfluxDB' + ); + return; + } else { + // Configure timer for storing event counts to InfluxDB + setInterval(() => { + globals.logger.verbose( + 'EVENT COUNTS: Timer for storing event counts to InfluxDB triggered' + ); + + storeRejectedEventCountInfluxDB(); + storeEventCountInfluxDB(); + }, globals.config.get('Butler-SOS.qlikSenseEvents.influxdb.writeFrequency')); + } +} diff --git a/src/lib/udp_handlers_log_events.js b/src/lib/udp_handlers_log_events.js index e3effd50..d9733a04 100644 --- a/src/lib/udp_handlers_log_events.js +++ b/src/lib/udp_handlers_log_events.js @@ -127,9 +127,39 @@ export function udpInitLogEventServer() { globals.logger.warn( `LOG EVENT: Received message that is not a recognised log event: ${msgShort}` ); + + // Is logging of event counts enabled? + if (globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.enable') === true) { + // Increase counter for log events + await globals.udpEvents.addLogEvent({ + eventName: 'Unknown', + host: 'Unknown', + subsystem: 'Unknown', + }); + } + return; } + // Add counter for received log events + // Is logging of event counts enabled? + if (globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.enable') === true) { + globals.logger.debug( + `LOG EVENT: Received message that is a recognised log event: ${msg[0]}` + ); + + // Increase counter for log events + // Make eventName lower case, also remove leading and trailing / + let eventName = msg[0].toLowerCase().replace('/', ''); + eventName = eventName.replace('/', ''); + + await globals.udpEvents.addLogEvent({ + eventName, + host: msg[5], + subsystem: msg[6], + }); + } + // Check if any of the log event sources are enabled in the configuration if ( (globals.config.get('Butler-SOS.logEvents.source.engine.enable') === true && @@ -334,8 +364,9 @@ export function udpInitLogEventServer() { // Determine if the message should be handled, based on settings in the config file if ( - globals.config.get('Butler-SOS.logEvents.appPerformanceMonitor.enable') === - false + globals.config.get( + 'Butler-SOS.logEvents.enginePerformanceMonitor.enable' + ) === false ) { globals.logger.debug( 'LOG EVENT: Qix performance monitoring is disabled in the configuration. Skipping event.' @@ -354,7 +385,7 @@ export function udpInitLogEventServer() { // Get the app performance monitor filter configuration from the config file, // so we don't have to read it every time we need some part of it const monitorFilterConfig = globals.config.get( - 'Butler-SOS.logEvents.appPerformanceMonitor.monitorFilter' + 'Butler-SOS.logEvents.enginePerformanceMonitor.monitorFilter' ); let acceptEvent = false; @@ -369,7 +400,7 @@ export function udpInitLogEventServer() { // Should we get app name from the app ID? if ( globals.config.get( - 'Butler-SOS.logEvents.appPerformanceMonitor.appNameLookup.enable' + 'Butler-SOS.logEvents.enginePerformanceMonitor.appNameLookup.enable' ) === true ) { // Get app name from app ID @@ -684,6 +715,27 @@ export function udpInitLogEventServer() { globals.logger.debug( 'LOG EVENT: Qix performance event does not match filters in the configuration. Skipping event.' ); + + // Is logging of rejected performance log events enabled? + if ( + globals.config.get( + 'Butler-SOS.logEvents.enginePerformanceMonitor.enable' + ) === true && + globals.config.get( + 'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.enable' + ) === true + ) { + // Increase counter for rejected performance log events + await globals.rejectedEvents.addRejectedLogEvent({ + eventName: 'qseow-qix-perf', + appId: eventAppId, + appName: eventAppName, + method: eventMethod, + objectType: eventObjectType, + processTime: parseFloat(msg[16]), + }); + } + return; } else { acceptEvent = true; diff --git a/src/lib/udp_handlers_user_activity.js b/src/lib/udp_handlers_user_activity.js index 4ceb4d1c..2471511a 100644 --- a/src/lib/udp_handlers_user_activity.js +++ b/src/lib/udp_handlers_user_activity.js @@ -77,9 +77,39 @@ export function udpInitUserActivityServer() { globals.logger.warn( `USER EVENT: Received message that is not a recognised user event: ${msgShort}` ); + + // Is logging of event counts enabled? + if (globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.enable') === true) { + // Increase counter for log events + await globals.udpEvents.addUserEvent({ + eventName: 'Unknown', + host: 'Unknown', + subsystem: 'Unknown', + }); + } + return; } + // Add counter for received user events + // Is logging of event counts enabled? + if (globals.config.get('Butler-SOS.qlikSenseEvents.eventCount.enable') === true) { + globals.logger.debug( + `USER EVENT: Received message that is a recognised user event: ${msg[0]}` + ); + + // Increase counter for user events + // Make eventName lower case, also remove leading and trailing / + let eventName = msg[0].toLowerCase().replace('/', ''); + eventName = eventName.replace('/', ''); + + await globals.udpEvents.addUserEvent({ + eventName, + host: msg[1], + subsystem: msg[5], + }); + } + // Build object and convert to JSON let msgObj; if (msg[0] === 'qseow-proxy-connection' || msg[0] === 'qseow-proxy-session') { From 9c1bb5bf2ec9f34a03c2ba80025ec408c6a856a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Wed, 11 Sep 2024 19:08:17 +0000 Subject: [PATCH 3/5] feat(qs-event): Categorise events from Qlik Sense as user created or automated Implements #889 --- src/lib/config-file-schema.js | 10 ++++++---- src/lib/post-to-influxdb.js | 2 ++ src/lib/udp_handlers_log_events.js | 23 +++++++++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/lib/config-file-schema.js b/src/lib/config-file-schema.js index 36c77aba..74616fbc 100755 --- a/src/lib/config-file-schema.js +++ b/src/lib/config-file-schema.js @@ -75,9 +75,11 @@ export const confifgFileSchema = { enable: 'boolean', influxdb: { measurementName: 'string', - "tags?": [ - "tag": 'string', - value: 'string', + 'tags?': [ + { + tag: 'string', + value: 'string', + }, ], }, }, @@ -204,7 +206,7 @@ export const confifgFileSchema = { }, trackRejectedEvents: { enable: 'boolean', - "tags?": [ + 'tags?': [ { tag: 'string', value: 'string', diff --git a/src/lib/post-to-influxdb.js b/src/lib/post-to-influxdb.js index 6e036f4a..a88a3c77 100755 --- a/src/lib/post-to-influxdb.js +++ b/src/lib/post-to-influxdb.js @@ -1002,6 +1002,7 @@ export async function postLogEventToInfluxdb(msg) { object_type: msg.object_type, proxy_session_id: msg.proxy_session_id, session_id: msg.session_id, + event_activity_source: msg.event_activity_source, }; // Tags that are empty in some cases. Only add if they are non-empty @@ -1241,6 +1242,7 @@ export async function postLogEventToInfluxdb(msg) { .tag('object_type', msg.object_type) .tag('proxy_session_id', msg.proxy_session_id) .tag('session_id', msg.session_id) + .tag('event_activity_source', msg.event_activity_source) .stringField('app_id', msg.app_id) .floatField('process_time', parseFloat(msg.process_time)) .floatField('work_time', parseFloat(msg.work_time)) diff --git a/src/lib/udp_handlers_log_events.js b/src/lib/udp_handlers_log_events.js index d9733a04..13c9b010 100644 --- a/src/lib/udp_handlers_log_events.js +++ b/src/lib/udp_handlers_log_events.js @@ -374,6 +374,28 @@ export function udpInitLogEventServer() { return; } + // Get source of event activity. + // + // The source is either user activity of some kind (e.g. opening an app, making a selection), or + // the result of some automated process (e.g. a scheduled app reload). + // + // The proxy session ID is used to determine the source of the event. + // If the proxy session ID is '0', the event is considered to be non-user activity, for example a scheduled reload. + // Otherwise, the event is considered to be the result of an action by a user, for example opening an app, making a selection, etc. + let eventActivitySource; + console.log(msg[15] + '---' + msg[8] + ': ' + msg[8]?.length); + if (msg[8] === '0') { + // Event is the result of an automated process + globals.logger.debug( + 'LOG EVENT: Qix performance event is non-user activity.' + ); + eventActivitySource = 'non-user'; + } else { + // Event is user activity + globals.logger.debug('LOG EVENT: Qix performance event is user activity.'); + eventActivitySource = 'user'; + } + // Does event match filters in the config file? // // There are two types of filters: @@ -784,6 +806,7 @@ export function udpInitLogEventServer() { ? parseInt(msg[24], 10) : -1, object_type: msg[25], + event_activity_source: eventActivitySource, }; // Different log events deliver QSEoW user directory/user differently. From 0ed25d576cc19dc9368999a8a9ec170d3b7fb62a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Wed, 11 Sep 2024 19:43:48 +0000 Subject: [PATCH 4/5] fix(config)!: Make naming of InfluxDB tags consistent across entire config file Fixes #890 --- src/config/production_template.yaml | 16 ++++++------- src/globals.js | 2 +- src/lib/config-file-schema.js | 8 +++---- src/lib/config-file-verify.js | 4 ---- src/lib/post-to-influxdb.js | 20 ++++++++-------- src/lib/post-to-mqtt.js | 4 ++-- src/lib/post-to-new-relic.js | 37 +++++++---------------------- 7 files changed, 33 insertions(+), 58 deletions(-) diff --git a/src/config/production_template.yaml b/src/config/production_template.yaml index b56a5f36..7538bbca 100644 --- a/src/config/production_template.yaml +++ b/src/config/production_template.yaml @@ -89,9 +89,9 @@ Butler-SOS: influxdb: measurementName: event_count # Name of the InfluxDB measurement where event count is stored tags: # Tags are added to the data before it's stored in InfluxDB - # - tag: env + # - name: env # value: DEV - # - tag: foo + # - name: foo # value: bar rejectedEventCount: enable: true # Should rejected events be counted and stored in InfluxDB? @@ -111,9 +111,9 @@ Butler-SOS: serverHost: # Host/IP where user event server will listen for events from Sense portUserActivityEvents: 9997 # Port on which user event server will listen for events from Sense tags: # Tags are added to the data before it's stored in InfluxDB - # - tag: env + # - name: env # value: DEV - # - tag: foo + # - name: foo # value: bar sendToMQTT: enable: false # Set to true if user events should be forwarded as MQTT messages @@ -148,9 +148,9 @@ Butler-SOS: serverHost: # Host/IP where log event server will listen for events from Sense portLogEvents: 9996 # Port on which log event server will listen for events from Sense tags: - # - tag: env + # - name: env # value: DEV - # - tag: foo + # - name: foo # value: bar source: engine: @@ -233,9 +233,9 @@ Butler-SOS: trackRejectedEvents: enable: false # Should events that are rejected by the app performance monitor be tracked? tags: # Tags are added to the data before it's stored in InfluxDB - # - tag: env + # - name: env # value: DEV - # - tag: foo + # - name: foo # value: bar monitorFilter: # What objects should be monitored? Entire apps or just specific object(s) within some specific app(s)? # Two kinds of monitoring can be done: diff --git a/src/globals.js b/src/globals.js index a73238d2..f6a4ed80 100755 --- a/src/globals.js +++ b/src/globals.js @@ -422,7 +422,7 @@ class Settings { )}` ); - tagValuesLogEvent.push(entry.tag); + tagValuesLogEvent.push(entry.name); }); } diff --git a/src/lib/config-file-schema.js b/src/lib/config-file-schema.js index 74616fbc..3cdeeb5d 100755 --- a/src/lib/config-file-schema.js +++ b/src/lib/config-file-schema.js @@ -77,7 +77,7 @@ export const confifgFileSchema = { measurementName: 'string', 'tags?': [ { - tag: 'string', + name: 'string', value: 'string', }, ], @@ -105,7 +105,7 @@ export const confifgFileSchema = { }, 'tags?': [ { - tag: 'string', + name: 'string', value: 'string', }, ], @@ -150,7 +150,7 @@ export const confifgFileSchema = { }, 'tags?': [ { - tag: 'string', + name: 'string', value: 'string', }, ], @@ -208,7 +208,7 @@ export const confifgFileSchema = { enable: 'boolean', 'tags?': [ { - tag: 'string', + name: 'string', value: 'string', }, ], diff --git a/src/lib/config-file-verify.js b/src/lib/config-file-verify.js index e9fea22b..14005976 100755 --- a/src/lib/config-file-verify.js +++ b/src/lib/config-file-verify.js @@ -68,11 +68,9 @@ export async function verifyConfigFile() { const serverTagsDefinition = globals.config.get( 'Butler-SOS.serversToMonitor.serverTagsDefinition' ); - // eslint-disable-next-line no-restricted-syntax for (const tag of serverTagsDefinition) { // Check that all servers have this tag const servers = globals.config.get('Butler-SOS.serversToMonitor.servers'); - // eslint-disable-next-line no-restricted-syntax for (const server of servers) { // Check if server.serverTags.tag is defined if (server?.serverTags === null || !server?.serverTags[tag]) { @@ -90,9 +88,7 @@ export async function verifyConfigFile() { // Now ensure that the tags defined for each server are valid and that there are no extra tags there const servers = globals.config.get('Butler-SOS.serversToMonitor.servers'); - // eslint-disable-next-line no-restricted-syntax for (const server of servers) { - // eslint-disable-next-line no-restricted-syntax for (const tag in server.serverTags) { if (!serverTagsDefinition.includes(tag)) { globals.logger.error( diff --git a/src/lib/post-to-influxdb.js b/src/lib/post-to-influxdb.js index a88a3c77..3dd63bc4 100755 --- a/src/lib/post-to-influxdb.js +++ b/src/lib/post-to-influxdb.js @@ -712,7 +712,7 @@ export async function postUserEventToInfluxdb(msg) { ) { const configTags = globals.config.get('Butler-SOS.userEvents.tags'); for (const item of configTags) { - tags[item.tag] = item.value; + tags[item.name] = item.value; } } @@ -824,7 +824,7 @@ export async function postUserEventToInfluxdb(msg) { ) { const configTags = globals.config.get('Butler-SOS.userEvents.tags'); for (const item of configTags) { - point.tag(item.tag, item.value); + point.tag(item.name, item.value); } } @@ -1043,7 +1043,7 @@ export async function postLogEventToInfluxdb(msg) { ) { const configTags = globals.config.get('Butler-SOS.logEvents.tags'); for (const item of configTags) { - tags[item.tag] = item.value; + tags[item.name] = item.value; } } @@ -1280,7 +1280,7 @@ export async function postLogEventToInfluxdb(msg) { ) { const configTags = globals.config.get('Butler-SOS.logEvents.tags'); for (const item of configTags) { - point.tag(item.tag, item.value); + point.tag(item.name, item.value); } } @@ -1357,7 +1357,7 @@ export async function storeEventCountInfluxDB() { 'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags' ); for (const item of configTags) { - point.tags[item.tag] = item.value; + point.tags[item.name] = item.value; } } @@ -1391,7 +1391,7 @@ export async function storeEventCountInfluxDB() { 'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags' ); for (const item of configTags) { - point.tags[item.tag] = item.value; + point.tags[item.name] = item.value; } } @@ -1468,7 +1468,7 @@ export async function storeEventCountInfluxDB() { 'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags' ); for (const item of configTags) { - point.tag(item.tag, item.value); + point.tag(item.name, item.value); } } @@ -1497,7 +1497,7 @@ export async function storeEventCountInfluxDB() { 'Butler-SOS.qlikSenseEvents.eventCount.influxdb.tags' ); for (const item of configTags) { - point.tag(item.tag, item.value); + point.tag(item.name, item.value); } } @@ -1578,7 +1578,7 @@ export async function storeRejectedEventCountInfluxDB() { 'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags' ); for (const item of configTags) { - tags[item.tag] = item.value; + tags[item.name] = item.value; } } @@ -1699,7 +1699,7 @@ export async function storeRejectedEventCountInfluxDB() { 'Butler-SOS.logEvents.enginePerformanceMonitor.trackRejectedEvents.tags' ); for (const item of configTags) { - point.tag(item.tag, item.value); + point.tag(item.name, item.value); } } diff --git a/src/lib/post-to-mqtt.js b/src/lib/post-to-mqtt.js index fa7bfe09..f8739b53 100755 --- a/src/lib/post-to-mqtt.js +++ b/src/lib/post-to-mqtt.js @@ -135,7 +135,7 @@ export function postUserEventToMQTT(msg) { const configTags = globals.config.get('Butler-SOS.userEvents.tags'); // eslint-disable-next-line no-restricted-syntax for (const item of configTags) { - payload.tags[item.tag] = item.value; + payload.tags[item.name] = item.value; } } @@ -221,7 +221,7 @@ export function postLogEventToMQTT(msg) { const configTags = globals.config.get('Butler-SOS.logEvents.tags'); // eslint-disable-next-line no-restricted-syntax for (const item of configTags) { - payload.tags[item.tag] = item.value; + payload.tags[item.name] = item.value; } } diff --git a/src/lib/post-to-new-relic.js b/src/lib/post-to-new-relic.js index 6074e283..e24e95d7 100755 --- a/src/lib/post-to-new-relic.js +++ b/src/lib/post-to-new-relic.js @@ -1,6 +1,3 @@ -/* eslint-disable prefer-destructuring */ -/* eslint-disable no-unused-vars */ - import crypto from 'crypto'; import axios from 'axios'; @@ -86,7 +83,6 @@ export async function postHealthMetricsToNewRelic(_host, body, tags) { 'Butler-SOS.newRelic.metric.attribute.static' ); - // eslint-disable-next-line no-restricted-syntax for (const item of staticAttributes) { attributes[item.name] = item.value; } @@ -281,7 +277,6 @@ export async function postHealthMetricsToNewRelic(_host, body, tags) { globals.config.get('Butler-SOS.newRelic.metric.header').length > 0 ) { const configHeaders = globals.config.get('Butler-SOS.newRelic.metric.header'); - // eslint-disable-next-line no-restricted-syntax for (const header of configHeaders) { headers[header.name] = header.value; } @@ -300,7 +295,6 @@ export async function postHealthMetricsToNewRelic(_host, body, tags) { `HEALTH METRICS NEW RELIC: Complete New Relic config=${JSON.stringify(nrAccounts)}` ); - // eslint-disable-next-line no-restricted-syntax for (const accountName of globals.config.get( 'Butler-SOS.userEvents.sendToNewRelic.destinationAccount' )) { @@ -323,7 +317,6 @@ export async function postHealthMetricsToNewRelic(_host, body, tags) { } else { headers['Api-Key'] = newRelicConfig[0].insertApiKey; - // eslint-disable-next-line no-await-in-loop const res = await axios.post(remoteUrl, payload, { headers, timeout: 10000 }); globals.logger.debug( @@ -373,7 +366,6 @@ export async function postProxySessionsToNewRelic(userSessions) { 'Butler-SOS.newRelic.metric.attribute.static' ); - // eslint-disable-next-line no-restricted-syntax for (const item of staticAttributes) { attributes[item.name] = item.value; } @@ -432,7 +424,6 @@ export async function postProxySessionsToNewRelic(userSessions) { globals.config.get('Butler-SOS.newRelic.metric.header').length > 0 ) { const configHeaders = globals.config.get('Butler-SOS.newRelic.metric.header'); - // eslint-disable-next-line no-restricted-syntax for (const header of configHeaders) { headers[header.name] = header.value; } @@ -450,7 +441,6 @@ export async function postProxySessionsToNewRelic(userSessions) { `PROXY SESSIONS NEW RELIC: Complete New Relic config=${JSON.stringify(nrAccounts)}` ); - // eslint-disable-next-line no-restricted-syntax for (const accountName of globals.config.get( 'Butler-SOS.userEvents.sendToNewRelic.destinationAccount' )) { @@ -477,7 +467,6 @@ export async function postProxySessionsToNewRelic(userSessions) { `PROXY SESSIONS NEW RELIC: Proxy session count for server "${userSessions.host}", virtual proxy "${userSessions.virtualProxy}": ${userSessions.sessionCount}` ); - // eslint-disable-next-line no-await-in-loop const res = await axios.post(remoteUrl, payload, { headers, timeout: 5000 }); globals.logger.debug( @@ -513,13 +502,15 @@ export async function postButlerSOSUptimeToNewRelic(fields) { const ts = new Date().getTime(); // Timestamp in millisec // Add static fields to attributes if they exist - if (globals.config.has('Butler-SOS.uptimeMonitor.storeNewRelic.attribute.static') && globals.config.get('Butler-SOS.uptimeMonitor.storeNewRelic.attribute.static') !== null) { + if ( + globals.config.has('Butler-SOS.uptimeMonitor.storeNewRelic.attribute.static') && + globals.config.get('Butler-SOS.uptimeMonitor.storeNewRelic.attribute.static') !== null + ) { const staticAttributes = globals.config.get( 'Butler-SOS.uptimeMonitor.storeNewRelic.attribute.static' ); - // staticAttributes is an array of objects. Null - // eslint-disable-next-line no-restricted-syntax + // staticAttributes is an array of objects. Null for (const item of staticAttributes) { attributes[item.name] = item.value; } @@ -600,7 +591,6 @@ export async function postButlerSOSUptimeToNewRelic(fields) { 'Content-Type': 'application/json', }; - // eslint-disable-next-line no-restricted-syntax for (const header of globals.config.get('Butler-SOS.newRelic.metric.header')) { headers[header.name] = header.value; } @@ -617,7 +607,6 @@ export async function postButlerSOSUptimeToNewRelic(fields) { `UPTIME NEW RELIC: Complete New Relic config=${JSON.stringify(nrAccounts)}` ); - // eslint-disable-next-line no-restricted-syntax for (const accountName of globals.config.get( 'Butler-SOS.uptimeMonitor.storeNewRelic.destinationAccount' )) { @@ -638,7 +627,6 @@ export async function postButlerSOSUptimeToNewRelic(fields) { } else { headers['Api-Key'] = newRelicConfig[0].insertApiKey; - // eslint-disable-next-line no-await-in-loop const res = await axios.post(remoteUrl, payload, { headers, timeout: 5000 }); globals.logger.debug( @@ -719,9 +707,8 @@ export async function postUserEventToNewRelic(msg) { globals.config.get('Butler-SOS.userEvents.tags').length > 0 ) { const configTags = globals.config.get('Butler-SOS.userEvents.tags'); - // eslint-disable-next-line no-restricted-syntax for (const item of configTags) { - attributes[item.tag] = item.value; + attributes[item.name] = item.value; } } @@ -748,7 +735,6 @@ export async function postUserEventToNewRelic(msg) { globals.config.get('Butler-SOS.newRelic.event.header').length > 0 ) { const configHeaders = globals.config.get('Butler-SOS.newRelic.event.header'); - // eslint-disable-next-line no-restricted-syntax for (const header of configHeaders) { headers[header.name] = header.value; } @@ -766,7 +752,6 @@ export async function postUserEventToNewRelic(msg) { `USER EVENT NEW RELIC: Complete New Relic config=${JSON.stringify(nrAccounts)}` ); - // eslint-disable-next-line no-restricted-syntax for (const accountName of globals.config.get( 'Butler-SOS.userEvents.sendToNewRelic.destinationAccount' )) { @@ -792,7 +777,6 @@ export async function postUserEventToNewRelic(msg) { // Add API key for this NR account as http header headers['Api-Key'] = newRelicConfig[0].insertApiKey; - // eslint-disable-next-line no-await-in-loop const res = await axios.post(eventUrl, payload, { headers, timeout: 10000 }); globals.logger.debug( @@ -974,9 +958,8 @@ export async function postLogEventToNewRelic(msg) { globals.config.get('Butler-SOS.logEvents.tags').length > 0 ) { const configTags = globals.config.get('Butler-SOS.logEvents.tags'); - // eslint-disable-next-line no-restricted-syntax for (const item of configTags) { - attributes[item.tag] = item.value; + attributes[item.name] = item.value; } } @@ -987,7 +970,6 @@ export async function postLogEventToNewRelic(msg) { globals.config.get('Butler-SOS.newRelic.event.attribute.static').length > 0 ) { const configTags = globals.config.get('Butler-SOS.newRelic.event.attribute.static'); - // eslint-disable-next-line no-restricted-syntax for (const item of configTags) { attributes[item.name] = item.value; } @@ -1030,7 +1012,6 @@ export async function postLogEventToNewRelic(msg) { globals.config.get('Butler-SOS.newRelic.event.header').length > 0 ) { const configHeaders = globals.config.get('Butler-SOS.newRelic.event.header'); - // eslint-disable-next-line no-restricted-syntax for (const header of configHeaders) { headers[header.name] = header.value; } @@ -1043,12 +1024,11 @@ export async function postLogEventToNewRelic(msg) { let nrAccounts = globals.config.get('Butler-SOS.thirdPartyToolsCredentials.newRelic'); if (nrAccounts === null) { nrAccounts = []; - } + } globals.logger.debug( `LOG EVENT NEW RELIC: Complete New Relic config=${JSON.stringify(nrAccounts)}` ); - // eslint-disable-next-line no-restricted-syntax for (const accountName of globals.config.get( 'Butler-SOS.logEvents.sendToNewRelic.destinationAccount' )) { @@ -1078,7 +1058,6 @@ export async function postLogEventToNewRelic(msg) { // Add API key for this NR account as http header headers['Api-Key'] = newRelicConfig[0].insertApiKey; - // eslint-disable-next-line no-await-in-loop const res = await axios.post(eventUrl, payload, { headers, timeout: 10000 }); globals.logger.debug( From 6c4e02f0cc86e1010ef065b5cd40a01135ee9cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Wed, 11 Sep 2024 19:46:03 +0000 Subject: [PATCH 5/5] chore(deps): Update dependencies --- package-lock.json | 49 ++++++++++++++++++++++++++++++----------------- package.json | 8 ++++---- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/package-lock.json b/package-lock.json index a4600369..80ecaf1e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@influxdata/influxdb-client": "^1.35.0", "@influxdata/influxdb-client-apis": "^1.35.0", "async-mutex": "^0.5.0", - "axios": "^1.7.5", + "axios": "^1.7.7", "commander": "^12.1.0", "config": "^3.3.12", "fastify": "^4.28.1", @@ -41,15 +41,15 @@ "yaml-validator": "^5.0.1" }, "devDependencies": { - "@eslint/js": "^9.9.1", + "@eslint/js": "^9.10.0", "esbuild": "^0.23.1", - "eslint": "^9.9.1", + "eslint": "^9.10.0", "eslint-config-prettier": "^9.1.0", "eslint-formatter-table": "^7.32.1", "eslint-plugin-prettier": "^5.2.1", "globals": "^15.9.0", "prettier": "^3.3.3", - "snyk": "^1.1293.0" + "snyk": "^1.1293.1" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -586,9 +586,9 @@ } }, "node_modules/@eslint/js": { - "version": "9.9.1", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.9.1.tgz", - "integrity": "sha512-xIDQRsfg5hNBqHz04H1R3scSVwmI+KUbqjsQKHKQ1DAUSaUjYPReZZmS/5PNiKu1fUvzDd6H7DEDKACSEhu+TQ==", + "version": "9.10.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.10.0.tgz", + "integrity": "sha512-fuXtbiP5GWIn8Fz+LWoOMVf/Jxm+aajZYkhi6CuEm4SxymFM+eUWzbO9qXT+L0iCkL5+KGYMCSGxo686H19S1g==", "dev": true, "license": "MIT", "engines": { @@ -605,6 +605,19 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@eslint/plugin-kit": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.1.0.tgz", + "integrity": "sha512-autAXT203ixhqei9xt+qkYOvY8l6LAFIdT2UXc/RPNeUVfqRF1BV94GTJyVPFKT8nFM6MyVJhjLj9E8JWvf5zQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "levn": "^0.4.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, "node_modules/@fastify/accept-negotiator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@fastify/accept-negotiator/-/accept-negotiator-1.1.0.tgz", @@ -1210,9 +1223,9 @@ } }, "node_modules/axios": { - "version": "1.7.5", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.5.tgz", - "integrity": "sha512-fZu86yCo+svH3uqJ/yTdQ0QHpQu5oL+/QE+QPSv6BZSkDAoky9vytxp7u5qk83OJFS3kEBcesWni9WTZAv3tSw==", + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", "license": "MIT", "dependencies": { "follow-redirects": "^1.15.6", @@ -1640,9 +1653,9 @@ } }, "node_modules/eslint": { - "version": "9.9.1", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.9.1.tgz", - "integrity": "sha512-dHvhrbfr4xFQ9/dq+jcVneZMyRYLjggWjk6RVsIiHsP8Rz6yZ8LvZ//iU4TrZF+SXWG+JkNF2OyiZRvzgRDqMg==", + "version": "9.10.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.10.0.tgz", + "integrity": "sha512-Y4D0IgtBZfOcOUAIQTSXBKoNGfY0REGqHJG6+Q81vNippW5YlKjHFj4soMxamKK1NXHUWuBZTLdU3Km+L/pcHw==", "dev": true, "license": "MIT", "dependencies": { @@ -1650,7 +1663,8 @@ "@eslint-community/regexpp": "^4.11.0", "@eslint/config-array": "^0.18.0", "@eslint/eslintrc": "^3.1.0", - "@eslint/js": "9.9.1", + "@eslint/js": "9.10.0", + "@eslint/plugin-kit": "^0.1.0", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.3.0", "@nodelib/fs.walk": "^1.2.8", @@ -1673,7 +1687,6 @@ "is-glob": "^4.0.0", "is-path-inside": "^3.0.3", "json-stable-stringify-without-jsonify": "^1.0.1", - "levn": "^0.4.1", "lodash.merge": "^4.6.2", "minimatch": "^3.1.2", "natural-compare": "^1.4.0", @@ -3678,9 +3691,9 @@ } }, "node_modules/snyk": { - "version": "1.1293.0", - "resolved": "https://registry.npmjs.org/snyk/-/snyk-1.1293.0.tgz", - "integrity": "sha512-kAMTPHXlOJhcogYk1P2GMmnIIao7ZiQDCznSIdUbQGW68hoaWKghVUlLz4ny+LjjqemE1KII/amVwoFTRoNIlA==", + "version": "1.1293.1", + "resolved": "https://registry.npmjs.org/snyk/-/snyk-1.1293.1.tgz", + "integrity": "sha512-CnbNrsEUMGfajfJ5/03BIgx1ixWKr9Kk+9xDw6sZqKy4K5K01DkyUp/V+WjbCfjr0li9+aE7u70s276KEOuiNA==", "dev": true, "hasInstallScript": true, "license": "Apache-2.0", diff --git a/package.json b/package.json index 9ecfe2d4..89480701 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "@influxdata/influxdb-client": "^1.35.0", "@influxdata/influxdb-client-apis": "^1.35.0", "async-mutex": "^0.5.0", - "axios": "^1.7.5", + "axios": "^1.7.7", "commander": "^12.1.0", "config": "^3.3.12", "fastify": "^4.28.1", @@ -62,15 +62,15 @@ "yaml-validator": "^5.0.1" }, "devDependencies": { - "@eslint/js": "^9.9.1", + "@eslint/js": "^9.10.0", "esbuild": "^0.23.1", - "eslint": "^9.9.1", + "eslint": "^9.10.0", "eslint-config-prettier": "^9.1.0", "eslint-formatter-table": "^7.32.1", "eslint-plugin-prettier": "^5.2.1", "globals": "^15.9.0", "prettier": "^3.3.3", - "snyk": "^1.1293.0" + "snyk": "^1.1293.1" }, "pkg": { "assets": [