From 640d8e890ef087f1c7671ac9cb9a19807a01eff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Tue, 4 Jun 2024 07:03:13 +0000 Subject: [PATCH 1/5] buiild: Update to latest release-please version Implements #812 --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ce49e029..a88906c6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,7 +20,7 @@ jobs: - name: Show github.ref run: echo "$GITHUB_REF" - - uses: google-github-actions/release-please-action@v4 + - uses: googleapis/release-please-action@v4 id: release if: github.repository_owner == 'ptarmiganlabs' with: From 1ba4391df5fca5d20aaed587adad7097c4beb143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Tue, 4 Jun 2024 07:44:41 +0000 Subject: [PATCH 2/5] fix(telemetry): Update telemetry data to reflect current config options Fixes #813 --- src/lib/telemetry.js | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/lib/telemetry.js b/src/lib/telemetry.js index b3902f90..abf06d14 100644 --- a/src/lib/telemetry.js +++ b/src/lib/telemetry.js @@ -10,14 +10,17 @@ const callRemoteURL = async function reportTelemetry() { let heartbeat = false; let dockerHealthCheck = false; let uptimeMonitor = false; + let uptimeMonitorNewRelic = false; let userEventsEnable = false; let userEventsMQTTEnable = false; let userEventsInfluxDBEnable = false; + let userEventsNewRelicEnable = false; let logEventsProxyEnable = false; let logEventsSchedulerEnable = false; let logEventsRepositoryEnable = false; let logEventsMQTTEnable = false; let logEventsInfluxDBEnable = false; + let logEventsNewRelicEnable = false; let logdbEnable = false; let mqttEnable = false; let newRelicEnable = false; @@ -39,6 +42,10 @@ const callRemoteURL = async function reportTelemetry() { uptimeMonitor = true; } + if (globals.config.get('Butler-SOS.uptimeMonitor.storeNewRelic.enable') === true) { + uptimeMonitorNewRelic = true; + } + if (globals.config.get('Butler-SOS.userEvents.enable') === true) { userEventsEnable = true; } @@ -51,6 +58,10 @@ const callRemoteURL = async function reportTelemetry() { userEventsInfluxDBEnable = true; } + if (globals.config.get('Butler-SOS.userEvents.sendToNewRelic.enable') === true) { + userEventsNewRelicEnable = true; + } + if (globals.config.get('Butler-SOS.logEvents.source.proxy.enable') === true) { logEventsProxyEnable = true; } @@ -71,6 +82,10 @@ const callRemoteURL = async function reportTelemetry() { logEventsInfluxDBEnable = true; } + if (globals.config.get('Butler-SOS.logEvents.sendToNewRelic.enable') === true) { + logEventsNewRelicEnable = true; + } + if (globals.config.get('Butler-SOS.logdb.enable') === true) { logdbEnable = true; } @@ -122,23 +137,24 @@ const callRemoteURL = async function reportTelemetry() { feature_heartbeat: heartbeat, feature_dockerHealthCheck: dockerHealthCheck, feature_uptimeMonitor: uptimeMonitor, - feature_uptimeMonitor_storeInInfluxdb: globals.config.get( - 'Butler-SOS.uptimeMonitor.storeInInfluxdb.butlerSOSMemoryUsage' - ), + feature_uptimeMonitor_storeNewRelic: uptimeMonitorNewRelic, feature_udpServer: globals.config.get('Butler-SOS.userEvents.enable'), feature_userEvents: userEventsEnable, feature_userEventsMQTT: userEventsMQTTEnable, feature_userEventsInfluxdb: userEventsInfluxDBEnable, + feature_userEventsNewRelic: userEventsNewRelicEnable, feature_logEventsProxy: logEventsProxyEnable, feature_logEventsScheduler: logEventsSchedulerEnable, feature_logEventsRepository: logEventsRepositoryEnable, feature_logEventsMQTT: logEventsMQTTEnable, feature_logEventsInfluxdb: logEventsInfluxDBEnable, + feature_logEventsNewRelic: logEventsNewRelicEnable, feature_logdb: logdbEnable, feature_mqtt: mqttEnable, feature_newRelic: newRelicEnable, feature_prometheus: prometheusEnable, feature_influxdb: influxdbEnable, + feature_influxdb_version: globals.config.get('Butler-SOS.influxdbConfig.version'), feature_appNames: appNamesExtractEnable, feature_userSessions: userSessionsEnable, @@ -159,23 +175,26 @@ const callRemoteURL = async function reportTelemetry() { heartbeat, dockerHealthCheck, uptimeMonitor, - uptimeMonitor_storeInInfluxdb: globals.config.get( - 'Butler-SOS.uptimeMonitor.storeInInfluxdb.butlerSOSMemoryUsage' - ), + uptimeMonitorNewRelic, udpServer: globals.config.get('Butler-SOS.userEvents.enable'), userEvents: userEventsEnable, userEventsMQTT: userEventsMQTTEnable, userEventsInfluxdb: userEventsInfluxDBEnable, + userEventsNewRelic: userEventsNewRelicEnable, logEventsProxy: logEventsProxyEnable, logEventsScheduler: logEventsSchedulerEnable, logEventsRepository: logEventsRepositoryEnable, logEventsMQTT: logEventsMQTTEnable, logEventsInfluxdb: logEventsInfluxDBEnable, + logEventsNewRelic: logEventsNewRelicEnable, logdb: logdbEnable, mqtt: mqttEnable, newRelic: newRelicEnable, prometheus: prometheusEnable, influxdb: influxdbEnable, + influxdbVersion: globals.config.get( + 'Butler-SOS.influxdbConfig.version' + ), appNames: appNamesExtractEnable, userSessions: userSessionsEnable, }, From 2384b6cb20325b0085335a5563dbaf8be22ffccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Tue, 4 Jun 2024 07:46:31 +0000 Subject: [PATCH 3/5] fix(influxdb): Flush data to InfluxDB v2 every 5 seconds --- src/globals.js | 2 +- src/lib/post-to-influxdb.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/globals.js b/src/globals.js index f43831be..886cc7d4 100755 --- a/src/globals.js +++ b/src/globals.js @@ -661,7 +661,7 @@ async function initInfluxDB() { defaultTags: tags, /* maximum time in millis to keep points in an unflushed batch, 0 means don't periodically flush */ - flushInterval: 1000, + flushInterval: 5000, /* maximum size of the retry buffer - it contains items that could not be sent for the first time */ // maxBufferLines: 30_000, diff --git a/src/lib/post-to-influxdb.js b/src/lib/post-to-influxdb.js index b21988c2..445a0a2c 100755 --- a/src/lib/post-to-influxdb.js +++ b/src/lib/post-to-influxdb.js @@ -625,7 +625,7 @@ async function postButlerSOSMemoryUsageToInfluxdb(memory) { // }, /* maximum time in millis to keep points in an unflushed batch, 0 means don't periodically flush */ - flushInterval: 1000, + flushInterval: 5000, /* maximum size of the retry buffer - it contains items that could not be sent for the first time */ // maxBufferLines: 30_000, @@ -778,7 +778,7 @@ async function postUserEventToInfluxdb(msg) { // }, /* maximum time in millis to keep points in an unflushed batch, 0 means don't periodically flush */ - flushInterval: 1000, + flushInterval: 5000, /* maximum size of the retry buffer - it contains items that could not be sent for the first time */ // maxBufferLines: 30_000, @@ -1060,7 +1060,7 @@ async function postLogEventToInfluxdb(msg) { // }, /* maximum time in millis to keep points in an unflushed batch, 0 means don't periodically flush */ - flushInterval: 1000, + flushInterval: 5000, /* maximum size of the retry buffer - it contains items that could not be sent for the first time */ // maxBufferLines: 30_000, From a30c34b3e300eaf0d551d2c2afcf63f66e0b4edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Tue, 4 Jun 2024 08:52:20 +0000 Subject: [PATCH 4/5] feat(config): Verify server tags when Butler SOS starts Implements #815 --- src/lib/config-file-verify.js | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/lib/config-file-verify.js b/src/lib/config-file-verify.js index 6f828bf5..ae58f507 100755 --- a/src/lib/config-file-verify.js +++ b/src/lib/config-file-verify.js @@ -56,6 +56,61 @@ async function verifyConfigFile() { } } + // Verify that server tags are correctly defined + // In the config file section `Butler-SOS.serversToMonitor.serverTagsDefinition` it's possible to define zero or more tags that can be set for each server that is to be monitored. + // When Butler SOS is started, do the following checks: + // 1. All tags present in `Butler-SOS.serversToMonitor.serverTagsDefinition` must be set for each server in `SOS.serversToMonitor.servers[]` + // 2. The tags specified for each server in `SOS.serversToMonitor.servers[].serverTags` must be present in `Butler-SOS.serversToMonitor.serverTagsDefinition` + // If either of the conditions above is false, an error should be logged and Butler SOS should not start. + try { + + // Loop over all defined server tags + const serverTagsDefinition = config.get( + 'Butler-SOS.serversToMonitor.serverTagsDefinition' + ); + // eslint-disable-next-line no-restricted-syntax + for (const tag of serverTagsDefinition) { + // Check that all servers have this tag + const servers = config.get('Butler-SOS.serversToMonitor.servers'); + // eslint-disable-next-line no-restricted-syntax + for (const server of servers) { + // Check if server.serverTags.tag is defined + if (server?.serverTags === null || !server?.serverTags[tag]) { + logger.error( + `VERIFY CONFIG FILE: Server tag "${tag}" is not defined for server "${server.serverName}". Exiting.` + ); + process.exit(1); + } else { + logger.verbose( + `VERIFY CONFIG FILE: Server tag "${tag}" is defined for server "${server.serverName}".` + ); + } + } + } + + // Now ensure that the tags defined for each server are valid and that there are no extra tags there + const servers = config.get('Butler-SOS.serversToMonitor.servers'); + // eslint-disable-next-line no-restricted-syntax + for (const server of servers) { + // eslint-disable-next-line no-restricted-syntax + for (const tag in server.serverTags) { + if (!serverTagsDefinition.includes(tag)) { + logger.error( + `VERIFY CONFIG FILE: Server tag "${tag}" for server "${server.serverName}" is not defined in Butler-SOS.serversToMonitor.serverTagsDefinition. Exiting.` + ); + process.exit(1); + } else { + logger.verbose( + `VERIFY CONFIG FILE: Server tag "${tag}" is defined in Butler-SOS.serversToMonitor.serverTagsDefinition.` + ); + } + } + } + } catch (err) { + logger.error(`VERIFY CONFIG FILE: Server tags verification failed. ${err}`); + process.exit(1); + } + logger.info(`VERIFY CONFIG FILE: Your config file at ${configFile} is valid, good work!`); return 'boolean'; From 45312d7e565c07381c7c149361c8a7c9d240085b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Tue, 4 Jun 2024 09:53:38 +0000 Subject: [PATCH 5/5] fix(influxdb): Storing proxy session data in InfluxDB v1 works again Fixes #816 --- src/globals.js | 22 +++++++++++++++++++++- src/lib/proxysessionmetrics.js | 7 ++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/globals.js b/src/globals.js index 886cc7d4..d322ddb2 100755 --- a/src/globals.js +++ b/src/globals.js @@ -322,6 +322,11 @@ const tagValuesLogEventLogDb = tagValues.slice(); tagValuesLogEventLogDb.push('source_process'); tagValuesLogEventLogDb.push('log_level'); +// Create tags for user sessions +const tagValuesUserProxySessions = tagValues.slice(); +tagValuesUserProxySessions.push('user_session_virtual_proxy'); +tagValuesUserProxySessions.push('user_session_host'); + // Show Influxdb config if (config.get('Butler-SOS.influxdbConfig.enable') === true) { logger.info(`CONFIG: Influxdb enabled: true`); @@ -358,7 +363,7 @@ if (config.get('Butler-SOS.influxdbConfig.enable') === true) { // Set up Influxdb client let influx; -let influxWriteApi = []; +const influxWriteApi = []; if (config.get('Butler-SOS.influxdbConfig.enable') === true) { if (config.get('Butler-SOS.influxdbConfig.version') === 1) { // Set up Influxdb v1 client @@ -483,6 +488,21 @@ if (config.get('Butler-SOS.influxdbConfig.enable') === true) { }, tags: ['butler_sos_instance', 'version'], }, + { + measurement: 'user_session_summary', + fields: { + session_count: Influx.FieldType.INTEGER, + session_user_id_list: Influx.FieldType.STRING, + }, + tags: tagValuesUserProxySessions, + }, + { + measurement: 'user_session_list', + fields: { + session_user_id_list: Influx.FieldType.STRING, + }, + tags: tagValuesUserProxySessions, + }, // { // measurement: 'user_events', // fields: { diff --git a/src/lib/proxysessionmetrics.js b/src/lib/proxysessionmetrics.js index 37d987ff..8dc0b4b8 100755 --- a/src/lib/proxysessionmetrics.js +++ b/src/lib/proxysessionmetrics.js @@ -40,11 +40,8 @@ function prepUserSessionMetrics(serverName, host, virtualProxy, body, tags) { // Build tags structure, adding tags for virtual proxy and host the session is associated with // Start with common/shared set of tags, then add user session specific tags userProxySessionsData.tags = { ...tags }; - // Esnure the virtual proxy does not begin or end with a slash or backslash - userProxySessionsData.tags.user_session_virtual_proxy = virtualProxy.replace( - /^[/\\]+|[/\\]+$/g, - '' - ); + + userProxySessionsData.tags.user_session_virtual_proxy = virtualProxy; userProxySessionsData.tags.user_session_host = host; // Build comma separated list of all user IDs connected via the current virtual proxy