From 86dc2369f4c6bd80f3ef2ad2d43d858b9b112410 Mon Sep 17 00:00:00 2001 From: prlanzarin <4529051+prlanzarin@users.noreply.github.com> Date: Tue, 31 May 2022 02:02:05 +0000 Subject: [PATCH 1/2] feat: add base Prometheus instrumentation Instrument the application to export basic operational metrics alongside prom-client's default metrics. Current set of exposed metrics as well as instructions on how to enable it can be checked in /docs/prometheus.md. --- config/settings.json.template | 8 ++ docs/prometheus.md | 51 ++++++++++ index.js | 2 + lib/etherpad/api.js | 19 +++- lib/utils/http-server.js | 43 +++++++++ lib/utils/prometheus/index.js | 77 +++++++++++++++ lib/utils/prometheus/prometheus-agent.js | 113 +++++++++++++++++++++++ package-lock.json | 46 +++++++++ package.json | 1 + 9 files changed, 358 insertions(+), 2 deletions(-) create mode 100644 docs/prometheus.md create mode 100644 lib/utils/http-server.js create mode 100644 lib/utils/prometheus/index.js create mode 100644 lib/utils/prometheus/prometheus-agent.js diff --git a/config/settings.json.template b/config/settings.json.template index 824d52a..8fc0aed 100644 --- a/config/settings.json.template +++ b/config/settings.json.template @@ -41,5 +41,13 @@ "from-etherpad-redis-channel" ] } + }, + "prometheus": { + "enabled": false, + "host": "localhost", + "port": 9003, + "path": "/metrics", + "collectCustomMetrics": true, + "collectDefaultMetrics": true } } diff --git a/docs/prometheus.md b/docs/prometheus.md new file mode 100644 index 0000000..2d1ffbd --- /dev/null +++ b/docs/prometheus.md @@ -0,0 +1,51 @@ +# Prometheus metrics + +bbb-pads provides direct Prometheus instrumentation for monitoring purposes. +The instrumentation code is **disabled by default**. + +The underlying mechanisms of the Prometheus client as well as the default Node.js +metrics come from https://github.com/siimon/prom-client. + +## Enabling instrumentation + +It can be enabled via configuration file (settings.json). + +### Configuration file (settings.json) + +See the `prometheus` object in `/config/settings.json.template`. + +The default configuration is: + +```JSON5 +"prometheus": { + // Whether to enable or disable metric exporting altogether. + "enabled": false, + // host: scrape route host + "host": "localhost", + // port: scrape route port + "port": "9003", + // path: metrics endpoint path + "path": "/metrics", + // collectCustomMetrics: whether custom bbb-pads metrics should be exported + "collectCustomMetrics": true + // collectDefaultMetrics: whether default Node.js metrics should be exported + "collectDefaultMetrics": true +} +``` + +## Exposed metrics + +The custom metric set currently is: + +``` +# HELP bbb_pads_etherpad_requests_total Total Etherpad API requests +# TYPE bbb_pads_etherpad_requests_total counter +bbb_pads_etherpad_requests_total{method=""} 0 + +# HELP bbb_pads_etherpad_requests_errors_total Total Etherpad API request failures +# TYPE bbb_pads_etherpad_requests_errors_total counter +bbb_pads_etherpad_requests_errors_total{method=""} 0 + +``` + +The default Node.js metrics come from https://github.com/siimon/prom-client. diff --git a/index.js b/index.js index 64b0201..20f6095 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ const subscriber = require('./lib/redis/subscriber'); const monitor = require('./lib/utils/monitor'); const server = require('./lib/express/server'); const Logger = require('./lib/utils/logger'); +const prometheus = require('./lib/utils/prometheus'); const logger = new Logger('bbb-pads'); @@ -30,6 +31,7 @@ const start = () => { subscriber.start(); server.start(); monitor.start(); + prometheus.start(); }).catch(() => abort('key-mismatch')); }).catch((error) => { logger.warn('starting', error); diff --git a/lib/etherpad/api.js b/lib/etherpad/api.js index 26c2173..ad82619 100644 --- a/lib/etherpad/api.js +++ b/lib/etherpad/api.js @@ -5,6 +5,10 @@ const { } = require('./methods'); const config = require('../../config'); const Logger = require('../utils/logger'); +const { + registerAPIError, + registerAPICall, +} = require('../utils/prometheus'); const logger = new Logger('api'); @@ -41,7 +45,13 @@ const locked = (id) => { const call = (method, params = {}) => { return new Promise((resolve, reject) => { - if (!validate(method, params)) return reject(); + registerAPICall(method); + + if (!validate(method, params)) { + registerAPIError(method); + + return reject(); + } const id = buildId(method, params); if (locked(id)) return reject(); @@ -55,6 +65,7 @@ const call = (method, params = {}) => { const { status } = response; if (status !== 200) { logger.error('call', { status }); + registerAPIError(method); return reject(); } @@ -67,6 +78,7 @@ const call = (method, params = {}) => { if (code !== 0) { logger.error('call', { message }); + registerAPIError(method); return reject(); } @@ -74,7 +86,10 @@ const call = (method, params = {}) => { logger.debug('call', { method, data }); resolve(data); - }).catch(() => reject()).finally(() => release(id)); + }).catch(() => { + registerAPIError(method); + reject(); + }).finally(() => release(id)); }); }; diff --git a/lib/utils/http-server.js b/lib/utils/http-server.js new file mode 100644 index 0000000..3989644 --- /dev/null +++ b/lib/utils/http-server.js @@ -0,0 +1,43 @@ +"use strict"; + +const http = require("http"); +const Logger = require('./logger.js'); + +const logger = new Logger('http-server'); + +module.exports = class HttpServer { + constructor(host, port, callback) { + this.host = host; + this.port = port; + this.requestCallback = callback; + } + + start () { + this.server = http.createServer(this.requestCallback) + .on('error', this.handleError.bind(this)) + .on('clientError', this.handleError.bind(this)); + } + + close (callback) { + return this.server.close(callback); + } + + handleError (error) { + if (error.code === 'EADDRINUSE') { + logger.error('EADDRINUSE', { host: this.host, port: this.port }); + this.server.close(); + } else if (error.code === 'ECONNRESET') { + Logger.warn('ECONNRESET'); + } else { + Logger.error('failure', { errorMessage: error.message, errorCode: error.code }); + } + } + + getServerObject() { + return this.server; + } + + listen(callback) { + this.server.listen(this.port, this.host, callback); + } +} diff --git a/lib/utils/prometheus/index.js b/lib/utils/prometheus/index.js new file mode 100644 index 0000000..763e083 --- /dev/null +++ b/lib/utils/prometheus/index.js @@ -0,0 +1,77 @@ +const config = require('../../../config'); +const PrometheusAgent = require('./prometheus-agent.js'); +const { Counter } = require('prom-client'); +const Logger = require('../logger.js'); + +const logger = new Logger('prometheus'); +const { prometheus = {} } = config; +const PREFIX = 'bbb_pads_'; +const PROM_NAMES = { + ETH_REQS_TOTAL: `${PREFIX}etherpad_requests_total`, + ETH_REQS_ERRORS: `${PREFIX}etherpad_requests_errors_total`, +} +const { + enabled: PROM_ENABLED = false, + host: PROM_HOST = 'localhost', + port: PROM_PORT = '9003', + path: PROM_PATH = '/metrics', + collectDefaultMetrics: COLLECT_DEFAULT_METRICS = false, + collectCustomMetrics: COLLECT_CUSTOM_METRICS = false, +} = prometheus; +const PADSPrometheusAgent = new PrometheusAgent(PROM_HOST, PROM_PORT, { + path: PROM_PATH, + prefix: PREFIX, + collectDefaultMetrics: COLLECT_DEFAULT_METRICS, +}); + +let PADS_METRICS; +const _buildDefaultMetrics = () => { + if (PADS_METRICS == null) { + PADS_METRICS = { + [PROM_NAMES.ETH_REQS_TOTAL]: new Counter({ + name: PROM_NAMES.ETH_REQS_TOTAL, + help: 'Total Etherpad API requests', + labelNames: ['method'], + }), + [PROM_NAMES.ETH_REQS_ERRORS]: new Counter({ + name: PROM_NAMES.ETH_REQS_ERRORS, + help: 'Total Etherpad API request failures', + labelNames: ['method'], + }), + } + } + + return PADS_METRICS; +}; + +const start = () => { + if (PROM_ENABLED) { + try { + if (COLLECT_CUSTOM_METRICS) { + PADSPrometheusAgent.injectMetrics(_buildDefaultMetrics()); + } + + PADSPrometheusAgent.start(); + } catch (error) { + logger.error('prometheus-startup', { + errorCode: error.code, errorMessage: error.message + }); + } + } +} + +const registerAPIError = (method) => { + if (method == null) return; + PADSPrometheusAgent.increment(PROM_NAMES.ETH_REQS_ERRORS, { method }); +}; + +const registerAPICall = (method) => { + if (method == null) return; + PADSPrometheusAgent.increment(PROM_NAMES.ETH_REQS_TOTAL, { method }); +} + +module.exports = { + start, + registerAPIError, + registerAPICall, +}; diff --git a/lib/utils/prometheus/prometheus-agent.js b/lib/utils/prometheus/prometheus-agent.js new file mode 100644 index 0000000..ce14015 --- /dev/null +++ b/lib/utils/prometheus/prometheus-agent.js @@ -0,0 +1,113 @@ +const { + register, + collectDefaultMetrics, +} = require('prom-client'); +const HTTPServer = require('../http-server.js'); +const Logger = require('../logger.js'); + +const logger = new Logger('prometheus'); + +module.exports = class PrometheusScrapeAgent { + constructor (host, port, options) { + this.host = host; + this.port = port; + this.metrics = {}; + this.started = false; + + this.path = options.path || '/metrics'; + this.collectDefaultMetrics = options.collectDefaultMetrics || false; + this.metricsPrefix = options.prefix || ''; + this.collectionTimeout = options.collectionTimeout || 10000; + } + + getMetric (name) { + return this.metrics[name]; + } + + async collect (response) { + try { + response.writeHead(200, { 'Content-Type': register.contentType }); + const content = await register.metrics(); + response.end(content); + } catch (error) { + response.writeHead(500) + response.end(error.message); + logger.error('collecting-metrics', { + errorCode: error.code, errorMessage: error.message + }); + } + } + + getMetricsHandler (request, response) { + switch (request.method) { + case 'GET': + if (request.url === this.path) return this.collect(response); + response.writeHead(404).end(); + break; + default: + response.writeHead(501) + response.end(); + break; + } + } + + start (requestHandler = this.getMetricsHandler.bind(this)) { + if (this.collectDefaultMetrics) collectDefaultMetrics({ + prefix: this.metricsPrefix, + timeout: this.collectionTimeout, + }); + + this.metricsServer = new HTTPServer(this.host, this.port, requestHandler); + this.metricsServer.start(); + this.metricsServer.listen(); + this.started = true; + } + + injectMetrics (metricsDictionary) { + this.metrics = { ...this.metrics, ...metricsDictionary } + } + + increment (metricName, labelsObject) { + if (!this.started) return; + + const metric = this.metrics[metricName]; + if (metric) { + metric.inc(labelsObject) + } + } + + decrement (metricName, labelsObject) { + if (!this.started) return; + + const metric = this.metrics[metricName]; + if (metric) { + metric.dec(labelsObject) + } + } + + set (metricName, value, labelsObject = {}) { + if (!this.started) return; + + const metric = this.metrics[metricName]; + if (metric) { + metric.set(labelsObject, value) + } + } + + setCollectorWithGenerator (metricName, generator) { + const metric = this.getMetric(metricName); + if (metric) { + metric.collect = () => { + metric.set(generator()); + }; + } + } + + setCollector (metricName, collector) { + const metric = this.getMetric(metricName); + + if (metric) { + metric.collect = collector.bind(metric); + } + } +} diff --git a/package-lock.json b/package-lock.json index 0885d32..1a6d31c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "express": "^4.17.3", "http-proxy": "^1.18.1", "lodash": "^4.17.21", + "prom-client": "^14.0.1", "redis": "^3.1.2" }, "devDependencies": { @@ -1371,6 +1372,11 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" + }, "node_modules/body-parser": { "version": "1.19.2", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz", @@ -4138,6 +4144,17 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/prom-client": { + "version": "14.0.1", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-14.0.1.tgz", + "integrity": "sha512-HxTArb6fkOntQHoRGvv4qd/BkorjliiuO2uSWC2KC17MUTKYttWdDoXX/vxOhQdkoECEM9BBH0pj2l8G8kev6w==", + "dependencies": { + "tdigest": "^0.1.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -4669,6 +4686,14 @@ "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "dev": true }, + "node_modules/tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/terminal-link": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", @@ -6167,6 +6192,11 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" + }, "body-parser": { "version": "1.19.2", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz", @@ -8276,6 +8306,14 @@ } } }, + "prom-client": { + "version": "14.0.1", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-14.0.1.tgz", + "integrity": "sha512-HxTArb6fkOntQHoRGvv4qd/BkorjliiuO2uSWC2KC17MUTKYttWdDoXX/vxOhQdkoECEM9BBH0pj2l8G8kev6w==", + "requires": { + "tdigest": "^0.1.1" + } + }, "prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -8668,6 +8706,14 @@ "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "dev": true }, + "tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "requires": { + "bintrees": "1.0.2" + } + }, "terminal-link": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", diff --git a/package.json b/package.json index 6a1efe8..8dd0e08 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "express": "^4.17.3", "http-proxy": "^1.18.1", "lodash": "^4.17.21", + "prom-client": "^14.0.1", "redis": "^3.1.2" }, "devDependencies": { From 478d48c1c682542ea754a13230736c12bdbc7c9e Mon Sep 17 00:00:00 2001 From: prlanzarin <4529051+prlanzarin@users.noreply.github.com> Date: Fri, 12 Aug 2022 13:30:14 +0000 Subject: [PATCH 2/2] 1.3.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index b438ac4..40463aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "bbb-pads", - "version": "1.2.2", + "version": "1.3.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "bbb-pads", - "version": "1.2.2", + "version": "1.3.0", "license": "LGPL-3.0", "dependencies": { "@mconf/bbb-diff": "^1.2.0", diff --git a/package.json b/package.json index 0638e79..ce24acb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bbb-pads", - "version": "1.2.2", + "version": "1.3.0", "description": "BigBlueButton's pads manager", "engines": { "node": ">=16"