From 83424af8d17dbb65f23c049e40ee777362894a33 Mon Sep 17 00:00:00 2001 From: Kevin Petremann Date: Sun, 25 Jun 2023 10:57:51 +0200 Subject: [PATCH 1/3] fix: crash when metric_name contains invalid chars Prometheus metric name must match the regex [a-zA-Z_:][a-zA-Z0-9_:]* Before this commit, an invalid metric name was causing a crash. https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels --- mqtt_exporter/main.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mqtt_exporter/main.py b/mqtt_exporter/main.py index 5dc8709..51c321f 100644 --- a/mqtt_exporter/main.py +++ b/mqtt_exporter/main.py @@ -68,10 +68,13 @@ def _create_prometheus_metric(prom_metric_name): if settings.MQTT_EXPOSE_CLIENT_ID: labels.append("client_id") - prom_metrics[prom_metric_name] = Gauge( - prom_metric_name, "metric generated from MQTT message.", labels - ) - LOG.info("creating prometheus metric: %s", prom_metric_name) + try: + prom_metrics[prom_metric_name] = Gauge( + prom_metric_name, "metric generated from MQTT message.", labels + ) + LOG.info("creating prometheus metric: %s", prom_metric_name) + except ValueError as error: + LOG.error("unable to create prometheus metric '%s': %s", prom_metric_name, error) def _add_prometheus_sample(topic, prom_metric_name, metric_value, client_id): From ed110bb9d7b2f3ad0331cb0fdbfa3cc510980228 Mon Sep 17 00:00:00 2001 From: Kevin Petremann Date: Sun, 25 Jun 2023 11:16:47 +0200 Subject: [PATCH 2/3] feat: normalize prometheus metric name Ensure the Prometheus metric name are respecting Prometheus convention. --- mqtt_exporter/main.py | 22 +++++++++++++++++++- tests/unit/__init__.py | 1 + tests/unit/test_normalize_prometheus_name.py | 15 +++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tests/unit/__init__.py create mode 100644 tests/unit/test_normalize_prometheus_name.py diff --git a/mqtt_exporter/main.py b/mqtt_exporter/main.py index 51c321f..324f8ec 100644 --- a/mqtt_exporter/main.py +++ b/mqtt_exporter/main.py @@ -9,7 +9,7 @@ import sys import paho.mqtt.client as mqtt -from prometheus_client import Counter, Gauge, start_http_server +from prometheus_client import Counter, Gauge, metrics, start_http_server from mqtt_exporter import settings @@ -61,6 +61,24 @@ def subscribe(client, _, __, result_code, *args): LOG.error("MQTT %s", mqtt.connack_string(result_code)) +def _normalize_prometheus_metric_name(prom_metric_name): + """Transform an invalid prometheus metric to a valid one. + + https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels + """ + if metrics.METRIC_NAME_RE.match(prom_metric_name): + return prom_metric_name + + # clean invalid characted + prom_metric_name = re.sub(r"[^a-zA-Z0-9_:]", "", prom_metric_name) + + # ensure to start with valid character + if not re.match(r"^[a-zA-Z_:]", prom_metric_name): + prom_metric_name = ":" + prom_metric_name + + return prom_metric_name + + def _create_prometheus_metric(prom_metric_name): """Create Prometheus metric if does not exist.""" if not prom_metrics.get(prom_metric_name): @@ -68,6 +86,8 @@ def _create_prometheus_metric(prom_metric_name): if settings.MQTT_EXPOSE_CLIENT_ID: labels.append("client_id") + prom_metric_name = _normalize_prometheus_metric_name(prom_metric_name) + try: prom_metrics[prom_metric_name] = Gauge( prom_metric_name, "metric generated from MQTT message.", labels diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e0310a0 --- /dev/null +++ b/tests/unit/__init__.py @@ -0,0 +1 @@ +"""Unit tests.""" diff --git a/tests/unit/test_normalize_prometheus_name.py b/tests/unit/test_normalize_prometheus_name.py new file mode 100644 index 0000000..5c875ee --- /dev/null +++ b/tests/unit/test_normalize_prometheus_name.py @@ -0,0 +1,15 @@ +"""Tests of Prometheus normalization metrics.""" +from mqtt_exporter.main import _normalize_prometheus_metric_name + + +def test_normalize_prometheus_metric_name(): + """Test _normalize_prometheus_metric_name.""" + tests = { + "1234invalid": ":1234invalid", + "valid1234": "valid1234", + "_this_is_valid": "_this_is_valid", + "not_so_valid%_name": "not_so_valid_name", + } + + for candidate, wanted in tests.items(): + assert _normalize_prometheus_metric_name(candidate) == wanted From 97d54b560b636a3a876dfd31307a2d3fb1a69e93 Mon Sep 17 00:00:00 2001 From: Kevin Petremann Date: Sun, 25 Jun 2023 13:55:06 +0200 Subject: [PATCH 3/3] fix: keyError when Prometheuse metric name is normalized prom_metric_name was only changed for _create_prometheus_metric and not _add_prometheus_sample. --- mqtt_exporter/main.py | 23 +++++++++++++---------- tests/functional/test_expose_metrics.py | 4 ++-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/mqtt_exporter/main.py b/mqtt_exporter/main.py index 324f8ec..3a50dc4 100644 --- a/mqtt_exporter/main.py +++ b/mqtt_exporter/main.py @@ -86,18 +86,16 @@ def _create_prometheus_metric(prom_metric_name): if settings.MQTT_EXPOSE_CLIENT_ID: labels.append("client_id") - prom_metric_name = _normalize_prometheus_metric_name(prom_metric_name) - - try: - prom_metrics[prom_metric_name] = Gauge( - prom_metric_name, "metric generated from MQTT message.", labels - ) - LOG.info("creating prometheus metric: %s", prom_metric_name) - except ValueError as error: - LOG.error("unable to create prometheus metric '%s': %s", prom_metric_name, error) + prom_metrics[prom_metric_name] = Gauge( + prom_metric_name, "metric generated from MQTT message.", labels + ) + LOG.info("creating prometheus metric: %s", prom_metric_name) def _add_prometheus_sample(topic, prom_metric_name, metric_value, client_id): + if prom_metric_name not in prom_metrics: + return + labels = {settings.TOPIC_LABEL: topic} if settings.MQTT_EXPOSE_CLIENT_ID: labels["client_id"] = client_id @@ -165,7 +163,12 @@ def _parse_metrics(data, topic, client_id, prefix=""): .replace("/", "_") ) prom_metric_name = re.sub(r"\((.*?)\)", "", prom_metric_name) - _create_prometheus_metric(prom_metric_name) + prom_metric_name = _normalize_prometheus_metric_name(prom_metric_name) + try: + _create_prometheus_metric(prom_metric_name) + except ValueError as error: + LOG.error("unable to create prometheus metric '%s': %s", prom_metric_name, error) + return # expose the sample to prometheus _add_prometheus_sample(topic, prom_metric_name, metric_value, client_id) diff --git a/tests/functional/test_expose_metrics.py b/tests/functional/test_expose_metrics.py index 9dd931a..2924317 100644 --- a/tests/functional/test_expose_metrics.py +++ b/tests/functional/test_expose_metrics.py @@ -20,10 +20,10 @@ def _exec(client_id, mocker): userdata = {"client_id": client_id} msg = mocker.Mock() msg.topic = "zigbee2mqtt/garage" - msg.payload = '{"temperature": "23.5", "humidity": "40.5"}' + msg.payload = '{"temperature°C": "23.5", "humidity": "40.5"}' main.expose_metrics(None, userdata, msg) - temperatures = main.prom_metrics["mqtt_temperature"].collect() + temperatures = main.prom_metrics["mqtt_temperatureC"].collect() humidity = main.prom_metrics["mqtt_humidity"].collect() return temperatures, humidity