From 2ce4a94f6ae1dcb3d0519d0c47b49dbd77463d21 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 20 Sep 2023 21:22:36 +0200 Subject: [PATCH] Add `explore dashboards --data-details` option This extends the output by many more details about data inquiry/queries. --- CHANGES.rst | 2 + README.rst | 6 +++ grafana_wtf/commands.py | 10 ++++- grafana_wtf/core.py | 6 +-- grafana_wtf/model.py | 90 ++++++++++++++++++++++++++++++++++++++--- tests/test_commands.py | 33 ++++++++++++++- 6 files changed, 135 insertions(+), 12 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 65d0f46..e8c53a9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,8 @@ in progress - CI: Update to Grafana 8.5.27, 9.5.8, and 10.1.1 - Grafana 9.3: Work around delete folder operation returning empty body - Grafana 9.5: Use standard UUIDs instead of short UIDs +- Add ``explore dashboards --data-details`` option, to extend the output + by many more details about data inquiry / queries. Thanks, @meyerder. 2023-07-30 0.15.2 ================= diff --git a/README.rst b/README.rst index 405e48d..2c16b3b 100644 --- a/README.rst +++ b/README.rst @@ -154,6 +154,12 @@ How to find dashboards which use non-existing data sources? # Display only dashboards which have missing data sources, along with their names. grafana-wtf explore dashboards --format=json | jq '.[] | select( .datasources_missing ) | .dashboard + {ds_missing: .datasources_missing[] | [.name]}' +How to list all queries used in all dashboards? +:: + + grafana-wtf explore dashboards --data-details --format=json | \ + jq -r '.[].details | values[] | .[].query // "null"' + Searching for strings ===================== diff --git a/grafana_wtf/commands.py b/grafana_wtf/commands.py index 654d863..b12af66 100644 --- a/grafana_wtf/commands.py +++ b/grafana_wtf/commands.py @@ -29,7 +29,7 @@ def run(): Usage: grafana-wtf [options] info grafana-wtf [options] explore datasources - grafana-wtf [options] explore dashboards + grafana-wtf [options] explore dashboards [--data-details] grafana-wtf [options] find [] grafana-wtf [options] replace [--dry-run] grafana-wtf [options] log [] [--number=] [--head=] [--tail=] [--reverse] [--sql=] @@ -91,6 +91,12 @@ def run(): # Display all dashboards using data sources with a specific type. Here: InfluxDB. grafana-wtf explore dashboards --format=json | jq 'select( .[] | .datasources | .[].type=="influxdb" )' + # Display dashboards and many more details about where data source queries are happening. + # Specifically, within "panels/targets", "annotations", and "templating" slots. + grafana-wtf explore dashboards --data-details --format=json + + # Display all database queries within dashboards. + grafana-wtf explore dashboards --data-details --format=json | jq -r '.[].details | values[] | .[].query // "null"' Find dashboards and data sources: @@ -298,7 +304,7 @@ def run(): output_results(output_format, results) if options.explore and options.dashboards: - results = engine.explore_dashboards() + results = engine.explore_dashboards(with_data_details=options.data_details) output_results(output_format, results) if options.info: diff --git a/grafana_wtf/core.py b/grafana_wtf/core.py index 8bacf5d..a7f980c 100644 --- a/grafana_wtf/core.py +++ b/grafana_wtf/core.py @@ -447,7 +447,7 @@ def explore_datasources(self): return response - def explore_dashboards(self): + def explore_dashboards(self, with_data_details: bool = False): # Prepare indexes, mapping dashboards by uid, datasources by name # as well as dashboards to datasources and vice versa. ix = Indexer(engine=self) @@ -481,8 +481,8 @@ def explore_dashboards(self): dashboard=dashboard, datasources=datasources_existing, grafana_url=self.grafana_url ) - # Format results in a more compact form, using only a subset of all the attributes. - result = item.format_compact() + # Format results, using only a subset of all the attributes. + result = item.format(with_data_details=with_data_details) # Add information about missing data sources. if datasources_missing: diff --git a/grafana_wtf/model.py b/grafana_wtf/model.py index 963867c..23bc5c8 100644 --- a/grafana_wtf/model.py +++ b/grafana_wtf/model.py @@ -91,21 +91,99 @@ class DashboardExplorationItem: datasources: List[Munch] grafana_url: str - def format_compact(self): + def format(self, with_data_details: bool = False): + """ + Generate a representation from selected information. + + - dashboard + - datasources + - details + - panels/targets + - annotations + - templating + """ dbshort = OrderedDict( title=self.dashboard.dashboard.title, uid=self.dashboard.dashboard.uid, path=self.dashboard.meta.url, url=urljoin(self.grafana_url, self.dashboard.meta.url), ) - item = OrderedDict(dashboard=dbshort) + + dsshort = [] for datasource in self.datasources: - item.setdefault("datasources", []) - dsshort = OrderedDict( + item = OrderedDict( uid=datasource.get("uid"), name=datasource.name, type=datasource.type, url=datasource.url, ) - item["datasources"].append(dsshort) - return item + dsshort.append(item) + + data = Munch(dashboard=dbshort, datasources=dsshort) + if with_data_details: + data.details = self.collect_data_details() + return data + + def collect_data_details(self): + """ + Collect details concerned about data from dashboard information. + """ + + dbdetails = DashboardDetails(dashboard=self.dashboard) + + ds_panels = self.collect_data_nodes(dbdetails.panels) + ds_annotations = self.collect_data_nodes(dbdetails.annotations) + ds_templating = self.collect_data_nodes(dbdetails.templating) + + targets = [] + for panel in ds_panels: + panel_item = self._format_panel_compact(panel) + for target in panel.targets: + target["_panel"] = panel_item + targets.append(target) + + response = OrderedDict(targets=targets, annotations=ds_annotations, templating=ds_templating) + + return response + + @staticmethod + def collect_data_nodes(element): + """ + Select all element nodes which have a "datasource" attribute. + """ + element = element or [] + items = [] + + def add(item): + if item is not None and item not in items: + items.append(item) + + for node in element: + if "datasource" in node and node["datasource"]: + add(node) + + return items + + @staticmethod + def _format_panel_compact(panel): + """ + Return a compact representation of panel information. + """ + attributes = ["id", "title", "type", "datasource"] + data = OrderedDict() + for attribute in attributes: + data[attribute] = panel.get(attribute) + return data + + @staticmethod + def _format_data_node_compact(item: Dict) -> Dict: + """ + Return a compact representation of an element concerned about data. + """ + data = OrderedDict() + data["datasource"] = item.get("datasource") + data["type"] = item.get("type") + for key, value in item.items(): + if "query" in key.lower(): + data[key] = value + return data diff --git a/tests/test_commands.py b/tests/test_commands.py index e67f1e2..0ff5595 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,5 +1,6 @@ import warnings +from munch import munchify from packaging import version warnings.filterwarnings("ignore", category=DeprecationWarning, module=".*docopt.*") @@ -463,6 +464,36 @@ def test_explore_dashboards_grafana7up(grafana_version, ldi_resources, capsys, c assert dashboard["datasources_missing"][0]["type"] is None +def test_explore_dashboards_data_details(ldi_resources, capsys, caplog): + """ + Explore more details of dashboards, wrt. to data and queries. + """ + + # Only provision specific dashboard. + ldi_resources(dashboards=["tests/grafana/dashboards/ldi-v33.json"]) + + # Compute exploration. + set_command("explore dashboards --data-details", "--format=yaml") + + # Run command and capture YAML output. + with caplog.at_level(logging.DEBUG): + grafana_wtf.commands.run() + captured = capsys.readouterr() + data = yaml.safe_load(captured.out) + + # Proof the output is correct. + assert len(data) == 1 + dashboard = munchify(data[0]) + assert dashboard.details.targets[0]._panel.id == 18 + assert dashboard.details.targets[0]._panel.type == "graph" + assert dashboard.details.targets[0]._panel.datasource.type == "influxdb" + assert dashboard.details.targets[0]._panel.datasource.uid == "PDF2762CDFF14A314" + assert dashboard.details.targets[0].fields == [{'func': 'mean', 'name': 'P1'}] + assert dashboard.details.templating[0].query == \ + "SELECT osm_country_code AS __value, country_and_countrycode AS __text " \ + "FROM ldi_network ORDER BY osm_country_code" + + def test_explore_dashboards_empty_annotations(grafana_version, create_datasource, create_dashboard, capsys, caplog): # Create a dashboard with an anomalous value in the "annotations" slot. dashboard = mkdashboard(title="foo") @@ -488,7 +519,7 @@ def test_explore_dashboards_empty_annotations(grafana_version, create_datasource assert len(dashboard["dashboard"]["uid"]) == 36 else: assert len(dashboard["dashboard"]["uid"]) == 9 - assert "datasources" not in dashboard + assert dashboard["datasources"] == [] assert "datasources_missing" not in dashboard