From 1bd0881c5876af9e764cab805dd0268490936256 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 15 Oct 2023 21:06:18 +0200 Subject: [PATCH] Add `--queries-only` option to `explore dashboards` subcommand --- CHANGES.rst | 1 + README.rst | 4 +- grafana_wtf/commands.py | 8 +- grafana_wtf/core.py | 6 +- grafana_wtf/model.py | 183 ++++++++++++++++++++++++++-------------- tests/test_commands.py | 10 +-- 6 files changed, 138 insertions(+), 74 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 510decf..f21e1e0 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,7 @@ in progress - Fix wrong ``jq`` commands in documentation. Thanks, @rahulnandan. - Fix collecting data information from dashboards w/o ``targets`` slots in panels +- Add ``--queries-only`` option to ``explore dashboards`` subcommand 2023-10-03 0.16.0 ================= diff --git a/README.rst b/README.rst index 2139a77..0bceddc 100644 --- a/README.rst +++ b/README.rst @@ -206,8 +206,8 @@ How to find dashboards using specific data sources? How to list all queries used in all dashboards? :: - grafana-wtf explore dashboards --data-details --format=json | \ - jq -r '.[].details | values[] | .[].query // "null"' + grafana-wtf explore dashboards --data-details --queries-only --format=json | \ + jq '.[].details | values[] | .[] | .expr,.jql,.query,.rawSql | select( . != null and . != "" )' Searching for strings diff --git a/grafana_wtf/commands.py b/grafana_wtf/commands.py index 2e690bb..f4fcbbd 100644 --- a/grafana_wtf/commands.py +++ b/grafana_wtf/commands.py @@ -33,7 +33,7 @@ def run(): Usage: grafana-wtf [options] info grafana-wtf [options] explore datasources - grafana-wtf [options] explore dashboards [--data-details] + grafana-wtf [options] explore dashboards [--data-details] [--queries-only] grafana-wtf [options] find [] grafana-wtf [options] replace [--dry-run] grafana-wtf [options] log [] [--number=] [--head=] [--tail=] [--reverse] [--sql=] @@ -98,11 +98,11 @@ def run(): grafana-wtf explore dashboards --format=json | jq '.[] | select(.datasources | .[].type=="influxdb")' # Display dashboards and many more details about where data source queries are happening. - # Specifically, within "panels/targets", "annotations", and "templating" slots. + # Specifically, within "panels", "annotations", and "templating" slots. grafana-wtf explore dashboards --data-details --format=json # Display all database queries within dashboards. - grafana-wtf explore dashboards --data-details --format=json | jq -r '.[].details | values[] | .[].query // "null"' + grafana-wtf explore dashboards --data-details --queries-only --format=json | jq '.[].details | values[] | .[] | .expr,.jql,.query,.rawSql | select( . != null and . != "" )' Find dashboards and data sources: @@ -315,7 +315,7 @@ def run(): output_results(output_format, results) if options.explore and options.dashboards: - results = engine.explore_dashboards(with_data_details=options.data_details) + results = engine.explore_dashboards(with_data_details=options.data_details, queries_only=options.queries_only) output_results(output_format, results) if options.info: diff --git a/grafana_wtf/core.py b/grafana_wtf/core.py index 91d1487..4f831eb 100644 --- a/grafana_wtf/core.py +++ b/grafana_wtf/core.py @@ -447,7 +447,7 @@ def explore_datasources(self): return response - def explore_dashboards(self, with_data_details: bool = False): + def explore_dashboards(self, with_data_details: bool = False, queries_only: bool = False): # Prepare indexes, mapping dashboards by uid, datasources by name # as well as dashboards to datasources and vice versa. ix = Indexer(engine=self) @@ -482,7 +482,9 @@ def explore_dashboards(self, with_data_details: bool = False): ) # Format results, using only a subset of all the attributes. - result = item.format(with_data_details=with_data_details) + result = item.format(with_data_details=with_data_details, queries_only=queries_only) + if result is None: + continue # Add information about missing data sources. if datasources_missing: diff --git a/grafana_wtf/model.py b/grafana_wtf/model.py index 6d26be6..fd27fe4 100644 --- a/grafana_wtf/model.py +++ b/grafana_wtf/model.py @@ -44,6 +44,118 @@ def templating(self) -> List: return self.dashboard.dashboard.get("templating", {}).get("list", []) +@dataclasses.dataclass +class DashboardDataDetails: + """ + Manage details concerned about "data"-relevant information, + a subset of the complete bunch of dashboard information. + """ + + panels: List[Dict] + annotations: List[Dict] + templating: List[Dict] + + def to_munch(self): + return Munch(panels=self.panels, annotations=self.annotations, templating=self.templating) + + @classmethod + def from_dashboard_details(cls, dbdetails: DashboardDetails): + ds_panels = cls.collect_data_nodes(dbdetails.panels) + ds_annotations = cls.collect_data_nodes(dbdetails.annotations) + ds_templating = cls.collect_data_nodes(dbdetails.templating) + + # Inline panel information into data details. + targets = [] + for panel in ds_panels: + panel_item = cls._format_panel_compact(panel) + if "targets" in panel: + for target in panel.targets: + target["_panel"] = panel_item + targets.append(target) + + return cls(panels=targets, annotations=ds_annotations, templating=ds_templating) + + @staticmethod + def collect_data_nodes(element): + """ + Select all element nodes which have a "datasource" attribute. + """ + element = element or [] + items = [] + + def add(item): + if item is not None and item not in items: + items.append(item) + + for node in element: + if "datasource" in node and node["datasource"]: + add(node) + + return items + + @staticmethod + def _format_panel_compact(panel): + """ + Return a compact representation of panel information. + """ + attributes = ["id", "title", "type", "datasource"] + data = OrderedDict() + for attribute in attributes: + data[attribute] = panel.get(attribute) + return data + + @staticmethod + def _format_data_node_compact(item: Dict) -> Dict: + """ + Return a compact representation of an element concerned about data. + """ + data = OrderedDict() + data["datasource"] = item.get("datasource") + data["type"] = item.get("type") + for key, value in item.items(): + if "query" in key.lower(): + data[key] = value + return data + + def queries_only(self): + """ + Return a representation of data details information, only where query expressions are present. + """ + # All attributes containing query-likes. + attributes_query_likes = ["expr", "jql", "query", "rawSql", "target"] + + attributes = [ + # Carry over datasource and panel references for informational purposes. + "datasource", + "_panel", + ] + attributes_query_likes + + def transform(section): + new_items = [] + for item in section: + new_item = OrderedDict() + for key, value in item.items(): + if key in attributes: + new_item[key] = value + if any(attr in attributes_query_likes for attr in new_item.keys()): + # Filter annotations without any value. + if "target" in new_item and isinstance(new_item["target"], dict): + if new_item["target"].get("type") == "dashboard": + continue + # Unnest items with nested "query" slot. + for slot in ["query", "target"]: + if slot in new_item and isinstance(new_item[slot], dict) and "query" in new_item[slot]: + new_item["query"] = new_item[slot]["query"] + new_items.append(new_item) + return new_items + + return DashboardDataDetails( + panels=transform(self.panels), + annotations=transform(self.annotations), + templating=transform(self.templating), + ) + + @dataclasses.dataclass class DatasourceItem: uid: Optional[str] = None @@ -94,7 +206,7 @@ class DashboardExplorationItem: datasources: List[Munch] grafana_url: str - def format(self, with_data_details: bool = False): + def format(self, with_data_details: bool = False, queries_only: bool = False): """ Generate a representation from selected information. @@ -124,70 +236,19 @@ def format(self, with_data_details: bool = False): data = Munch(dashboard=dbshort, datasources=dsshort) if with_data_details: - data.details = self.collect_data_details() + details = self.collect_data_details(queries_only=queries_only) + if not details.panels and not details.annotations and not details.templating: + return None + data.details = details.to_munch() return data - def collect_data_details(self): + def collect_data_details(self, queries_only: bool = False): """ Collect details concerned about data from dashboard information. """ dbdetails = DashboardDetails(dashboard=self.dashboard) - - ds_panels = self.collect_data_nodes(dbdetails.panels) - ds_annotations = self.collect_data_nodes(dbdetails.annotations) - ds_templating = self.collect_data_nodes(dbdetails.templating) - - targets = [] - for panel in ds_panels: - panel_item = self._format_panel_compact(panel) - if "targets" in panel: - for target in panel.targets: - target["_panel"] = panel_item - targets.append(target) - - response = OrderedDict(targets=targets, annotations=ds_annotations, templating=ds_templating) - - return response - - @staticmethod - def collect_data_nodes(element): - """ - Select all element nodes which have a "datasource" attribute. - """ - element = element or [] - items = [] - - def add(item): - if item is not None and item not in items: - items.append(item) - - for node in element: - if "datasource" in node and node["datasource"]: - add(node) - - return items - - @staticmethod - def _format_panel_compact(panel): - """ - Return a compact representation of panel information. - """ - attributes = ["id", "title", "type", "datasource"] - data = OrderedDict() - for attribute in attributes: - data[attribute] = panel.get(attribute) - return data - - @staticmethod - def _format_data_node_compact(item: Dict) -> Dict: - """ - Return a compact representation of an element concerned about data. - """ - data = OrderedDict() - data["datasource"] = item.get("datasource") - data["type"] = item.get("type") - for key, value in item.items(): - if "query" in key.lower(): - data[key] = value - return data + dbdatadetails = DashboardDataDetails.from_dashboard_details(dbdetails) + if queries_only: + dbdatadetails = dbdatadetails.queries_only() + return dbdatadetails diff --git a/tests/test_commands.py b/tests/test_commands.py index f7e78e6..eb641d8 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -487,11 +487,11 @@ def test_explore_dashboards_data_details(ldi_resources, capsys, caplog): # Proof the output is correct. assert len(data) == 1 dashboard = munchify(data[0]) - assert dashboard.details.targets[0]._panel.id == 18 - assert dashboard.details.targets[0]._panel.type == "graph" - assert dashboard.details.targets[0]._panel.datasource.type == "influxdb" - assert dashboard.details.targets[0]._panel.datasource.uid == "PDF2762CDFF14A314" - assert dashboard.details.targets[0].fields == [{"func": "mean", "name": "P1"}] + assert dashboard.details.panels[0]._panel.id == 18 + assert dashboard.details.panels[0]._panel.type == "graph" + assert dashboard.details.panels[0]._panel.datasource.type == "influxdb" + assert dashboard.details.panels[0]._panel.datasource.uid == "PDF2762CDFF14A314" + assert dashboard.details.panels[0].fields == [{"func": "mean", "name": "P1"}] assert ( dashboard.details.templating[0].query == "SELECT osm_country_code AS __value, country_and_countrycode AS __text "