Merge branch 'pmderodat/xunit-importer' into 'master'

Add a XUnit results importer for report indexes

See merge request it/e3-testsuite!10

NEWS.md

@@ -1,6 +1,7 @@
 26.0 (Not released yet)
 =======================
 
+* Add a XUnit results importer for report indexes.
 * `AdaCoreLegacyDriver`: avoid CRLF line endings in test scripts.
 * `AdaCoreLegacyDriver`: fix handling of non-ASCII test scripts.
 * Detailed logs in case of subprocess output decoding error.

setup.py

@@ -34,11 +34,12 @@
     namespace_packages=["e3"],
     entry_points={
         "console_scripts": [
-            "e3-test = e3.testsuite.main:main",
-            "e3-testsuite-report = e3.testsuite.report.display:main",
             "e3-find-skipped-tests = e3.testsuite.find_skipped_tests:main",
-            "e3-run-test-fragment = e3.testsuite.fragment:run_fragment",
+            "e3-convert-xunit = e3.testsuite.report.xunit:convert_main",
             "e3-opt-parser = e3.testsuite.optfileparser:main",
+            "e3-run-test-fragment = e3.testsuite.fragment:run_fragment",
+            "e3-test = e3.testsuite.main:main",
+            "e3-testsuite-report = e3.testsuite.report.display:main",
         ]
     },
 )

src/e3/testsuite/report/xunit.py

@@ -2,11 +2,18 @@
 
 from __future__ import annotations
 
+import argparse
+import itertools
+import os
+import re
 from typing import Optional
+import unicodedata
 import xml.etree.ElementTree as etree
 
+from e3.fs import mkdir
 from e3.testsuite.report.index import ReportIndex
-from e3.testsuite.result import TestStatus
+from e3.testsuite.result import TestResult, TestStatus
+import e3.yaml
 
 
 def add_time_attribute(elt: etree.Element, duration: Optional[float]) -> None:
@@ -125,3 +132,201 @@ def dump_xunit_report(name: str, index: ReportIndex, filename: str) -> None:
     # The report is ready: write it to the requested file
     tree = etree.ElementTree(testsuites)
     tree.write(filename, encoding="utf-8", xml_declaration=True)
+
+
+class XUnitImporter:
+    """Helper class to import results in a xUnit report into a report index."""
+
+    def __init__(
+        self,
+        index: ReportIndex,
+        xfails: dict[str, str] | None = None,
+    ):
+        """Create a XUnitImporter instance.
+
+        :param index: Report index into which to import xUnit results.
+        :param xfails: For each test result that is expected to fail, this dict
+            must contain a "test name" to "expected failure message"
+            (potentially an empty string) association.
+        """
+        self.index = index
+        self.xfails = xfails or {}
+
+    def run(self, filename: str) -> None:
+        """Read a xUnit report and import its results in the report index.
+
+        :param filename: Filename for the XML file that contains the xUnit
+            report.
+        """
+        doc = etree.parse(filename)
+        testsuites = doc.getroot()
+        assert testsuites.tag == "testsuites"
+
+        for testsuite in testsuites:
+            assert testsuite.tag == "testsuite"
+            testsuite_name = testsuite.attrib["name"]
+
+            for testcase in testsuite:
+                # Skip "properties", "system-out" and "system-err" elements so
+                # that we process only "testcase" ones.
+                if testcase.tag in {
+                    "properties", "system-out", "system-err"
+                }:
+                    continue
+                assert testcase.tag == "testcase"
+
+                testcase_name = testcase.attrib["name"]
+                time_str = testcase.attrib.get("time")
+
+                result = TestResult(
+                    self.get_test_name(testsuite_name, testcase_name)
+                )
+                result.time = float(time_str) if time_str else None
+                status = TestStatus.PASS
+                message: str | None = None
+
+                # Expect at most one "error", "failure" or "skipped" element.
+                # Presence of one such element or the absence of elements
+                # enables us to determine the test status.
+                count = len(testcase)
+                assert count <= 1
+                if count == 1:
+                    status_elt = testcase[0]
+                    tag = status_elt.tag
+                    if tag == "error":
+                        status = TestStatus.ERROR
+                    elif tag == "failure":
+                        status = TestStatus.FAIL
+                    elif tag == "skipped":
+                        # xUnit reports created by py.test may contain a "type"
+                        # attribute that disambiguates between skip and xfail
+                        # results.
+                        kind = status_elt.attrib.get("type")
+                        status = (
+                            TestStatus.XFAIL
+                            if kind == "pytest.xfail"
+                            else TestStatus.SKIP
+                        )
+                    else:  # no cover
+                        raise AssertionError(f"invalid status tag: {tag}")
+                    if isinstance(status_elt.text, str):
+                        result.log += status_elt.text
+
+                    message = status_elt.attrib.get("message")
+                else:
+                    message = None
+
+                # Now that the "unrefined" status for this result is known,
+                # apply XFAIL, if needed.
+                xfail_message = self.xfails.get(result.test_name)
+                if xfail_message is not None:
+                    # Depending on whether we have an XFAIL message and/or a
+                    # test result message, create a single message for the
+                    # result to store in the report.
+                    new_message = (
+                        f"{message} ({xfail_message})"
+                        if message
+                        else xfail_message
+                    ) if xfail_message else message
+
+                    # xUnit tests often use the ERROR status for issues that
+                    # are not testsuite bugs (i.e. for what we call "failures"
+                    # in e3-testsuite), so be pragmatic and allow them to be
+                    # covered by XFAIL.
+                    if status == TestStatus.PASS:
+                        status = TestStatus.XPASS
+                        message = new_message
+                    elif status in (TestStatus.FAIL, TestStatus.ERROR):
+                        status = TestStatus.XFAIL
+                        message = new_message
+
+                result.set_status(status, message)
+                self.index.save_and_add_result(result)
+
+    SLUG_RE = re.compile("[a-zA-Z0-9_.]+")
+
+    def slugify(self, name: str) -> str:
+        """Normalize a string so that it is an acceptable test name component.
+
+        :param name: Component (substring) for a name to turn into a test name
+            that is acceptable for e3-testsuite.
+        """
+        # Normalize the string, decomposing some codepoints into ASCII
+        # characters + modifiers
+        name = unicodedata.normalize("NFKD", name)
+
+        # Preserve only codepoints in [a-zA-Z0-9_.] and replace/collapse the
+        # rest with hyphens.
+        return "-".join(chunk for chunk in self.SLUG_RE.findall(name))
+
+    def get_unique_test_name(self, test_name: str) -> str:
+        """Return a test name that is guaranteed to be unique.
+
+        :param test_name: Candidate test name. If the report index already has
+            a test result with the same test name, this method generates
+            another one based on it.
+        """
+        result = test_name
+        counter = itertools.count(1)
+        while result in self.index.entries:
+            result = f"{test_name}.{next(counter)}"
+        return result
+
+    def get_test_name(self, testsuite_name: str, testcase_name: str) -> str:
+        """Combine xUnit testsuite/testcase names into a unique test name.
+
+        :param testsuite_name: Name associated with a xUnit <testsuite>
+            element.
+        :param testcase_name: Name associated with a xUnit <testcase> element.
+        """
+        return self.get_unique_test_name(
+            self.slugify(testsuite_name) + "." + self.slugify(testcase_name)
+        )
+
+
+def read_xfails_from_yaml(filename: str) -> dict[str, str]:
+    """
+    Read a XFAILs dict from a YAML file.
+
+    See the "xfails" parameter for XUnitImporter's constructor for the expected
+    YAML structure.
+    """
+    return e3.yaml.load_ordered(filename)
+
+
+def convert_main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(
+        description="Convert a xUnit testsuite report to e3-testsuite's"
+        " format."
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        help="Output directory for the converted report. By default, use the"
+        " current working directory.",
+    )
+    parser.add_argument(
+        "--xfails",
+        help="YAML file that describes expected failures. If provided, it must"
+        " contain a mapping from test name to expected failure messages.",
+    )
+    parser.add_argument(
+        "xml-report",
+        nargs="+",
+        help="xUnit XML reports to convert. If a directory is passed,"
+        " recursively look for all the files matching *.xml that it contains.",
+    )
+
+    args = parser.parse_args(argv)
+    index = ReportIndex(args.output or ".")
+    mkdir(index.results_dir)
+    xfails = read_xfails_from_yaml(args.xfails) if args.xfails else None
+    importer = XUnitImporter(index, xfails)
+    for path in getattr(args, "xml-report"):
+        if os.path.isdir(path):
+            for root, _, filenames in os.walk(path):
+                for f in filenames:
+                    importer.run(os.path.join(root, f))
+        else:
+            importer.run(path)
+    index.write()