Add OpenSearch failed Test Class and Name fields to test-report.yml (#…

…5101)

Signed-off-by: Peter Zhu <[email protected]>

.licenserc.json

@@ -30,6 +30,7 @@
     ".gz",
     ".toml",
     ".ini",
-    "gradle/wrapper"
+    "gradle/wrapper",
+    "index.html"
   ]
 }

Pipfile

@@ -42,6 +42,7 @@ idna = "~=3.10"
 certifi = "~=2024.7.4"
 types-urllib3 = "~=1.26.25.14"
 charset-normalizer = "~=2.1.1"
+beautifulsoup4 = "~=4.12.3"
 
 [dev-packages]
 

src/manifests/test_report_manifest.py

@@ -44,6 +44,8 @@ class TestReportManifest(ComponentManifest['TestReportManifest', 'TestComponents
                   - URL or local path to the OpenSearch cluster logs
                 cluster_stderr:
                   - URL or local path to the OpenSearch cluster error logs
+                failed test:
+                  - ClassName#TestName for failed test case
     """
 
     VERSIONS = {
@@ -90,7 +92,8 @@ class TestReportManifest(ComponentManifest['TestReportManifest', 'TestComponents
                                 "test_stdout": {"type": "string"},
                                 "test_stderr": {"type": "string"},
                                 "cluster_stdout": {"type": "list"},
-                                "cluster_stderr": {"type": "list"}
+                                "cluster_stderr": {"type": "list"},
+                                "failed_test": {"type": "list"}
                             }
                         },
                     },
@@ -188,6 +191,7 @@ def __init__(self, data: dict) -> None:
                 self.test_stderr = data["test_stderr"]
                 self.cluster_stdout = data["cluster_stdout"]
                 self.cluster_stderr = data["cluster_stderr"]
+                self.failed_test = data["failed_test"]
 
             def __to_dict__(self) -> dict:
                 return {
@@ -197,7 +201,8 @@ def __to_dict__(self) -> dict:
                     "test_stdout": self.test_stdout,
                     "test_stderr": self.test_stderr,
                     "cluster_stdout": self.cluster_stdout,
-                    "cluster_stderr": self.cluster_stderr
+                    "cluster_stderr": self.cluster_stderr,
+                    "failed_test": self.failed_test
                 }
 
 

src/report_workflow/test_report_runner.py

@@ -14,6 +14,7 @@
 
 import validators
 import yaml
+from bs4 import BeautifulSoup
 
 from manifests.bundle_manifest import BundleManifest
 from manifests.test_manifest import TestManifest
@@ -128,6 +129,7 @@ def component_entry(self, component_name: str, ci_group: int = None) -> Any:
             config_dict["test_stderr"] = get_test_logs(self.base_path, str(self.test_run_id), self.test_type, test_report_component_name, config, self.name)[1]
             config_dict["cluster_stdout"] = get_os_cluster_logs(self.base_path, str(self.test_run_id), self.test_type, test_report_component_name, config, self.name)[0]
             config_dict["cluster_stderr"] = get_os_cluster_logs(self.base_path, str(self.test_run_id), self.test_type, test_report_component_name, config, self.name)[1]
+            config_dict["failed_test"] = get_failed_tests(self.base_path, str(self.test_run_id), self.test_type, test_report_component_name, config, self.name)
             component["configs"].append(config_dict)
         return component
 
@@ -196,13 +198,57 @@ def get_os_cluster_logs(base_path: str, test_number: str, test_type: str, compon
 
     for ids in cluster_ids:
         if base_path.startswith("https://"):
-            os_stdout.append("/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config, f"local-cluster-logs/{ids}/stdout.txt"]))
-            os_stderr.append("/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config, f"local-cluster-logs/{ids}/stderr.txt"]))
+            os_stdout.append("/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config, "local-cluster-logs", ids, "stdout.txt"]))
+            os_stderr.append("/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config, "local-cluster-logs", ids, "stderr.txt"]))
         else:
-            os_stdout.append(os.path.join(base_path, "test-results", test_number, test_type, component_name, config, f"local-cluster-logs/{ids}/stdout.txt"))
-            os_stderr.append(os.path.join(base_path, "test-results", test_number, test_type, component_name, config, f"local-cluster-logs/{ids}/stderr.txt"))
+            os_stdout.append(os.path.join(base_path, "test-results", test_number, test_type, component_name, config, "local-cluster-logs", ids, "stdout.txt"))
+            os_stderr.append(os.path.join(base_path, "test-results", test_number, test_type, component_name, config, "local-cluster-logs", ids, "stderr.txt"))
 
     return [os_stdout, os_stderr]
 
 
+def get_failed_tests(base_path: str, test_number: str, test_type: str, component_name: str, config: str,
+                     product_name: str) -> typing.List[list]:
+    failed_test: list = []
+    result_path: str = ''
+    result_content: str = ''
+
+    if product_name == 'opensearch':
+        if base_path.startswith("https://"):
+            result_path = "/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config, "opensearch-integ-test", "index.html"])
+        else:
+            result_path = os.path.join(base_path, "test-results", test_number, test_type, component_name, config, "opensearch-integ-test", "index.html")
+    else:
+        logging.info("Not supporting OpenSearch-Dashboards Cypress Test Result Yet.")
+        failed_test.append("Test Result Not Available")
+        return failed_test
+
+    try:
+        if validators.url(result_path):
+            with urllib.request.urlopen(result_path) as f:
+                result_content = f.read().decode("utf-8")
+        else:
+            with open(result_path, "r", encoding='utf8') as f:
+                result_content = f.read()
+    except (FileNotFoundError, HTTPError):
+        logging.info(f"Component test results for {component_name} for {config} is missing or the base path is incorrect.")
+        failed_test.append("Test Result Not Available")
+        return failed_test
+
+    if result_content:
+        if ("infoBox success" in result_content) and ("<h2>Failed tests</h2>" not in result_content):
+            failed_test.append("No Failed Test")
+        else:
+            soup = BeautifulSoup(result_content, "html.parser")
+            target_div = soup.find("div", {"id": "tab0"})
+            target_a_hash = [a for li in target_div.find_all("li") for a in li.find_all("a", href=True) if "#" in a["href"]]
+            for a in target_a_hash:
+                failed_test.append(a["href"].replace("classes/", ""))
+    else:
+        logging.info(f"Component test results for {component_name} for {config} is empty in {result_path}.")
+        failed_test.append("Test Result Not Available")
+
+    return failed_test
+
+
 TestReportRunner.__test__ = False  # type:ignore

tests/tests_report_workflow/data/test-results/123/integ-test/geospatial/with-security/geospatial.yml

@@ -0,0 +1,48 @@
+component_name: geospatial
+run_id: 123123
+test_config: with-security
+test_result: PASS
+test_result_files:
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/stderr.txt
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/stdout.txt
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/local-cluster-logs/id-0/stderr.txt
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/local-cluster-logs/id-0/stdout.txt
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/local-cluster-logs/id-0/opensearch-service-logs/gc.log
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/local-cluster-logs/id-0/opensearch-service-logs/opensearchcluster1_index_search_slowlog.json
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/local-cluster-logs/id-0/opensearch-service-logs/opensearchcluster1_index_indexing_slowlog.log
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/local-cluster-logs/id-0/opensearch-service-logs/gc.log.00
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/local-cluster-logs/id-0/opensearch-service-logs/opensearchcluster1.log
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/local-cluster-logs/id-0/opensearch-service-logs/opensearchcluster1_server.json
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/local-cluster-logs/id-0/opensearch-service-logs/opensearchcluster1_deprecation.log
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/local-cluster-logs/id-0/opensearch-service-logs/opensearchcluster1_task_detailslog.json
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/local-cluster-logs/id-0/opensearch-service-logs/opensearchcluster1_deprecation.json
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/local-cluster-logs/id-0/opensearch-service-logs/opensearchcluster1_index_search_slowlog.log
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/local-cluster-logs/id-0/opensearch-service-logs/opensearchcluster1_task_detailslog.log
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/local-cluster-logs/id-0/opensearch-service-logs/opensearchcluster1_index_indexing_slowlog.json
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/index.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessorIT.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.search.aggregations.bucket.geogrid.GeoHexAggregationIT.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.index.mapper.xypoint.XYPointFieldMapperIT.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONActionIT.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.stats.upload.RestUploadStatsActionIT.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.index.query.xyshape.XYShapeQueryIT.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.processor.FeatureProcessorIT.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.index.query.xypoint.XYPointQueryIT.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.index.mapper.xyshape.XYShapeFieldMapperIT.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.plugin.GeospatialPluginIT.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.ip2geo.action.UpdateDatasourceIT.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/css/style.css
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/css/base-style.css
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.search.aggregations.bucket.geogrid.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.index.query.xypoint.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.plugin.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.ip2geo.action.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.rest.action.upload.geojson.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.stats.upload.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.processor.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.index.mapper.xyshape.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.index.mapper.xypoint.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.index.query.xyshape.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.ip2geo.processor.html
+- /opensearch-build/test-results/123123/integ-test/geospatial/with-security/opensearch-integ-test/js/report.js
+test_type: integ-test