added check_output_status function to check every test for single che…

…ck outputs

Signed-off-by: Anton Utz <[email protected]>

test/systemtest/test_separate_pkgs.py

@@ -22,6 +22,11 @@
 from test.systemtest._test_helpers import make_repo, remove_repo
 
 from ros_license_toolkit.main import main
+from ros_license_toolkit.checks import Status
+
+SUCCESS = Status.SUCCESS
+WARNING = Status.WARNING
+FAILURE = Status.FAILURE
 
 
 class TestPkgs(unittest.TestCase):
@@ -56,14 +61,17 @@ def test_pkg_both_tags_not_spdx(self):
         are SPDX conform"""
         process, stdout = open_subprocess("test_pkg_both_tags_not_spdx")
         self.assertEqual(os.EX_OK, process.returncode)
-        self.assertIn(b"LicenseTagIsInSpdxListCheck\n\x1b[93m WARNING", stdout)
+        self.assertTrue(check_output_status(stdout,
+                            SUCCESS, WARNING, WARNING, WARNING, WARNING))
 
     def test_pkg_both_tags_not_spdx_one_file_own(self):
         """Test on a package that has two licenses. One is self-defined, other
         one with not SPDX tag but therefore code and license file in SPDX"""
-        self.assertEqual(os.EX_DATAERR, main([
-            "test/_test_data/test_pkg_both_tags_not_spdx_one_file_own"
-        ]))
+        process, stdout = open_subprocess(
+            "test_pkg_both_tags_not_spdx_one_file_own")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(stdout,
+                            SUCCESS, WARNING, FAILURE, WARNING, WARNING))
 
     def test_pkg_has_code_disjoint(self):
         """Test on a package with two disjoint sets of source files under
@@ -75,8 +83,9 @@ def test_pkg_has_code_of_different_license(self):
         """Test on a package with source files under a license different
         from the package main license (here LGPL). It should fail, because
         the additional license is not declared in the package.xml."""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/test_pkg_has_code_of_different_license"]))
+        process, stdout = open_subprocess("test_pkg_has_code_of_different_license")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(stdout, exp_lic_in_code=FAILURE))
 
     def test_pkg_has_code_of_different_license_and_tag(self):
         """Test on a package with source files under a license different
@@ -90,9 +99,11 @@ def test_pkg_has_code_of_different_license_and_wrong_tag(self):
         """Test on a package with source files under a license different
         from the package main license, but the additional license is declared
         in the package.xml, but with the wrong name."""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/"
-             "test_pkg_has_code_of_different_license_and_wrong_tag"]))
+        process, stdout = open_subprocess(
+            "test_pkg_has_code_of_different_license_and_wrong_tag")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(stdout,
+            exp_lic_text_exits=FAILURE, exp_lic_in_code=FAILURE))
 
     def test_pkg_ignore_readme_contents(self):
         """Test on a package with readme files. READMEs mention licenses
@@ -105,7 +116,8 @@ def test_pkg_name_not_in_spdx(self):
         but its license tag BSD is not in SPDX format"""
         process, stdout = open_subprocess("test_pkg_name_not_in_spdx")
         self.assertEqual(os.EX_OK, process.returncode)
-        self.assertIn(b"WARNING", stdout)
+        self.assertTrue(check_output_status(stdout,
+                            SUCCESS, WARNING, WARNING, SUCCESS, WARNING))
 
     def test_pkg_no_file_attribute(self):
         """Test on a package with License file that is not referenced in
@@ -115,19 +127,25 @@ def test_pkg_no_file_attribute(self):
 
     def test_pkg_no_license(self):
         """Test on a package with no license declared in the package.xml."""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/test_pkg_no_license"]))
+        process, stdout = open_subprocess("test_pkg_no_license")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(stdout,
+                            FAILURE, SUCCESS, FAILURE, FAILURE, SUCCESS))
 
     def test_pkg_no_license_file(self):
         """Test on a package with no license text file."""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/test_pkg_no_license_file"]))
+        process, stdout = open_subprocess("test_pkg_no_license_file")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(
+            check_output_status(stdout, exp_lic_text_exits=FAILURE))
 
     def test_pkg_one_correct_one_license_file_missing(self):
         """Test on a package that has one correct license with file
         and code, but also one not known license tag without file"""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/test_pkg_one_correct_one_license_file_missing"]))
+        process, stdout = open_subprocess("test_pkg_one_correct_one_license_file_missing")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(stdout,
+                            SUCCESS, WARNING, FAILURE, FAILURE, SUCCESS))
 
     def test_pkg_spdx_name(self):
         """Test on a package with a license declared in the package.xml
@@ -149,36 +167,32 @@ def test_pkg_too_many_license_files(self):
         self.assertIn(b"bsd.LICENSE", stdout)
         self.assertIn(b"apl.LICENSE", stdout)
         self.assertNotIn(b"../../../LICENSE", stdout)
+        self.assertTrue(
+            check_output_status(stdout, exp_lic_files_referenced=FAILURE))
 
     def test_pkg_tag_not_spdx(self):
         """Test on a package that has one linked declaration, one code file
         but not in SPDX tag. Tag must be gotten from declaration."""
         process, stdout = open_subprocess("test_pkg_tag_not_spdx")
         self.assertEqual(os.EX_OK, process.returncode)
-        self.assertIn(b"WARNING", stdout)
-        self.assertIn(b"'code_with_afl.py' is of AFL-2.0 but its Tag is AFL.",
-                      stdout)
+        self.assertTrue(check_output_status(stdout,
+                            SUCCESS, WARNING, WARNING, WARNING, WARNING))
 
     def test_pkg_unknown_license(self):
         """Test on a package with an unknown license declared in the
         package.xml."""
-        # using subprocess.Popen instead of main() to capture stdout
-        with subprocess.Popen(
-            ["ros_license_toolkit",
-             "test/_test_data/test_pkg_unknown_license"],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE
-        ) as process:
-            stdout, _ = process.communicate()
-        self.assertNotEqual(os.EX_OK, process.returncode)
-        self.assertIn(b'not in SPDX list of licenses', stdout)
-        self.assertIn(b'my own fancy license 1.0', stdout)
+        process, stdout = open_subprocess("test_pkg_unknown_license")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(stdout,
+            exp_lic_tag_spdx=WARNING, exp_lic_text_exits=WARNING))
 
     def test_pkg_unknown_license_missing_file(self):
         """Test on a package that has an unknown license
         without a license file"""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/test_pkg_unknown_license_missing_file"]))
+        process, stdout = open_subprocess("test_pkg_unknown_license_missing_file")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(stdout,
+            exp_lic_tag_spdx=WARNING, exp_lic_text_exits=WARNING))
 
     def test_pkg_with_license_and_file(self):
         """Test on a package with a license declared in the package.xml
@@ -189,9 +203,11 @@ def test_pkg_with_license_and_file(self):
     def test_pkg_with_multiple_licenses_no_source_files_tag(self):
         """Test on a package with multiple licenses declared in the
         package.xml, none of which have source file tags."""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/"
-             "test_pkg_with_multiple_licenses_no_source_files_tag"]))
+        process, stdout = open_subprocess(
+            "test_pkg_with_multiple_licenses_no_source_files_tag")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(
+            check_output_status(stdout, exp_lic_tag_exists=FAILURE))
 
     def test_pkg_with_multiple_licenses_one_referenced_incorrect(self):
         """Test on a package with multiple licenses declared in the
@@ -201,12 +217,16 @@ def test_pkg_with_multiple_licenses_one_referenced_incorrect(self):
             "test_pkg_with_multiple_licenses_one_referenced_incorrect")
         self.assertEqual(os.EX_OK, process.returncode)
         self.assertIn(b"WARNING Licenses ['BSD'] are not in SPDX list", stdout)
+        self.assertTrue(check_output_status(stdout,
+                            SUCCESS, WARNING, WARNING, WARNING, WARNING))
 
     def test_pkg_wrong_license_file(self):
         """Test on a package with a license text file that does not match
         the license declared in the package.xml, both tag and file in spdx"""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/test_pkg_wrong_license_file"]))
+        process, stdout = open_subprocess("test_pkg_wrong_license_file")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(stdout, 
+            exp_lic_text_exits=FAILURE, exp_lic_files_referenced=SUCCESS))
 
 
 def open_subprocess(test_data_name: str):
@@ -220,6 +240,43 @@ def open_subprocess(test_data_name: str):
         stdout, _ = process.communicate()
     return process, stdout
 
+def check_output_status(output: str,
+                        exp_lic_tag_exists: Status = Status.SUCCESS,
+                        exp_lic_tag_spdx: Status = Status.SUCCESS,
+                        exp_lic_text_exits: Status = Status.SUCCESS,
+                        exp_lic_in_code: Status = Status.SUCCESS,
+                        exp_lic_files_referenced: Status = Status.SUCCESS
+                        ) -> bool:
+    """Check output of each check for expected status."""
+    # pylint: disable=too-many-arguments
+
+    real_lic_tag_exists = get_test_result(output, "LicenseTagExistsCheck")
+    real_lic_tag_spdx = get_test_result(output, "LicenseTagIsInSpdxListCheck")
+    real_lic_text_exits = get_test_result(output, "LicenseTextExistsCheck")
+    real_lic_in_code = get_test_result(output, "LicensesInCodeCheck")
+    real_lic_files_referenced = get_test_result(
+        output, "LicenseFilesReferencedCheck")
+
+    return exp_lic_tag_exists == real_lic_tag_exists \
+        and exp_lic_tag_spdx == real_lic_tag_spdx \
+        and exp_lic_text_exits == real_lic_text_exits \
+        and exp_lic_in_code == real_lic_in_code \
+        and exp_lic_files_referenced == real_lic_files_referenced
+
+def get_test_result(output: str, test_name: str) -> Status:
+    """Get single test result for specific test."""
+    lines = output.splitlines()
+    for i, line in enumerate(lines):
+        if test_name in str(line):
+            if i + 1 < len(lines):
+                result_line = str(lines[i + 1])
+    if "FAILURE" in result_line:
+        return FAILURE
+    if "WARNING" in result_line:
+        return WARNING
+    if "SUCCESS" in result_line:
+        return SUCCESS
+    return None
 
 if __name__ == '__main__':
     unittest.main()