From a3cac85b8c127e8178f3e6f221037ad5d11a2a5a Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Tue, 2 Apr 2024 10:56:44 +0200 Subject: [PATCH] changed each test to also check for warnings and specified which warning must be displayed Signed-off-by: Anton Utz --- test/systemtest/test_separate_pkgs.py | 185 ++++++++++++++++++-------- 1 file changed, 128 insertions(+), 57 deletions(-) diff --git a/test/systemtest/test_separate_pkgs.py b/test/systemtest/test_separate_pkgs.py index 94382ee..0fa3b86 100644 --- a/test/systemtest/test_separate_pkgs.py +++ b/test/systemtest/test_separate_pkgs.py @@ -20,8 +20,13 @@ import subprocess import unittest from test.systemtest._test_helpers import make_repo, remove_repo +from typing import Optional -from ros_license_toolkit.main import main +from ros_license_toolkit.checks import Status + +SUCCESS = Status.SUCCESS +WARNING = Status.WARNING +FAILURE = Status.FAILURE class TestPkgs(unittest.TestCase): @@ -54,16 +59,19 @@ def test_pkg_both_tags_not_spdx(self): """ Test on a package that has two different Licenses that both have a not SPDX conform Tag. License files and source files are SPDX conform""" - self.assertEqual(os.EX_OK, main([ - "test/_test_data/test_pkg_both_tags_not_spdx" - ])) + process, stdout = open_subprocess("test_pkg_both_tags_not_spdx") + self.assertEqual(os.EX_OK, process.returncode) + self.assertTrue(check_output_status( + stdout, SUCCESS, WARNING, WARNING, WARNING, WARNING)) def test_pkg_both_tags_not_spdx_one_file_own(self): """Test on a package that has two licenses. One is self-defined, other one with not SPDX tag but therefore code and license file in SPDX""" - self.assertEqual(os.EX_DATAERR, main([ - "test/_test_data/test_pkg_both_tags_not_spdx_one_file_own" - ])) + process, stdout = open_subprocess( + "test_pkg_both_tags_not_spdx_one_file_own") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status( + stdout, SUCCESS, WARNING, FAILURE, WARNING, WARNING)) def test_pkg_code_has_no_license(self): """Test on a package that has a correct package.xml with a license @@ -76,78 +84,96 @@ def test_pkg_code_has_no_license(self): def test_pkg_has_code_disjoint(self): """Test on a package with two disjoint sets of source files under a license different from the package main license.""" - self.assertEqual(os.EX_OK, main( - ["test/_test_data/test_pkg_has_code_disjoint"])) + process, stdout = open_subprocess("test_pkg_has_code_disjoint") + self.assertEqual(os.EX_OK, process.returncode) + self.assertTrue(check_output_status(stdout)) def test_pkg_has_code_of_different_license(self): """Test on a package with source files under a license different from the package main license (here LGPL). It should fail, because the additional license is not declared in the package.xml.""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/test_pkg_has_code_of_different_license"])) + process, stdout = open_subprocess( + "test_pkg_has_code_of_different_license") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status(stdout, exp_lic_in_code=FAILURE)) def test_pkg_has_code_of_different_license_and_tag(self): """Test on a package with source files under a license different from the package main license, but the additional license is declared in the package.xml.""" - self.assertEqual(os.EX_OK, main( - ["test/_test_data/" - "test_pkg_has_code_of_different_license_and_tag"])) + process, stdout = open_subprocess( + "test_pkg_has_code_of_different_license_and_tag") + self.assertEqual(os.EX_OK, process.returncode) + self.assertTrue(check_output_status(stdout)) def test_pkg_has_code_of_different_license_and_wrong_tag(self): """Test on a package with source files under a license different from the package main license, but the additional license is declared in the package.xml, but with the wrong name.""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/" - "test_pkg_has_code_of_different_license_and_wrong_tag"])) + process, stdout = open_subprocess( + "test_pkg_has_code_of_different_license_and_wrong_tag") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status( + stdout, exp_lic_text_exits=FAILURE, exp_lic_in_code=FAILURE)) def test_pkg_ignore_readme_contents(self): """Test on a package with readme files. READMEs mention licenses that are not in package and shall therefore be ignored.""" - test_result = main(["test/_test_data/test_pkg_ignore_readme_contents"]) - self.assertEqual(os.EX_OK, test_result) + process, stdout = open_subprocess("test_pkg_ignore_readme_contents") + self.assertEqual(os.EX_OK, process.returncode) + self.assertTrue(check_output_status(stdout)) def test_pkg_name_not_in_spdx(self): """Test on a package that has valid License file with BSD-3-Clause but its license tag BSD is not in SPDX format""" process, stdout = open_subprocess("test_pkg_name_not_in_spdx") self.assertEqual(os.EX_OK, process.returncode) - self.assertIn(b"WARNING", stdout) + self.assertTrue(check_output_status( + stdout, SUCCESS, WARNING, WARNING, SUCCESS, WARNING)) def test_pkg_no_file_attribute(self): """Test on a package with License file that is not referenced in package.xml""" - self.assertEqual(os.EX_OK, main( - ["test/_test_data/test_pkg_no_file_attribute"])) + process, stdout = open_subprocess("test_pkg_no_file_attribute") + self.assertEqual(os.EX_OK, process.returncode) + self.assertTrue(check_output_status(stdout)) def test_pkg_no_license(self): """Test on a package with no license declared in the package.xml.""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/test_pkg_no_license"])) + process, stdout = open_subprocess("test_pkg_no_license") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status( + stdout, FAILURE, SUCCESS, FAILURE, FAILURE, SUCCESS)) def test_pkg_no_license_file(self): """Test on a package with no license text file.""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/test_pkg_no_license_file"])) + process, stdout = open_subprocess("test_pkg_no_license_file") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status( + stdout, exp_lic_text_exits=FAILURE)) def test_pkg_one_correct_one_license_file_missing(self): """Test on a package that has one correct license with file and code, but also one not known license tag without file""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/test_pkg_one_correct_one_license_file_missing"])) + process, stdout = open_subprocess( + "test_pkg_one_correct_one_license_file_missing") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status( + stdout, SUCCESS, WARNING, FAILURE, FAILURE, SUCCESS)) def test_pkg_spdx_name(self): """Test on a package with a license declared in the package.xml with the SPDX name.""" - self.assertEqual(os.EX_OK, main( - ["test/_test_data/test_pkg_spdx_name"])) + process, stdout = open_subprocess("test_pkg_spdx_name") + self.assertEqual(os.EX_OK, process.returncode) + self.assertTrue(check_output_status(stdout)) def test_pkg_spdx_tag(self): """Test on a package with a license declared in the package.xml with the SPDX tag.""" - self.assertEqual(os.EX_OK, main( - ["test/_test_data/test_pkg_spdx_tag"])) + process, stdout = open_subprocess("test_pkg_spdx_tag") + self.assertEqual(os.EX_OK, process.returncode) + self.assertTrue(check_output_status(stdout)) def test_pkg_too_many_license_files(self): """"Test on a package with multiple License files that are not @@ -157,64 +183,69 @@ def test_pkg_too_many_license_files(self): self.assertIn(b"bsd.LICENSE", stdout) self.assertIn(b"apl.LICENSE", stdout) self.assertNotIn(b"../../../LICENSE", stdout) + self.assertTrue(check_output_status( + stdout, exp_lic_files_referenced=FAILURE)) def test_pkg_tag_not_spdx(self): """Test on a package that has one linked declaration, one code file but not in SPDX tag. Tag must be gotten from declaration.""" process, stdout = open_subprocess("test_pkg_tag_not_spdx") self.assertEqual(os.EX_OK, process.returncode) - self.assertIn(b"WARNING", stdout) - self.assertIn(b"'code_with_afl.py' is of AFL-2.0 but its Tag is AFL.", - stdout) + self.assertTrue(check_output_status( + stdout, SUCCESS, WARNING, WARNING, WARNING, WARNING)) def test_pkg_unknown_license(self): """Test on a package with an unknown license declared in the package.xml.""" - # using subprocess.Popen instead of main() to capture stdout - with subprocess.Popen( - ["ros_license_toolkit", - "test/_test_data/test_pkg_unknown_license"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) as process: - stdout, _ = process.communicate() - self.assertNotEqual(os.EX_OK, process.returncode) - self.assertIn(b'not in SPDX list of licenses', stdout) - self.assertIn(b'my own fancy license 1.0', stdout) + process, stdout = open_subprocess("test_pkg_unknown_license") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status( + stdout, exp_lic_tag_spdx=WARNING, exp_lic_text_exits=FAILURE)) def test_pkg_unknown_license_missing_file(self): """Test on a package that has an unknown license without a license file""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/test_pkg_unknown_license_missing_file"])) + process, stdout = open_subprocess( + "test_pkg_unknown_license_missing_file") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status( + stdout, exp_lic_tag_spdx=WARNING, exp_lic_text_exits=FAILURE)) def test_pkg_with_license_and_file(self): """Test on a package with a license declared in the package.xml and a matching license text file.""" - self.assertEqual(os.EX_OK, main( - ["test/_test_data/test_pkg_with_license_and_file"])) + process, stdout = open_subprocess("test_pkg_with_license_and_file") + self.assertEqual(os.EX_OK, process.returncode) + self.assertTrue(check_output_status(stdout)) def test_pkg_with_multiple_licenses_no_source_files_tag(self): """Test on a package with multiple licenses declared in the package.xml, none of which have source file tags.""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/" - "test_pkg_with_multiple_licenses_no_source_files_tag"])) + process, stdout = open_subprocess( + "test_pkg_with_multiple_licenses_no_source_files_tag") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status( + stdout, exp_lic_tag_exists=FAILURE)) def test_pkg_with_multiple_licenses_one_referenced_incorrect(self): """Test on a package with multiple licenses declared in the - package.xml. First has tag not in SPDX list with correct source file, - second is in SPDX.""" + package.xml. First has tag not in SPDX list with correct + source file, second is in SPDX.""" process, stdout = open_subprocess( "test_pkg_with_multiple_licenses_one_referenced_incorrect") self.assertEqual(os.EX_OK, process.returncode) self.assertIn(b"WARNING Licenses ['BSD'] are not in SPDX list", stdout) + self.assertTrue(check_output_status( + stdout, SUCCESS, WARNING, WARNING, WARNING, WARNING)) def test_pkg_wrong_license_file(self): """Test on a package with a license text file that does not match the license declared in the package.xml, both tag and file in spdx""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/test_pkg_wrong_license_file"])) + process, stdout = open_subprocess("test_pkg_wrong_license_file") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status( + stdout, exp_lic_text_exits=FAILURE, + exp_lic_files_referenced=WARNING)) def open_subprocess(test_data_name: str): @@ -229,5 +260,45 @@ def open_subprocess(test_data_name: str): return process, stdout +def check_output_status(output: str, + exp_lic_tag_exists: Status = Status.SUCCESS, + exp_lic_tag_spdx: Status = Status.SUCCESS, + exp_lic_text_exits: Status = Status.SUCCESS, + exp_lic_in_code: Status = Status.SUCCESS, + exp_lic_files_referenced: Status = Status.SUCCESS + ) -> bool: + """Check output of each check for expected status.""" + # pylint: disable=too-many-arguments + + real_lic_tag_exists = get_test_result(output, "LicenseTagExistsCheck") + real_lic_tag_spdx = get_test_result(output, "LicenseTagIsInSpdxListCheck") + real_lic_text_exits = get_test_result(output, "LicenseTextExistsCheck") + real_lic_in_code = get_test_result(output, "LicensesInCodeCheck") + real_lic_files_referenced = get_test_result( + output, "LicenseFilesReferencedCheck") + + return exp_lic_tag_exists == real_lic_tag_exists \ + and exp_lic_tag_spdx == real_lic_tag_spdx \ + and exp_lic_text_exits == real_lic_text_exits \ + and exp_lic_in_code == real_lic_in_code \ + and exp_lic_files_referenced == real_lic_files_referenced + + +def get_test_result(output: str, test_name: str) -> Optional[Status]: + """Get single test result for specific test.""" + lines = output.splitlines() + for i, line in enumerate(lines): + if test_name in str(line): + if i + 1 < len(lines): + result_line = str(lines[i + 1]) + if "FAILURE" in result_line: + return FAILURE + if "WARNING" in result_line: + return WARNING + if "SUCCESS" in result_line: + return SUCCESS + return None + + if __name__ == '__main__': unittest.main()