diff --git a/test/systemtest/test_separate_pkgs.py b/test/systemtest/test_separate_pkgs.py index 68b6777..ce895d1 100644 --- a/test/systemtest/test_separate_pkgs.py +++ b/test/systemtest/test_separate_pkgs.py @@ -22,6 +22,11 @@ from test.systemtest._test_helpers import make_repo, remove_repo from ros_license_toolkit.main import main +from ros_license_toolkit.checks import Status + +SUCCESS = Status.SUCCESS +WARNING = Status.WARNING +FAILURE = Status.FAILURE class TestPkgs(unittest.TestCase): @@ -56,14 +61,17 @@ def test_pkg_both_tags_not_spdx(self): are SPDX conform""" process, stdout = open_subprocess("test_pkg_both_tags_not_spdx") self.assertEqual(os.EX_OK, process.returncode) - self.assertIn(b"LicenseTagIsInSpdxListCheck\n\x1b[93m WARNING", stdout) + self.assertTrue(check_output_status(stdout, + SUCCESS, WARNING, WARNING, WARNING, WARNING)) def test_pkg_both_tags_not_spdx_one_file_own(self): """Test on a package that has two licenses. One is self-defined, other one with not SPDX tag but therefore code and license file in SPDX""" - self.assertEqual(os.EX_DATAERR, main([ - "test/_test_data/test_pkg_both_tags_not_spdx_one_file_own" - ])) + process, stdout = open_subprocess( + "test_pkg_both_tags_not_spdx_one_file_own") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status(stdout, + SUCCESS, WARNING, FAILURE, WARNING, WARNING)) def test_pkg_has_code_disjoint(self): """Test on a package with two disjoint sets of source files under @@ -75,8 +83,9 @@ def test_pkg_has_code_of_different_license(self): """Test on a package with source files under a license different from the package main license (here LGPL). It should fail, because the additional license is not declared in the package.xml.""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/test_pkg_has_code_of_different_license"])) + process, stdout = open_subprocess("test_pkg_has_code_of_different_license") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status(stdout, exp_lic_in_code=FAILURE)) def test_pkg_has_code_of_different_license_and_tag(self): """Test on a package with source files under a license different @@ -90,9 +99,11 @@ def test_pkg_has_code_of_different_license_and_wrong_tag(self): """Test on a package with source files under a license different from the package main license, but the additional license is declared in the package.xml, but with the wrong name.""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/" - "test_pkg_has_code_of_different_license_and_wrong_tag"])) + process, stdout = open_subprocess( + "test_pkg_has_code_of_different_license_and_wrong_tag") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status(stdout, + exp_lic_text_exits=FAILURE, exp_lic_in_code=FAILURE)) def test_pkg_ignore_readme_contents(self): """Test on a package with readme files. READMEs mention licenses @@ -105,7 +116,8 @@ def test_pkg_name_not_in_spdx(self): but its license tag BSD is not in SPDX format""" process, stdout = open_subprocess("test_pkg_name_not_in_spdx") self.assertEqual(os.EX_OK, process.returncode) - self.assertIn(b"WARNING", stdout) + self.assertTrue(check_output_status(stdout, + SUCCESS, WARNING, WARNING, SUCCESS, WARNING)) def test_pkg_no_file_attribute(self): """Test on a package with License file that is not referenced in @@ -115,19 +127,25 @@ def test_pkg_no_file_attribute(self): def test_pkg_no_license(self): """Test on a package with no license declared in the package.xml.""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/test_pkg_no_license"])) + process, stdout = open_subprocess("test_pkg_no_license") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status(stdout, + FAILURE, SUCCESS, FAILURE, FAILURE, SUCCESS)) def test_pkg_no_license_file(self): """Test on a package with no license text file.""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/test_pkg_no_license_file"])) + process, stdout = open_subprocess("test_pkg_no_license_file") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue( + check_output_status(stdout, exp_lic_text_exits=FAILURE)) def test_pkg_one_correct_one_license_file_missing(self): """Test on a package that has one correct license with file and code, but also one not known license tag without file""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/test_pkg_one_correct_one_license_file_missing"])) + process, stdout = open_subprocess("test_pkg_one_correct_one_license_file_missing") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status(stdout, + SUCCESS, WARNING, FAILURE, FAILURE, SUCCESS)) def test_pkg_spdx_name(self): """Test on a package with a license declared in the package.xml @@ -149,36 +167,32 @@ def test_pkg_too_many_license_files(self): self.assertIn(b"bsd.LICENSE", stdout) self.assertIn(b"apl.LICENSE", stdout) self.assertNotIn(b"../../../LICENSE", stdout) + self.assertTrue( + check_output_status(stdout, exp_lic_files_referenced=FAILURE)) def test_pkg_tag_not_spdx(self): """Test on a package that has one linked declaration, one code file but not in SPDX tag. Tag must be gotten from declaration.""" process, stdout = open_subprocess("test_pkg_tag_not_spdx") self.assertEqual(os.EX_OK, process.returncode) - self.assertIn(b"WARNING", stdout) - self.assertIn(b"'code_with_afl.py' is of AFL-2.0 but its Tag is AFL.", - stdout) + self.assertTrue(check_output_status(stdout, + SUCCESS, WARNING, WARNING, WARNING, WARNING)) def test_pkg_unknown_license(self): """Test on a package with an unknown license declared in the package.xml.""" - # using subprocess.Popen instead of main() to capture stdout - with subprocess.Popen( - ["ros_license_toolkit", - "test/_test_data/test_pkg_unknown_license"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) as process: - stdout, _ = process.communicate() - self.assertNotEqual(os.EX_OK, process.returncode) - self.assertIn(b'not in SPDX list of licenses', stdout) - self.assertIn(b'my own fancy license 1.0', stdout) + process, stdout = open_subprocess("test_pkg_unknown_license") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status(stdout, + exp_lic_tag_spdx=WARNING, exp_lic_text_exits=WARNING)) def test_pkg_unknown_license_missing_file(self): """Test on a package that has an unknown license without a license file""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/test_pkg_unknown_license_missing_file"])) + process, stdout = open_subprocess("test_pkg_unknown_license_missing_file") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status(stdout, + exp_lic_tag_spdx=WARNING, exp_lic_text_exits=WARNING)) def test_pkg_with_license_and_file(self): """Test on a package with a license declared in the package.xml @@ -189,9 +203,11 @@ def test_pkg_with_license_and_file(self): def test_pkg_with_multiple_licenses_no_source_files_tag(self): """Test on a package with multiple licenses declared in the package.xml, none of which have source file tags.""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/" - "test_pkg_with_multiple_licenses_no_source_files_tag"])) + process, stdout = open_subprocess( + "test_pkg_with_multiple_licenses_no_source_files_tag") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue( + check_output_status(stdout, exp_lic_tag_exists=FAILURE)) def test_pkg_with_multiple_licenses_one_referenced_incorrect(self): """Test on a package with multiple licenses declared in the @@ -201,12 +217,16 @@ def test_pkg_with_multiple_licenses_one_referenced_incorrect(self): "test_pkg_with_multiple_licenses_one_referenced_incorrect") self.assertEqual(os.EX_OK, process.returncode) self.assertIn(b"WARNING Licenses ['BSD'] are not in SPDX list", stdout) + self.assertTrue(check_output_status(stdout, + SUCCESS, WARNING, WARNING, WARNING, WARNING)) def test_pkg_wrong_license_file(self): """Test on a package with a license text file that does not match the license declared in the package.xml, both tag and file in spdx""" - self.assertEqual(os.EX_DATAERR, main( - ["test/_test_data/test_pkg_wrong_license_file"])) + process, stdout = open_subprocess("test_pkg_wrong_license_file") + self.assertEqual(os.EX_DATAERR, process.returncode) + self.assertTrue(check_output_status(stdout, + exp_lic_text_exits=FAILURE, exp_lic_files_referenced=SUCCESS)) def open_subprocess(test_data_name: str): @@ -220,6 +240,43 @@ def open_subprocess(test_data_name: str): stdout, _ = process.communicate() return process, stdout +def check_output_status(output: str, + exp_lic_tag_exists: Status = Status.SUCCESS, + exp_lic_tag_spdx: Status = Status.SUCCESS, + exp_lic_text_exits: Status = Status.SUCCESS, + exp_lic_in_code: Status = Status.SUCCESS, + exp_lic_files_referenced: Status = Status.SUCCESS + ) -> bool: + """Check output of each check for expected status.""" + # pylint: disable=too-many-arguments + + real_lic_tag_exists = get_test_result(output, "LicenseTagExistsCheck") + real_lic_tag_spdx = get_test_result(output, "LicenseTagIsInSpdxListCheck") + real_lic_text_exits = get_test_result(output, "LicenseTextExistsCheck") + real_lic_in_code = get_test_result(output, "LicensesInCodeCheck") + real_lic_files_referenced = get_test_result( + output, "LicenseFilesReferencedCheck") + + return exp_lic_tag_exists == real_lic_tag_exists \ + and exp_lic_tag_spdx == real_lic_tag_spdx \ + and exp_lic_text_exits == real_lic_text_exits \ + and exp_lic_in_code == real_lic_in_code \ + and exp_lic_files_referenced == real_lic_files_referenced + +def get_test_result(output: str, test_name: str) -> Status: + """Get single test result for specific test.""" + lines = output.splitlines() + for i, line in enumerate(lines): + if test_name in str(line): + if i + 1 < len(lines): + result_line = str(lines[i + 1]) + if "FAILURE" in result_line: + return FAILURE + if "WARNING" in result_line: + return WARNING + if "SUCCESS" in result_line: + return SUCCESS + return None if __name__ == '__main__': unittest.main()