From be4a47c59bc56cda2bcdd2e66246b4add1173ae5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 22:41:22 +0000 Subject: [PATCH 01/36] Update scancode-toolkit requirement from <=31.2.6 to <=32.0.7 Updates the requirements on [scancode-toolkit](https://github.com/nexB/scancode-toolkit) to permit the latest version. - [Release notes](https://github.com/nexB/scancode-toolkit/releases) - [Changelog](https://github.com/nexB/scancode-toolkit/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/nexB/scancode-toolkit/compare/v1.0.0...v32.0.7) --- updated-dependencies: - dependency-name: scancode-toolkit dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5874eb9..6a81fb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ # TODO v32 changes things, see # https://github.com/nexB/scancode-toolkit/blob/develop/ # CHANGELOG.rst#v3200---2023-05-23 - "scancode-toolkit<=31.2.6", + "scancode-toolkit<=32.0.7", "spdx-tools" ] requires-python = ">=3.7" From 53dc988bc90a98e39b594f1f75ac2c48a1bfcf43 Mon Sep 17 00:00:00 2001 From: Christian Henkel Date: Mon, 9 Oct 2023 10:51:02 +0200 Subject: [PATCH 02/36] working from 32.0.7 Signed-off-by: Christian Henkel --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6a81fb6..f0cb5f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ # TODO v32 changes things, see # https://github.com/nexB/scancode-toolkit/blob/develop/ # CHANGELOG.rst#v3200---2023-05-23 - "scancode-toolkit<=32.0.7", + "scancode-toolkit>=32.0.7", "spdx-tools" ] requires-python = ">=3.7" From 85df2866902403b7a72c02078f7c84a0d2c4bd77 Mon Sep 17 00:00:00 2001 From: Christian Henkel Date: Mon, 9 Oct 2023 12:03:16 +0200 Subject: [PATCH 03/36] is_license_text check fixed with new result semantics Signed-off-by: Christian Henkel --- src/ros_license_toolkit/common.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/ros_license_toolkit/common.py b/src/ros_license_toolkit/common.py index aee8ff6..c1d8b0e 100644 --- a/src/ros_license_toolkit/common.py +++ b/src/ros_license_toolkit/common.py @@ -18,11 +18,10 @@ from typing import Any, Dict +REQUIRED_PERCENTAGE_OF_LICENSE_TEXT = 99.0 + def is_license_text_file(scan_results: Dict[str, Any]) -> bool: """Check if a file is a license text file.""" - for _license in scan_results["licenses"]: - if _license["matched_rule"]["is_license_text"] and \ - _license["score"] >= 99: - return True - return False + return ( + scan_results["percentage_of_license_text"] >= REQUIRED_PERCENTAGE_OF_LICENSE_TEXT) From db494dfe52b894babaad631e7ecdcf6da31d7741 Mon Sep 17 00:00:00 2001 From: Christian Henkel Date: Mon, 9 Oct 2023 12:11:02 +0200 Subject: [PATCH 04/36] linting fix Signed-off-by: Christian Henkel --- src/ros_license_toolkit/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ros_license_toolkit/common.py b/src/ros_license_toolkit/common.py index c1d8b0e..4d3a514 100644 --- a/src/ros_license_toolkit/common.py +++ b/src/ros_license_toolkit/common.py @@ -24,4 +24,5 @@ def is_license_text_file(scan_results: Dict[str, Any]) -> bool: """Check if a file is a license text file.""" return ( - scan_results["percentage_of_license_text"] >= REQUIRED_PERCENTAGE_OF_LICENSE_TEXT) + scan_results["percentage_of_license_text"] >= + REQUIRED_PERCENTAGE_OF_LICENSE_TEXT) From ca0e3e1930f76952ff3e91465c56cd20b9044a29 Mon Sep 17 00:00:00 2001 From: Christian Henkel Date: Mon, 9 Oct 2023 14:09:00 +0200 Subject: [PATCH 05/36] handling of ignored files Signed-off-by: Christian Henkel --- src/ros_license_toolkit/checks.py | 122 ++++++++++++++++------------- src/ros_license_toolkit/common.py | 3 +- src/ros_license_toolkit/package.py | 31 +++++--- 3 files changed, 90 insertions(+), 66 deletions(-) diff --git a/src/ros_license_toolkit/checks.py b/src/ros_license_toolkit/checks.py index 5f13781..0cd20ea 100644 --- a/src/ros_license_toolkit/checks.py +++ b/src/ros_license_toolkit/checks.py @@ -58,15 +58,11 @@ def _success(self, reason: str): def __str__(self) -> str: """Return formatted string for normal output.""" - if self.success: - info: str = str( - type(self).__name__) + "\n" + \ - green(f" SUCCESS {self.reason}") - else: - info = str( - type(self).__name__) + "\n" + \ - red(f" FAILURE {self.reason}") - return info + return ( + str(type(self).__name__) + "\n" + green(f" SUCCESS {self.reason}") + if self.success + else str(type(self).__name__) + "\n" + red(f" FAILURE {self.reason}") + ) def verbose(self) -> str: """Return string with additional information for verbose output.""" @@ -113,12 +109,11 @@ class LicenseTagIsInSpdxListCheck(Check): """This ensures that the license tag is in the SPDX list of licenses.""" def _check(self, package: Package): - licenses_not_in_spdx_list = [] - for license_tag in package.license_tags.keys(): - if not is_license_name_in_spdx_list( - license_tag): - licenses_not_in_spdx_list.append(license_tag) - if len(licenses_not_in_spdx_list) > 0: + if licenses_not_in_spdx_list := [ + license_tag + for license_tag in package.license_tags.keys() + if not is_license_name_in_spdx_list(license_tag) + ]: self._failed( f"Licenses {licenses_not_in_spdx_list} are " "not in SPDX list of licenses." @@ -144,34 +139,36 @@ def _check(self, package: Package): license_text_file = license_tag.get_license_text_file() if not os.path.exists( os.path.join(package.abspath, license_text_file)): - license_tags_without_license_text[license_tag] =\ - f"License text file '{license_text_file}' does not exist." + license_tags_without_license_text[ + license_tag + ] = f"License text file '{license_text_file}' does not exist." continue if license_text_file not in found_license_texts: - license_tags_without_license_text[license_tag] =\ - f"License text file '{license_text_file}' not included" +\ - " in scan results." + print("{package.license_tags}") + license_tags_without_license_text[ + license_tag + ] = f"License text file '{license_text_file}' not included in scan results." continue if not is_license_text_file( found_license_texts[license_text_file]): - license_tags_without_license_text[license_tag] =\ - f"License text file '{license_text_file}' is not " +\ - "recognized as license text." + license_tags_without_license_text[ + license_tag + ] = f"License text file '{license_text_file}' is not recognized as license text." continue actual_license: Optional[str] = get_spdx_license_name( found_license_texts[license_text_file]) if actual_license is None: - license_tags_without_license_text[license_tag] =\ - f"License text file '{license_text_file}'" +\ - " is not recognized as license text." + license_tags_without_license_text[ + license_tag + ] = f"License text file '{license_text_file}' is not recognized as license text." continue if actual_license != license_tag.get_license_id(): license_tags_without_license_text[license_tag] =\ - f"License text file '{license_text_file}' is " +\ - f"of license {actual_license} but should be " +\ - f"{license_tag.get_license_id()}." + f"License text file '{license_text_file}' is " +\ + f"of license {actual_license} but should be " +\ + f"{license_tag.get_license_id()}." continue - if len(license_tags_without_license_text) > 0: + if license_tags_without_license_text: self._failed( "The following license tags do not have a valid license text " "file:\n" + "\n".join( @@ -200,47 +197,62 @@ def _check(self, package: Package): if fname in package.get_license_files(): # the actual license text files are not relevant for this continue - for found_license in found_licenses['licenses']: - found_license_str = found_license['spdx_license_key'] - if found_license_str not in declared_licenses: + found_licenses_str = found_licenses[ + 'detected_license_expression_spdx'] + licenses = found_licenses_str.split(' AND ') + for license_str in licenses: + if license_str not in declared_licenses: # this license is not declared by any license tag if fname not in files_with_uncovered_licenses: files_with_uncovered_licenses[fname] = [] files_with_uncovered_licenses[fname].append( - found_license_str) + license_str) continue if fname not in declared_licenses[ - found_license_str].source_files: + license_str].source_files: # this license is declared by a license tag but the file # is not listed in the source files of the license tag if fname not in files_not_matched_by_any_license_tag: files_not_matched_by_any_license_tag[fname] = [] files_not_matched_by_any_license_tag[fname].append( - found_license_str) + license_str) continue - if len(files_with_uncovered_licenses) > 0 or \ - len(files_not_matched_by_any_license_tag) > 0: + if files_with_uncovered_licenses: info_str = '' - if len(files_with_uncovered_licenses) > 0: - info_str += '\nThe following files contain licenses that ' +\ + info_str += '\nThe following files contain licenses that ' +\ 'are not covered by any license tag:\n' + '\n'.join( - [f" '{x[0]}': {x[1]}" for x in - files_with_uncovered_licenses.items()]) - elif len(files_not_matched_by_any_license_tag) > 0: - info_str += '\nThe following files contain licenses that ' +\ + [f" '{x[0]}': {x[1]}" for x in + files_with_uncovered_licenses.items()]) + self._extracted_from__check_47( + info_str, + files_with_uncovered_licenses, + files_not_matched_by_any_license_tag, + package, + ) + elif len(files_not_matched_by_any_license_tag) > 0: + info_str = '' + info_str += '\nThe following files contain licenses that ' +\ 'are covered by a license tag but are not listed in ' +\ 'the source files of the license tag:\n' + '\n'.join( - [f" '{x[0]}': {x[1]}" for x in - files_not_matched_by_any_license_tag.items()]) - assert info_str != '' - self._failed(info_str) - self.verbose_output = red( - '\n Relevant scan results:\n' + pformat( - list(filter( - lambda x: x[0] in files_with_uncovered_licenses or ( - x[0] in files_not_matched_by_any_license_tag), - package.found_files_w_licenses.items())))) - + [f" '{x[0]}': {x[1]}" for x in + files_not_matched_by_any_license_tag.items()]) + self._extracted_from__check_47( + info_str, + files_with_uncovered_licenses, + files_not_matched_by_any_license_tag, + package, + ) else: self._success('All licenses found in the code are covered by a ' 'license declaration.') + + # TODO Rename this here and in `_check` + def _extracted_from__check_47(self, info_str, files_with_uncovered_licenses, files_not_matched_by_any_license_tag, package): + assert info_str != '' + self._failed(info_str) + self.verbose_output = red( + '\n Relevant scan results:\n' + pformat( + list(filter( + lambda x: x[0] in files_with_uncovered_licenses or ( + x[0] in files_not_matched_by_any_license_tag), + package.found_files_w_licenses.items())))) diff --git a/src/ros_license_toolkit/common.py b/src/ros_license_toolkit/common.py index 4d3a514..3d551e2 100644 --- a/src/ros_license_toolkit/common.py +++ b/src/ros_license_toolkit/common.py @@ -18,11 +18,12 @@ from typing import Any, Dict -REQUIRED_PERCENTAGE_OF_LICENSE_TEXT = 99.0 +REQUIRED_PERCENTAGE_OF_LICENSE_TEXT = 95.0 def is_license_text_file(scan_results: Dict[str, Any]) -> bool: """Check if a file is a license text file.""" + print(f"{scan_results['percentage_of_license_text']=}") return ( scan_results["percentage_of_license_text"] >= REQUIRED_PERCENTAGE_OF_LICENSE_TEXT) diff --git a/src/ros_license_toolkit/package.py b/src/ros_license_toolkit/package.py index 16d286e..3f059c6 100644 --- a/src/ros_license_toolkit/package.py +++ b/src/ros_license_toolkit/package.py @@ -25,25 +25,27 @@ from rospkg.common import PACKAGE_FILE from scancode.api import get_licenses -from ros_license_toolkit.common import is_license_text_file +from ros_license_toolkit.common import is_license_text_file, \ + REQUIRED_PERCENTAGE_OF_LICENSE_TEXT from ros_license_toolkit.copyright import CopyrightPerPkg from ros_license_toolkit.license_tag import LicenseTag from ros_license_toolkit.repo import NotARepoError, Repo # files we ignore in scan results -IGNORED_FILES = [ +IGNORED_FILE_PREFIXES = [ "package.xml", "setup.py", "setup.cfg", - "CMakeLists.txt" + "CMakeLists.txt", + ".git/" ] def get_spdx_license_name(scan_results: Dict[str, Any]) -> Optional[str]: """Get the SPDX license name from scan results.""" - for _license in scan_results["licenses"]: - if _license["score"] >= 99: - return _license["spdx_license_key"] + if scan_results['percentage_of_license_text'] >=\ + REQUIRED_PERCENTAGE_OF_LICENSE_TEXT: + return scan_results['detected_license_expression_spdx'] return None @@ -116,27 +118,36 @@ def found_license_texts(self) -> Dict[str, Any]: def _run_scan_and_save_results(self): """Get a dict of files in the package and their license scan results. Note that the code is only evaluated on the first call.""" - if not (self._found_files_w_licenses is None or ( - self._found_license_texts is None)): + if ( + self._found_files_w_licenses is not None + and self._found_license_texts is not None + ): return self._found_files_w_licenses, self._found_license_texts self._found_files_w_licenses = {} self._found_license_texts = {} for (root, _, files) in os.walk(self.abspath): for fname in files: - if fname in IGNORED_FILES: - continue # Path relative to cwd fpath = os.path.join(root, fname) # Path relative to package root fpath_rel_to_pkg = self._get_path_relative_to_pkg(fpath) + print(f"0 {fpath_rel_to_pkg=}") + if any( + fpath_rel_to_pkg.startswith(ignored_file_prefix) for + ignored_file_prefix in IGNORED_FILE_PREFIXES + ): + continue scan_results = get_licenses(fpath) + print(f"1 {fpath_rel_to_pkg=}") if is_license_text_file(scan_results): self._found_license_texts[fpath_rel_to_pkg ] = scan_results + print("license text file") else: # not a license text file but also interesting self._found_files_w_licenses[fpath_rel_to_pkg ] = scan_results + print("not license text file") # look also in the repo for license text files if self.repo is not None: for path, res in self.repo.license_text_files.items(): From c53c22b4f2d7f53ffe971c3ddad786b0a4c8d366 Mon Sep 17 00:00:00 2001 From: Christian Henkel Date: Mon, 9 Oct 2023 14:22:13 +0200 Subject: [PATCH 06/36] formatting Signed-off-by: Christian Henkel --- src/ros_license_toolkit/checks.py | 49 ++++++++++++++++++------------ src/ros_license_toolkit/package.py | 2 +- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/ros_license_toolkit/checks.py b/src/ros_license_toolkit/checks.py index 0cd20ea..d4f7eae 100644 --- a/src/ros_license_toolkit/checks.py +++ b/src/ros_license_toolkit/checks.py @@ -58,11 +58,18 @@ def _success(self, reason: str): def __str__(self) -> str: """Return formatted string for normal output.""" - return ( - str(type(self).__name__) + "\n" + green(f" SUCCESS {self.reason}") - if self.success - else str(type(self).__name__) + "\n" + red(f" FAILURE {self.reason}") - ) + if self.success: + return ( + str(type(self).__name__) + + "\n" + + green(f" SUCCESS {self.reason}") + ) + else: + return ( + str(type(self).__name__) + + "\n" + + red(f" FAILURE {self.reason}") + ) def verbose(self) -> str: """Return string with additional information for verbose output.""" @@ -141,32 +148,36 @@ def _check(self, package: Package): os.path.join(package.abspath, license_text_file)): license_tags_without_license_text[ license_tag - ] = f"License text file '{license_text_file}' does not exist." + ] = f"License text file '{license_text_file}'" +\ + " does not exist." continue if license_text_file not in found_license_texts: print("{package.license_tags}") license_tags_without_license_text[ license_tag - ] = f"License text file '{license_text_file}' not included in scan results." + ] = f"License text file '{license_text_file}'" +\ + " not included in scan results." continue if not is_license_text_file( found_license_texts[license_text_file]): license_tags_without_license_text[ license_tag - ] = f"License text file '{license_text_file}' is not recognized as license text." + ] = f"License text file '{license_text_file}'" +\ + " is not recognized as license text." continue actual_license: Optional[str] = get_spdx_license_name( found_license_texts[license_text_file]) if actual_license is None: license_tags_without_license_text[ license_tag - ] = f"License text file '{license_text_file}' is not recognized as license text." + ] = f"License text file '{license_text_file}'" +\ + " is not recognized as license text." continue if actual_license != license_tag.get_license_id(): license_tags_without_license_text[license_tag] =\ - f"License text file '{license_text_file}' is " +\ - f"of license {actual_license} but should be " +\ - f"{license_tag.get_license_id()}." + f"License text file '{license_text_file}' is " +\ + f"of license {actual_license} but should be " +\ + f"{license_tag.get_license_id()}." continue if license_tags_without_license_text: self._failed( @@ -220,10 +231,10 @@ def _check(self, package: Package): if files_with_uncovered_licenses: info_str = '' info_str += '\nThe following files contain licenses that ' +\ - 'are not covered by any license tag:\n' + '\n'.join( + 'are not covered by any license tag:\n' + '\n'.join( [f" '{x[0]}': {x[1]}" for x in files_with_uncovered_licenses.items()]) - self._extracted_from__check_47( + self._print_info( info_str, files_with_uncovered_licenses, files_not_matched_by_any_license_tag, @@ -232,11 +243,11 @@ def _check(self, package: Package): elif len(files_not_matched_by_any_license_tag) > 0: info_str = '' info_str += '\nThe following files contain licenses that ' +\ - 'are covered by a license tag but are not listed in ' +\ - 'the source files of the license tag:\n' + '\n'.join( + 'are covered by a license tag but are not listed in ' +\ + 'the source files of the license tag:\n' + '\n'.join( [f" '{x[0]}': {x[1]}" for x in files_not_matched_by_any_license_tag.items()]) - self._extracted_from__check_47( + self._print_info( info_str, files_with_uncovered_licenses, files_not_matched_by_any_license_tag, @@ -246,8 +257,8 @@ def _check(self, package: Package): self._success('All licenses found in the code are covered by a ' 'license declaration.') - # TODO Rename this here and in `_check` - def _extracted_from__check_47(self, info_str, files_with_uncovered_licenses, files_not_matched_by_any_license_tag, package): + def _print_info(self, info_str, files_with_uncovered_licenses, + files_not_matched_by_any_license_tag, package): assert info_str != '' self._failed(info_str) self.verbose_output = red( diff --git a/src/ros_license_toolkit/package.py b/src/ros_license_toolkit/package.py index 3f059c6..55dadf0 100644 --- a/src/ros_license_toolkit/package.py +++ b/src/ros_license_toolkit/package.py @@ -44,7 +44,7 @@ def get_spdx_license_name(scan_results: Dict[str, Any]) -> Optional[str]: """Get the SPDX license name from scan results.""" if scan_results['percentage_of_license_text'] >=\ - REQUIRED_PERCENTAGE_OF_LICENSE_TEXT: + REQUIRED_PERCENTAGE_OF_LICENSE_TEXT: return scan_results['detected_license_expression_spdx'] return None From 073e6fb3dbe8249269fad28ab2c76d4183c53637 Mon Sep 17 00:00:00 2001 From: Christian Henkel Date: Mon, 9 Oct 2023 14:52:51 +0200 Subject: [PATCH 07/36] this is all WIP Signed-off-by: Christian Henkel --- src/ros_license_toolkit/copyright.py | 2 ++ test/systemtest/test_copyright.py | 11 +++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/ros_license_toolkit/copyright.py b/src/ros_license_toolkit/copyright.py index c768b7a..24fee6d 100644 --- a/src/ros_license_toolkit/copyright.py +++ b/src/ros_license_toolkit/copyright.py @@ -70,7 +70,9 @@ def __init__(self, pkg): cprs = set() for source_file in license_tag.source_files: fpath = os.path.join(self.pkg.abspath, source_file) + print(fpath) res = get_copyrights(fpath) + print(res) if len(res) == 0: continue for cpr in _get_copyright_strs_from_results(res): diff --git a/test/systemtest/test_copyright.py b/test/systemtest/test_copyright.py index 9ec0d46..577a2b8 100644 --- a/test/systemtest/test_copyright.py +++ b/test/systemtest/test_copyright.py @@ -15,6 +15,9 @@ # limitations under the License. import os + +import pytest + from test.systemtest._test_helpers import make_repo, remove_repo from ros_license_toolkit.copyright import CopyrightPerPkg @@ -42,7 +45,7 @@ def test_copyright(): cpr_secs = CopyrightPerPkg(pkg).copyright_strings assert len(cpr_secs) == 2 - +@pytest.mark.skip(reason="Currently, not the full copyright is extracted TODO") def test_copyright_to_string(): pkg_path = os.path.join(TEST_DATA_FOLDER, "test_pkg_has_code_disjoint") pkg = Package(pkg_path) @@ -96,6 +99,6 @@ def test_write_copyright_file(): ), "r") as f: expected = f.read() assert expected == output - remove_existing_copyright_file( - path=copyright_file_path) - make_repo(TEST_DATA_FOLDER) + # remove_existing_copyright_file( + # path=copyright_file_path) + remove_repo(TEST_DATA_FOLDER) From 5312b77d62d1919048f753d25f461cf2f4ec7951 Mon Sep 17 00:00:00 2001 From: Christian Henkel Date: Mon, 30 Oct 2023 13:33:51 +0100 Subject: [PATCH 08/36] Unnecessary "else" after "return" Signed-off-by: Christian Henkel --- src/ros_license_toolkit/checks.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/ros_license_toolkit/checks.py b/src/ros_license_toolkit/checks.py index d4f7eae..0925c8d 100644 --- a/src/ros_license_toolkit/checks.py +++ b/src/ros_license_toolkit/checks.py @@ -59,17 +59,12 @@ def _success(self, reason: str): def __str__(self) -> str: """Return formatted string for normal output.""" if self.success: - return ( - str(type(self).__name__) - + "\n" - + green(f" SUCCESS {self.reason}") - ) - else: - return ( - str(type(self).__name__) - + "\n" - + red(f" FAILURE {self.reason}") - ) + return \ + str(type(self).__name__) + "\n" + green( + f" SUCCESS {self.reason}") + return \ + str(type(self).__name__) + "\n" + red( + f" FAILURE {self.reason}") def verbose(self) -> str: """Return string with additional information for verbose output.""" From d74714721e5cb986fc2b5792e157059c5e995fcb Mon Sep 17 00:00:00 2001 From: Christian Henkel Date: Mon, 30 Oct 2023 13:42:12 +0100 Subject: [PATCH 09/36] still bad with 32.0.8 Signed-off-by: Christian Henkel --- pyproject.toml | 5 +---- test/systemtest/test_copyright.py | 3 --- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f0cb5f2..a3d7b22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,10 +27,7 @@ keywords = ["ros", "license", "toolkit", "oss", "packages"] dependencies = [ "gitpython", "rospkg", -# TODO v32 changes things, see -# https://github.com/nexB/scancode-toolkit/blob/develop/ -# CHANGELOG.rst#v3200---2023-05-23 - "scancode-toolkit>=32.0.7", + "scancode-toolkit>=32.0.8", "spdx-tools" ] requires-python = ">=3.7" diff --git a/test/systemtest/test_copyright.py b/test/systemtest/test_copyright.py index 24fd377..d20fd3e 100644 --- a/test/systemtest/test_copyright.py +++ b/test/systemtest/test_copyright.py @@ -18,8 +18,6 @@ import os from test.systemtest._test_helpers import make_repo, remove_repo -import pytest - from ros_license_toolkit.copyright import get_copyright_strings_per_pkg from ros_license_toolkit.package import Package, get_packages_in_path @@ -53,7 +51,6 @@ def test_copyright(): assert len(cpr_secs) == 2 -@pytest.mark.skip(reason="Currently, not the full copyright is extracted TODO") def test_copyright_to_string(): """Test if correct content in copyright sections is found.""" pkg_path = os.path.join(TEST_DATA_FOLDER, "test_pkg_has_code_disjoint") From 8f1830444bbcefe4ad179412456fb31e6930f32d Mon Sep 17 00:00:00 2001 From: Christian Henkel Date: Mon, 30 Oct 2023 14:38:58 +0100 Subject: [PATCH 10/36] ignoring those rules Signed-off-by: Christian Henkel --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 7e71b44..1deabf5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [flake8] -ignore = Q000, I100, I201 +ignore = Q000, I100, I201, W503, W504 [bumpver] From f633c28e877da696ae78ed7dd630b9dbd11bc80c Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Fri, 23 Feb 2024 16:43:03 +0100 Subject: [PATCH 11/36] changed method get_spdx_license_name back, WIP Signed-off-by: Anton Utz --- src/ros_license_toolkit/package.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/ros_license_toolkit/package.py b/src/ros_license_toolkit/package.py index 4d16841..c6068c5 100644 --- a/src/ros_license_toolkit/package.py +++ b/src/ros_license_toolkit/package.py @@ -26,8 +26,8 @@ from rospkg.common import PACKAGE_FILE from scancode.api import get_licenses -from ros_license_toolkit.common import (REQUIRED_PERCENTAGE_OF_LICENSE_TEXT, - is_license_text_file) +from ros_license_toolkit.common import (is_license_text_file, + REQUIRED_PERCENTAGE_OF_LICENSE_TEXT) from ros_license_toolkit.copyright import get_copyright_strings_per_pkg from ros_license_toolkit.license_tag import LicenseTag from ros_license_toolkit.repo import NotARepoError, Repo @@ -44,15 +44,10 @@ def get_spdx_license_name(scan_results: Dict[str, Any]) -> Optional[str]: """Get the SPDX license name from scan results.""" - return next( - ( - _license["spdx_license_key"] - for _license in scan_results["licenses"] - if _license["score"] >= 99 - ), - None, - ) - + if scan_results['percentage_of_license_text'] >=\ + REQUIRED_PERCENTAGE_OF_LICENSE_TEXT: + return scan_results['detected_license_expression_spdx'] + return None class PackageException(Exception): """Exception raised when a package is invalid.""" From 1e7a49939763e083aace91a53b8adf9fbf0f0fa7 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Tue, 27 Feb 2024 15:04:34 +0100 Subject: [PATCH 12/36] scancode get_copyrights function filtering word 'anything', changed to second Signed-off-by: Anton Utz --- src/ros_license_toolkit/common.py | 1 - src/ros_license_toolkit/package.py | 2 -- .../copyright_file_contents/test_pkg_has_code_disjoint | 2 +- .../also_src/more_code_with_Apache.py | 2 +- test/systemtest/test_copyright.py | 2 +- 5 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/ros_license_toolkit/common.py b/src/ros_license_toolkit/common.py index 3d551e2..809a1a0 100644 --- a/src/ros_license_toolkit/common.py +++ b/src/ros_license_toolkit/common.py @@ -23,7 +23,6 @@ def is_license_text_file(scan_results: Dict[str, Any]) -> bool: """Check if a file is a license text file.""" - print(f"{scan_results['percentage_of_license_text']=}") return ( scan_results["percentage_of_license_text"] >= REQUIRED_PERCENTAGE_OF_LICENSE_TEXT) diff --git a/src/ros_license_toolkit/package.py b/src/ros_license_toolkit/package.py index c6068c5..c8b5451 100644 --- a/src/ros_license_toolkit/package.py +++ b/src/ros_license_toolkit/package.py @@ -138,12 +138,10 @@ def _run_scan_and_save_results(self): if is_license_text_file(scan_results): self._found_license_texts[fname ] = scan_results - print("license text file") else: # not a license text file but also interesting self._found_files_w_licenses[fname ] = scan_results - print("not license text file") # look also in the repo for license text files if self.repo is not None: for path, res in self.repo.license_text_files.items(): diff --git a/test/_test_data/copyright_file_contents/test_pkg_has_code_disjoint b/test/_test_data/copyright_file_contents/test_pkg_has_code_disjoint index 334d48d..f43d3f0 100644 --- a/test/_test_data/copyright_file_contents/test_pkg_has_code_disjoint +++ b/test/_test_data/copyright_file_contents/test_pkg_has_code_disjoint @@ -22,7 +22,7 @@ License: BSD-3-Clause Files: src/** also_src/** Copyright: 2000 An - 2002 Another + 2002 Second License: Apache-2.0 Apache License diff --git a/test/_test_data/test_pkg_has_code_disjoint/also_src/more_code_with_Apache.py b/test/_test_data/test_pkg_has_code_disjoint/also_src/more_code_with_Apache.py index a9ebbef..cb1e1e8 100644 --- a/test/_test_data/test_pkg_has_code_disjoint/also_src/more_code_with_Apache.py +++ b/test/_test_data/test_pkg_has_code_disjoint/also_src/more_code_with_Apache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002 Another Author +# Copyright (C) 2002 Second Author # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/test/systemtest/test_copyright.py b/test/systemtest/test_copyright.py index 6ece26b..381220a 100644 --- a/test/systemtest/test_copyright.py +++ b/test/systemtest/test_copyright.py @@ -61,7 +61,7 @@ def test_copyright_to_string(): assert 'Foo Bar' in str(cprs) assert '2000' in str(cprs) assert '2002' in str(cprs) - assert 'Another' in str(cprs) + assert 'Second' in str(cprs) def test_get_copyright_file_contents(): From b5d3b25402e72d63a0d1516444d02648f3aaa09e Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Tue, 27 Feb 2024 15:14:00 +0100 Subject: [PATCH 13/36] linter issues Signed-off-by: Anton Utz --- src/ros_license_toolkit/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ros_license_toolkit/package.py b/src/ros_license_toolkit/package.py index c8b5451..1360c30 100644 --- a/src/ros_license_toolkit/package.py +++ b/src/ros_license_toolkit/package.py @@ -26,8 +26,8 @@ from rospkg.common import PACKAGE_FILE from scancode.api import get_licenses -from ros_license_toolkit.common import (is_license_text_file, - REQUIRED_PERCENTAGE_OF_LICENSE_TEXT) +from ros_license_toolkit.common import (REQUIRED_PERCENTAGE_OF_LICENSE_TEXT, + is_license_text_file) from ros_license_toolkit.copyright import get_copyright_strings_per_pkg from ros_license_toolkit.license_tag import LicenseTag from ros_license_toolkit.repo import NotARepoError, Repo @@ -49,6 +49,7 @@ def get_spdx_license_name(scan_results: Dict[str, Any]) -> Optional[str]: return scan_results['detected_license_expression_spdx'] return None + class PackageException(Exception): """Exception raised when a package is invalid.""" From fd8d6de089374ec9e85b00ea753f649f51332c36 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Tue, 27 Feb 2024 16:16:19 +0100 Subject: [PATCH 14/36] Added spaces around comment for scancode to recognize copyright message Signed-off-by: Anton Utz --- .../another_path/yet_another_file.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/_test_data/test_pkg_has_code_of_different_license_and_tag/another_path/yet_another_file.xml b/test/_test_data/test_pkg_has_code_of_different_license_and_tag/another_path/yet_another_file.xml index 443c64d..0ba5c64 100644 --- a/test/_test_data/test_pkg_has_code_of_different_license_and_tag/another_path/yet_another_file.xml +++ b/test/_test_data/test_pkg_has_code_of_different_license_and_tag/another_path/yet_another_file.xml @@ -1,2 +1,2 @@ - + This is under BSD-3-Clause, just fyi ... \ No newline at end of file From 648de940f0657c9eb110ec51f6328bfbf80edb38 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Mon, 4 Mar 2024 16:04:22 +0100 Subject: [PATCH 15/36] added check for only one license text which is in spdx to start handling wrong tags as the license of the file Signed-off-by: Anton Utz --- src/ros_license_toolkit/checks.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/ros_license_toolkit/checks.py b/src/ros_license_toolkit/checks.py index 706b935..0420e51 100644 --- a/src/ros_license_toolkit/checks.py +++ b/src/ros_license_toolkit/checks.py @@ -204,6 +204,15 @@ def _check(self, package: Package): "license text:\n" + "\n".join( [f" '{x[0]}': {x[1]}" for x in license_tags_without_license_text.items()])) + if len(found_license_texts) == 1: + # if exactly one license text is found, + # treat wrong license tag internally as this license + #optional check for similarity between tag and file + license_file_tag_spdx: Optional[str] = \ + found_license_texts[next(iter(found_license_texts))]\ + ['detected_license_expression_spdx'] + if license_file_tag_spdx: + print('here') else: self._failed( "The following license tags do not " From 59be16e7d310de0f7af8428e3f0307f4b58d2e50 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Mon, 4 Mar 2024 18:43:25 +0100 Subject: [PATCH 16/36] When only one SPDX License File exists, make a warning to change the wrong tag Signed-off-by: Anton Utz --- src/ros_license_toolkit/checks.py | 21 +++++++++++++++++++-- src/ros_license_toolkit/package.py | 4 ++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/ros_license_toolkit/checks.py b/src/ros_license_toolkit/checks.py index 0420e51..9d97529 100644 --- a/src/ros_license_toolkit/checks.py +++ b/src/ros_license_toolkit/checks.py @@ -205,14 +205,14 @@ def _check(self, package: Package): [f" '{x[0]}': {x[1]}" for x in license_tags_without_license_text.items()])) if len(found_license_texts) == 1: - # if exactly one license text is found, + # if exactly one license text is found, # treat wrong license tag internally as this license #optional check for similarity between tag and file license_file_tag_spdx: Optional[str] = \ found_license_texts[next(iter(found_license_texts))]\ ['detected_license_expression_spdx'] if license_file_tag_spdx: - print('here') + package.inofficial_license_tag = license_file_tag_spdx else: self._failed( "The following license tags do not " @@ -238,6 +238,7 @@ def _check(self, package: Package): declared_licenses = package.license_tags files_with_uncovered_licenses: Dict[str, List[str]] = {} files_not_matched_by_any_license_tag: Dict[str, List[str]] = {} + files_with_inofficial_tag: Dict[str, List[str]] = {} for fname, found_licenses in package.found_files_w_licenses.items(): if fname in package.get_license_files(): @@ -248,6 +249,14 @@ def _check(self, package: Package): licenses = found_licenses_str.split(' AND ') for license_str in licenses: if license_str not in declared_licenses: + # this license has an inofficial tag + if license_str == package.inofficial_license_tag: + if fname not in files_with_inofficial_tag: + files_with_inofficial_tag[fname] = [] + files_with_inofficial_tag[fname].append(license_str) + files_with_inofficial_tag[fname].append( + next(iter(declared_licenses))) + continue # this license is not declared by any license tag if fname not in files_with_uncovered_licenses: files_with_uncovered_licenses[fname] = [] @@ -275,6 +284,14 @@ def _check(self, package: Package): files_not_matched_by_any_license_tag, package, ) + elif files_with_inofficial_tag: + info_str = '' + info_str += 'For the following files, please change the ' +\ + 'License Tag in the package file to SPDX format:\n' +\ + '\n'.join( + [f" '{x[0]}' is of {x[1][0]} but its Tag is {x[1][1]}." for x in + files_with_inofficial_tag.items()]) + self._warning(info_str) elif len(files_not_matched_by_any_license_tag) > 0: info_str = '' info_str += '\nThe following files contain licenses that ' +\ diff --git a/src/ros_license_toolkit/package.py b/src/ros_license_toolkit/package.py index 1360c30..3300fb2 100644 --- a/src/ros_license_toolkit/package.py +++ b/src/ros_license_toolkit/package.py @@ -94,6 +94,10 @@ def __init__(self, path: str, repo: Optional[Repo] = None): # this is Optional, because it is only evaluated on the first call self._license_tags: Optional[Dict[str, LicenseTag]] = None + # If the tag is wrong (like BSD) but the actual license can + # be found out through declaration, this field contains the tag + self.inofficial_license_tag: Optional[str] = None + def _get_path_relative_to_pkg(self, path: str) -> str: """Get path relative to pkg root""" return os.path.relpath(path, self.abspath) From 27393ff9af4b3333b77254327a1300bb038fd1e6 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Tue, 5 Mar 2024 12:12:23 +0100 Subject: [PATCH 17/36] Added check if multiple files exist Signed-off-by: Anton Utz --- src/ros_license_toolkit/checks.py | 23 ++++++++++++----------- src/ros_license_toolkit/package.py | 2 +- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/ros_license_toolkit/checks.py b/src/ros_license_toolkit/checks.py index 9d97529..d421591 100644 --- a/src/ros_license_toolkit/checks.py +++ b/src/ros_license_toolkit/checks.py @@ -153,6 +153,7 @@ def _check(self, package: Package): return license_tags_without_license_text: Dict[LicenseTag, str] = {} missing_license_texts_status: Dict[LicenseTag, Status] = {} + files_with_wrong_tags: Dict[LicenseTag, Dict[str, str]] = {} found_license_texts = package.found_license_texts for license_tag in package.license_tags.values(): if not license_tag.has_license_text_file(): @@ -195,6 +196,9 @@ def _check(self, package: Package): f"of license {actual_license} but tag is " +\ f"{license_tag.get_license_id()}." missing_license_texts_status[license_tag] = Status.WARNING + files_with_wrong_tags[license_tag] = \ + {'actual_license': actual_license, + 'license_tag': license_tag.get_license_id()} continue if len(license_tags_without_license_text) > 0: if max(missing_license_texts_status.values()) == Status.WARNING: @@ -204,15 +208,12 @@ def _check(self, package: Package): "license text:\n" + "\n".join( [f" '{x[0]}': {x[1]}" for x in license_tags_without_license_text.items()])) - if len(found_license_texts) == 1: + for entry in files_with_wrong_tags.items(): # if exactly one license text is found, # treat wrong license tag internally as this license - #optional check for similarity between tag and file - license_file_tag_spdx: Optional[str] = \ - found_license_texts[next(iter(found_license_texts))]\ - ['detected_license_expression_spdx'] - if license_file_tag_spdx: - package.inofficial_license_tag = license_file_tag_spdx + # optional check for similarity between tag and file + package.inofficial_license_tag[entry[1]['actual_license']]\ + = entry[1]['license_tag'] else: self._failed( "The following license tags do not " @@ -250,12 +251,12 @@ def _check(self, package: Package): for license_str in licenses: if license_str not in declared_licenses: # this license has an inofficial tag - if license_str == package.inofficial_license_tag: + if license_str in package.inofficial_license_tag.keys(): if fname not in files_with_inofficial_tag: files_with_inofficial_tag[fname] = [] files_with_inofficial_tag[fname].append(license_str) files_with_inofficial_tag[fname].append( - next(iter(declared_licenses))) + package.inofficial_license_tag[license_str]) continue # this license is not declared by any license tag if fname not in files_with_uncovered_licenses: @@ -289,8 +290,8 @@ def _check(self, package: Package): info_str += 'For the following files, please change the ' +\ 'License Tag in the package file to SPDX format:\n' +\ '\n'.join( - [f" '{x[0]}' is of {x[1][0]} but its Tag is {x[1][1]}." for x in - files_with_inofficial_tag.items()]) + [f" '{x[0]}' is of {x[1][0]} but its Tag is {x[1][1]}." + for x in files_with_inofficial_tag.items()]) self._warning(info_str) elif len(files_not_matched_by_any_license_tag) > 0: info_str = '' diff --git a/src/ros_license_toolkit/package.py b/src/ros_license_toolkit/package.py index 3300fb2..a916c57 100644 --- a/src/ros_license_toolkit/package.py +++ b/src/ros_license_toolkit/package.py @@ -96,7 +96,7 @@ def __init__(self, path: str, repo: Optional[Repo] = None): # If the tag is wrong (like BSD) but the actual license can # be found out through declaration, this field contains the tag - self.inofficial_license_tag: Optional[str] = None + self.inofficial_license_tag: Dict[str, str] = {} def _get_path_relative_to_pkg(self, path: str) -> str: """Get path relative to pkg root""" From 28cede10da7f734d6dd9f1cce9f135b9e57078a1 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Tue, 5 Mar 2024 15:23:03 +0100 Subject: [PATCH 18/36] restructured LicenseTextExistsCheck Signed-off-by: Anton Utz --- src/ros_license_toolkit/checks.py | 105 ++++++++++++++++-------------- 1 file changed, 55 insertions(+), 50 deletions(-) diff --git a/src/ros_license_toolkit/checks.py b/src/ros_license_toolkit/checks.py index d421591..4e8536e 100644 --- a/src/ros_license_toolkit/checks.py +++ b/src/ros_license_toolkit/checks.py @@ -19,7 +19,7 @@ import os from enum import IntEnum from pprint import pformat -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from ros_license_toolkit.license_tag import (LicenseTag, is_license_name_in_spdx_list) @@ -146,87 +146,92 @@ def _check(self, package: Package): class LicenseTextExistsCheck(Check): """This ensures that the license text file referenced by the tag exists.""" + license_tags_without_license_text: Dict[LicenseTag, str] = {} + missing_license_texts_status: Dict[LicenseTag, Status] = {} + files_with_wrong_tags: Dict[LicenseTag, Dict[str, str]] = {} + found_license_texts: Dict[str, Any] = {} def _check(self, package: Package): if len(package.license_tags) == 0: self._failed("No license tag defined.") return - license_tags_without_license_text: Dict[LicenseTag, str] = {} - missing_license_texts_status: Dict[LicenseTag, Status] = {} - files_with_wrong_tags: Dict[LicenseTag, Dict[str, str]] = {} - found_license_texts = package.found_license_texts + self.found_license_texts = package.found_license_texts + self._check_each_license_tag(package) + if len(self.license_tags_without_license_text) > 0: + if max(self.missing_license_texts_status. + values()) == Status.WARNING: + self._warning( + "Since they are not in the SPDX list, " + "we can not check if these tags have the correct " + "license text:\n" + "\n".join( + [f" '{x[0]}': {x[1]}" for x in + self.license_tags_without_license_text.items()])) + for entry in self.files_with_wrong_tags.items(): + # For every entry that has spdx license file but + # not matching, not spdx tag (like BSD) + package.inofficial_license_tag[entry[1]['actual_license']]\ + = entry[1]['license_tag'] + else: + self._failed( + "The following license tags do not " + "have a valid license text " + "file:\n" + "\n".join( + [f" '{x[0]}': {x[1]}" for x in + self.license_tags_without_license_text.items()])) + self.verbose_output = red( + "\n".join([f" '{x[0]}': {x[1]}" for x in + self.found_license_texts.items()])) + else: + self._success("All license tags have a valid license text file.") + + def _check_each_license_tag(self, package: Package) -> None: + """Checks each license tag and adds them to class dicts""" for license_tag in package.license_tags.values(): if not license_tag.has_license_text_file(): - license_tags_without_license_text[ + self.license_tags_without_license_text[ license_tag] = "No license text file defined." - missing_license_texts_status[license_tag] = Status.FAILURE + self.missing_license_texts_status[license_tag] = Status.FAILURE continue license_text_file = license_tag.get_license_text_file() if not os.path.exists( os.path.join(package.abspath, license_text_file)): - license_tags_without_license_text[license_tag] =\ + self.license_tags_without_license_text[license_tag] =\ f"License text file '{license_text_file}' does not exist." - missing_license_texts_status[license_tag] = Status.FAILURE + self.missing_license_texts_status[license_tag] = Status.FAILURE continue - if license_text_file not in found_license_texts: - license_tags_without_license_text[license_tag] =\ + if license_text_file not in self.found_license_texts: + self.license_tags_without_license_text[license_tag] =\ f"License text file '{license_text_file}' not included" +\ " in scan results." - missing_license_texts_status[license_tag] = Status.FAILURE + self.missing_license_texts_status[license_tag] = Status.FAILURE continue if not is_license_text_file( - found_license_texts[license_text_file]): - license_tags_without_license_text[license_tag] =\ + self.found_license_texts[license_text_file]): + self.license_tags_without_license_text[license_tag] =\ f"License text file '{license_text_file}' is not " +\ "recognized as license text." - missing_license_texts_status[license_tag] = Status.FAILURE + self.missing_license_texts_status[license_tag] = Status.FAILURE continue actual_license: Optional[str] = get_spdx_license_name( - found_license_texts[license_text_file]) + self.found_license_texts[license_text_file]) if actual_license is None: - license_tags_without_license_text[ + self.license_tags_without_license_text[ license_tag ] = f"License text file '{license_text_file}'" +\ " is not recognized as license text." - missing_license_texts_status[license_tag] = Status.FAILURE + self.missing_license_texts_status[license_tag] = Status.FAILURE continue if actual_license != license_tag.get_license_id(): - license_tags_without_license_text[license_tag] =\ + self.license_tags_without_license_text[license_tag] =\ f"License text file '{license_text_file}' is " +\ f"of license {actual_license} but tag is " +\ f"{license_tag.get_license_id()}." - missing_license_texts_status[license_tag] = Status.WARNING - files_with_wrong_tags[license_tag] = \ - {'actual_license': actual_license, - 'license_tag': license_tag.get_license_id()} + self.missing_license_texts_status[license_tag] = Status.WARNING + if license_tag.get_license_id() is not None: + self.files_with_wrong_tags[license_tag] = \ + {'actual_license': actual_license, + 'license_tag': license_tag.get_license_id()} continue - if len(license_tags_without_license_text) > 0: - if max(missing_license_texts_status.values()) == Status.WARNING: - self._warning( - "Since they are not in the SPDX list, " - "we can not check if these tags have the correct " - "license text:\n" + "\n".join( - [f" '{x[0]}': {x[1]}" for x in - license_tags_without_license_text.items()])) - for entry in files_with_wrong_tags.items(): - # if exactly one license text is found, - # treat wrong license tag internally as this license - # optional check for similarity between tag and file - package.inofficial_license_tag[entry[1]['actual_license']]\ - = entry[1]['license_tag'] - else: - self._failed( - "The following license tags do not " - "have a valid license text " - "file:\n" + "\n".join( - [f" '{x[0]}': {x[1]}" for x in - license_tags_without_license_text.items()])) - self.verbose_output = red( - "\n".join([f" '{x[0]}': {x[1]}" for x in - found_license_texts.items()])) - - else: - self._success("All license tags have a valid license text file.") class LicensesInCodeCheck(Check): From eec394c070000f19df74d2c9dc0537776ed5b41d Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Tue, 5 Mar 2024 17:47:06 +0100 Subject: [PATCH 19/36] reverted restructure of check since pytest has trouble with it Signed-off-by: Anton Utz --- src/ros_license_toolkit/checks.py | 102 ++++++++++++++---------------- 1 file changed, 48 insertions(+), 54 deletions(-) diff --git a/src/ros_license_toolkit/checks.py b/src/ros_license_toolkit/checks.py index 4e8536e..356d6e9 100644 --- a/src/ros_license_toolkit/checks.py +++ b/src/ros_license_toolkit/checks.py @@ -146,93 +146,87 @@ def _check(self, package: Package): class LicenseTextExistsCheck(Check): """This ensures that the license text file referenced by the tag exists.""" - license_tags_without_license_text: Dict[LicenseTag, str] = {} - missing_license_texts_status: Dict[LicenseTag, Status] = {} - files_with_wrong_tags: Dict[LicenseTag, Dict[str, str]] = {} - found_license_texts: Dict[str, Any] = {} def _check(self, package: Package): if len(package.license_tags) == 0: self._failed("No license tag defined.") return - self.found_license_texts = package.found_license_texts - self._check_each_license_tag(package) - if len(self.license_tags_without_license_text) > 0: - if max(self.missing_license_texts_status. - values()) == Status.WARNING: - self._warning( - "Since they are not in the SPDX list, " - "we can not check if these tags have the correct " - "license text:\n" + "\n".join( - [f" '{x[0]}': {x[1]}" for x in - self.license_tags_without_license_text.items()])) - for entry in self.files_with_wrong_tags.items(): - # For every entry that has spdx license file but - # not matching, not spdx tag (like BSD) - package.inofficial_license_tag[entry[1]['actual_license']]\ - = entry[1]['license_tag'] - else: - self._failed( - "The following license tags do not " - "have a valid license text " - "file:\n" + "\n".join( - [f" '{x[0]}': {x[1]}" for x in - self.license_tags_without_license_text.items()])) - self.verbose_output = red( - "\n".join([f" '{x[0]}': {x[1]}" for x in - self.found_license_texts.items()])) - else: - self._success("All license tags have a valid license text file.") - - def _check_each_license_tag(self, package: Package) -> None: - """Checks each license tag and adds them to class dicts""" + license_tags_without_license_text: Dict[LicenseTag, str] = {} + missing_license_texts_status: Dict[LicenseTag, Status] = {} + files_with_wrong_tags: Dict[LicenseTag, Dict[str, str]] = {} + found_license_texts = package.found_license_texts for license_tag in package.license_tags.values(): if not license_tag.has_license_text_file(): - self.license_tags_without_license_text[ + license_tags_without_license_text[ license_tag] = "No license text file defined." - self.missing_license_texts_status[license_tag] = Status.FAILURE + missing_license_texts_status[license_tag] = Status.FAILURE continue license_text_file = license_tag.get_license_text_file() if not os.path.exists( os.path.join(package.abspath, license_text_file)): - self.license_tags_without_license_text[license_tag] =\ + license_tags_without_license_text[license_tag] =\ f"License text file '{license_text_file}' does not exist." - self.missing_license_texts_status[license_tag] = Status.FAILURE + missing_license_texts_status[license_tag] = Status.FAILURE continue - if license_text_file not in self.found_license_texts: - self.license_tags_without_license_text[license_tag] =\ + if license_text_file not in found_license_texts: + license_tags_without_license_text[license_tag] =\ f"License text file '{license_text_file}' not included" +\ " in scan results." - self.missing_license_texts_status[license_tag] = Status.FAILURE + missing_license_texts_status[license_tag] = Status.FAILURE continue if not is_license_text_file( - self.found_license_texts[license_text_file]): - self.license_tags_without_license_text[license_tag] =\ + found_license_texts[license_text_file]): + license_tags_without_license_text[license_tag] =\ f"License text file '{license_text_file}' is not " +\ "recognized as license text." - self.missing_license_texts_status[license_tag] = Status.FAILURE + missing_license_texts_status[license_tag] = Status.FAILURE continue actual_license: Optional[str] = get_spdx_license_name( - self.found_license_texts[license_text_file]) + found_license_texts[license_text_file]) if actual_license is None: - self.license_tags_without_license_text[ + license_tags_without_license_text[ license_tag ] = f"License text file '{license_text_file}'" +\ " is not recognized as license text." - self.missing_license_texts_status[license_tag] = Status.FAILURE + missing_license_texts_status[license_tag] = Status.FAILURE continue if actual_license != license_tag.get_license_id(): - self.license_tags_without_license_text[license_tag] =\ + license_tags_without_license_text[license_tag] =\ f"License text file '{license_text_file}' is " +\ f"of license {actual_license} but tag is " +\ f"{license_tag.get_license_id()}." - self.missing_license_texts_status[license_tag] = Status.WARNING - if license_tag.get_license_id() is not None: - self.files_with_wrong_tags[license_tag] = \ - {'actual_license': actual_license, - 'license_tag': license_tag.get_license_id()} + missing_license_texts_status[license_tag] = Status.WARNING + files_with_wrong_tags[license_tag] = \ + {'actual_license': actual_license, + 'license_tag': license_tag.get_license_id()} continue + if len(license_tags_without_license_text) > 0: + if max(missing_license_texts_status.values()) == Status.WARNING: + self._warning( + "Since they are not in the SPDX list, " + "we can not check if these tags have the correct " + "license text:\n" + "\n".join( + [f" '{x[0]}': {x[1]}" for x in + license_tags_without_license_text.items()])) + for entry in files_with_wrong_tags.items(): + # if exactly one license text is found, + # treat wrong license tag internally as this license + # optional check for similarity between tag and file + package.inofficial_license_tag[entry[1]['actual_license']]\ + = entry[1]['license_tag'] + else: + self._failed( + "The following license tags do not " + "have a valid license text " + "file:\n" + "\n".join( + [f" '{x[0]}': {x[1]}" for x in + license_tags_without_license_text.items()])) + self.verbose_output = red( + "\n".join([f" '{x[0]}': {x[1]}" for x in + found_license_texts.items()])) + else: + self._success("All license tags have a valid license text file.") class LicensesInCodeCheck(Check): """Check if all found licenses have a declaration in the package.xml.""" From b7859b49ebccbe3d7d68d993c61ebd3764e8b1b3 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Wed, 6 Mar 2024 11:54:47 +0100 Subject: [PATCH 20/36] moved LicenseTextExists checks to seperate function Signed-off-by: Anton Utz --- src/ros_license_toolkit/checks.py | 93 ++++++++++++++++++------------- 1 file changed, 54 insertions(+), 39 deletions(-) diff --git a/src/ros_license_toolkit/checks.py b/src/ros_license_toolkit/checks.py index 356d6e9..502ca22 100644 --- a/src/ros_license_toolkit/checks.py +++ b/src/ros_license_toolkit/checks.py @@ -154,79 +154,94 @@ def _check(self, package: Package): license_tags_without_license_text: Dict[LicenseTag, str] = {} missing_license_texts_status: Dict[LicenseTag, Status] = {} files_with_wrong_tags: Dict[LicenseTag, Dict[str, str]] = {} - found_license_texts = package.found_license_texts + + found_license_texts = self._check_licenses( + package, + license_tags_without_license_text, + missing_license_texts_status, + files_with_wrong_tags) + + if len(license_tags_without_license_text) > 0: + if max(missing_license_texts_status.values()) == Status.WARNING: + self._warning( + "Since they are not in the SPDX list, " + "we can not check if these tags have the correct " + "license text:\n" + "\n".join( + [f" '{x[0]}': {x[1]}" for x in + license_tags_without_license_text.items()])) + for entry in files_with_wrong_tags.items(): + # if exactly one license text is found, + # treat wrong license tag internally as this license + # optional check for similarity between tag and file + package.inofficial_license_tag[entry[1]['actual_license']]\ + = entry[1]['license_tag'] + else: + self._failed( + "The following license tags do not " + "have a valid license text " + "file:\n" + "\n".join( + [f" '{x[0]}': {x[1]}" for x in + license_tags_without_license_text.items()])) + self.verbose_output = red( + "\n".join([f" '{x[0]}': {x[1]}" for x in + found_license_texts.items()])) + else: + self._success("All license tags have a valid license text file.") + + def _check_licenses(self, + package: Package, + tags_without_text: Dict[LicenseTag, str], + missing_texts_status: Dict[LicenseTag, Status], + files_with_wrong_tags: Dict[LicenseTag, Dict[str, str]] + ) -> Dict[str, Any]: + found_license_texts: Dict[str, Any] = package.found_license_texts for license_tag in package.license_tags.values(): if not license_tag.has_license_text_file(): - license_tags_without_license_text[ + tags_without_text[ license_tag] = "No license text file defined." - missing_license_texts_status[license_tag] = Status.FAILURE + missing_texts_status[license_tag] = Status.FAILURE continue license_text_file = license_tag.get_license_text_file() if not os.path.exists( os.path.join(package.abspath, license_text_file)): - license_tags_without_license_text[license_tag] =\ + tags_without_text[license_tag] =\ f"License text file '{license_text_file}' does not exist." - missing_license_texts_status[license_tag] = Status.FAILURE + missing_texts_status[license_tag] = Status.FAILURE continue if license_text_file not in found_license_texts: - license_tags_without_license_text[license_tag] =\ + tags_without_text[license_tag] =\ f"License text file '{license_text_file}' not included" +\ " in scan results." - missing_license_texts_status[license_tag] = Status.FAILURE + missing_texts_status[license_tag] = Status.FAILURE continue if not is_license_text_file( found_license_texts[license_text_file]): - license_tags_without_license_text[license_tag] =\ + tags_without_text[license_tag] =\ f"License text file '{license_text_file}' is not " +\ "recognized as license text." - missing_license_texts_status[license_tag] = Status.FAILURE + missing_texts_status[license_tag] = Status.FAILURE continue actual_license: Optional[str] = get_spdx_license_name( found_license_texts[license_text_file]) if actual_license is None: - license_tags_without_license_text[ + tags_without_text[ license_tag ] = f"License text file '{license_text_file}'" +\ " is not recognized as license text." - missing_license_texts_status[license_tag] = Status.FAILURE + missing_texts_status[license_tag] = Status.FAILURE continue if actual_license != license_tag.get_license_id(): - license_tags_without_license_text[license_tag] =\ + tags_without_text[license_tag] =\ f"License text file '{license_text_file}' is " +\ f"of license {actual_license} but tag is " +\ f"{license_tag.get_license_id()}." - missing_license_texts_status[license_tag] = Status.WARNING + missing_texts_status[license_tag] = Status.WARNING files_with_wrong_tags[license_tag] = \ {'actual_license': actual_license, 'license_tag': license_tag.get_license_id()} continue - if len(license_tags_without_license_text) > 0: - if max(missing_license_texts_status.values()) == Status.WARNING: - self._warning( - "Since they are not in the SPDX list, " - "we can not check if these tags have the correct " - "license text:\n" + "\n".join( - [f" '{x[0]}': {x[1]}" for x in - license_tags_without_license_text.items()])) - for entry in files_with_wrong_tags.items(): - # if exactly one license text is found, - # treat wrong license tag internally as this license - # optional check for similarity between tag and file - package.inofficial_license_tag[entry[1]['actual_license']]\ - = entry[1]['license_tag'] - else: - self._failed( - "The following license tags do not " - "have a valid license text " - "file:\n" + "\n".join( - [f" '{x[0]}': {x[1]}" for x in - license_tags_without_license_text.items()])) - self.verbose_output = red( - "\n".join([f" '{x[0]}': {x[1]}" for x in - found_license_texts.items()])) + return found_license_texts - else: - self._success("All license tags have a valid license text file.") class LicensesInCodeCheck(Check): """Check if all found licenses have a declaration in the package.xml.""" From 9aefe6a68a98f318dc423247351a1bc1215a54d5 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Wed, 6 Mar 2024 12:05:12 +0100 Subject: [PATCH 21/36] made own evaluation function for licenseInCodeCheck Signed-off-by: Anton Utz --- src/ros_license_toolkit/checks.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/ros_license_toolkit/checks.py b/src/ros_license_toolkit/checks.py index 502ca22..f99dc17 100644 --- a/src/ros_license_toolkit/checks.py +++ b/src/ros_license_toolkit/checks.py @@ -287,6 +287,17 @@ def _check(self, package: Package): files_not_matched_by_any_license_tag[fname].append( license_str) continue + self._evaluate_result(package, + files_with_uncovered_licenses, + files_not_matched_by_any_license_tag, + files_with_inofficial_tag) + + def _evaluate_result(self, + package: Package, + files_with_uncovered_licenses: Dict[str, List[str]], + files_not_matched_by_any_tag: Dict[str, List[str]], + files_with_inofficial_tag: Dict[str, List[str]] + ) -> None: if files_with_uncovered_licenses: info_str = '' info_str += '\nThe following files contain licenses that ' +\ @@ -296,7 +307,7 @@ def _check(self, package: Package): self._print_info( info_str, files_with_uncovered_licenses, - files_not_matched_by_any_license_tag, + files_not_matched_by_any_tag, package, ) elif files_with_inofficial_tag: @@ -307,17 +318,17 @@ def _check(self, package: Package): [f" '{x[0]}' is of {x[1][0]} but its Tag is {x[1][1]}." for x in files_with_inofficial_tag.items()]) self._warning(info_str) - elif len(files_not_matched_by_any_license_tag) > 0: + elif len(files_not_matched_by_any_tag) > 0: info_str = '' info_str += '\nThe following files contain licenses that ' +\ 'are covered by a license tag but are not listed in ' +\ 'the source files of the license tag:\n' + '\n'.join( [f" '{x[0]}': {x[1]}" for x in - files_not_matched_by_any_license_tag.items()]) + files_not_matched_by_any_tag.items()]) self._print_info( info_str, files_with_uncovered_licenses, - files_not_matched_by_any_license_tag, + files_not_matched_by_any_tag, package, ) else: From 945cd93fc1bf9f00648c031da37d612525141ac5 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Wed, 6 Mar 2024 12:22:55 +0100 Subject: [PATCH 22/36] added irgnore flag for too many attributes localy to package.py since this is the only place where saving inofficial licenses makes sense Signed-off-by: Anton Utz --- src/ros_license_toolkit/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ros_license_toolkit/package.py b/src/ros_license_toolkit/package.py index a916c57..6ab22bc 100644 --- a/src/ros_license_toolkit/package.py +++ b/src/ros_license_toolkit/package.py @@ -68,6 +68,9 @@ class Package: """This represents a ros package, defined by its `path` (absolute) and results within it.""" + # pylint: disable=too-many-instance-attributes + # Eight is reasonable in this case. + def __init__(self, path: str, repo: Optional[Repo] = None): # absolute path to this package self.abspath: str = path From 622738eac823cd7e16ce673269f772b627634fa1 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Wed, 6 Mar 2024 12:25:11 +0100 Subject: [PATCH 23/36] Converted check_licenses mehtod to function Signed-off-by: Anton Utz --- src/ros_license_toolkit/checks.py | 110 +++++++++++++++--------------- 1 file changed, 56 insertions(+), 54 deletions(-) diff --git a/src/ros_license_toolkit/checks.py b/src/ros_license_toolkit/checks.py index f99dc17..d8ee48b 100644 --- a/src/ros_license_toolkit/checks.py +++ b/src/ros_license_toolkit/checks.py @@ -155,7 +155,7 @@ def _check(self, package: Package): missing_license_texts_status: Dict[LicenseTag, Status] = {} files_with_wrong_tags: Dict[LicenseTag, Dict[str, str]] = {} - found_license_texts = self._check_licenses( + found_license_texts = check_licenses( package, license_tags_without_license_text, missing_license_texts_status, @@ -188,59 +188,61 @@ def _check(self, package: Package): else: self._success("All license tags have a valid license text file.") - def _check_licenses(self, - package: Package, - tags_without_text: Dict[LicenseTag, str], - missing_texts_status: Dict[LicenseTag, Status], - files_with_wrong_tags: Dict[LicenseTag, Dict[str, str]] - ) -> Dict[str, Any]: - found_license_texts: Dict[str, Any] = package.found_license_texts - for license_tag in package.license_tags.values(): - if not license_tag.has_license_text_file(): - tags_without_text[ - license_tag] = "No license text file defined." - missing_texts_status[license_tag] = Status.FAILURE - continue - license_text_file = license_tag.get_license_text_file() - if not os.path.exists( - os.path.join(package.abspath, license_text_file)): - tags_without_text[license_tag] =\ - f"License text file '{license_text_file}' does not exist." - missing_texts_status[license_tag] = Status.FAILURE - continue - if license_text_file not in found_license_texts: - tags_without_text[license_tag] =\ - f"License text file '{license_text_file}' not included" +\ - " in scan results." - missing_texts_status[license_tag] = Status.FAILURE - continue - if not is_license_text_file( - found_license_texts[license_text_file]): - tags_without_text[license_tag] =\ - f"License text file '{license_text_file}' is not " +\ - "recognized as license text." - missing_texts_status[license_tag] = Status.FAILURE - continue - actual_license: Optional[str] = get_spdx_license_name( - found_license_texts[license_text_file]) - if actual_license is None: - tags_without_text[ - license_tag - ] = f"License text file '{license_text_file}'" +\ - " is not recognized as license text." - missing_texts_status[license_tag] = Status.FAILURE - continue - if actual_license != license_tag.get_license_id(): - tags_without_text[license_tag] =\ - f"License text file '{license_text_file}' is " +\ - f"of license {actual_license} but tag is " +\ - f"{license_tag.get_license_id()}." - missing_texts_status[license_tag] = Status.WARNING - files_with_wrong_tags[license_tag] = \ - {'actual_license': actual_license, - 'license_tag': license_tag.get_license_id()} - continue - return found_license_texts + +def check_licenses(package: Package, + tags_without_text: Dict[LicenseTag, str], + missing_texts_status: Dict[LicenseTag, Status], + files_with_wrong_tags: Dict[LicenseTag, Dict[str, str]] + ) -> Dict[str, Any]: + '''checks each license tag for the corresponding license text. Also + detects inofficial licenses (When tag is other than SPDX license file)''' + found_license_texts: Dict[str, Any] = package.found_license_texts + for license_tag in package.license_tags.values(): + if not license_tag.has_license_text_file(): + tags_without_text[ + license_tag] = "No license text file defined." + missing_texts_status[license_tag] = Status.FAILURE + continue + license_text_file = license_tag.get_license_text_file() + if not os.path.exists( + os.path.join(package.abspath, license_text_file)): + tags_without_text[license_tag] =\ + f"License text file '{license_text_file}' does not exist." + missing_texts_status[license_tag] = Status.FAILURE + continue + if license_text_file not in found_license_texts: + tags_without_text[license_tag] =\ + f"License text file '{license_text_file}' not included" +\ + " in scan results." + missing_texts_status[license_tag] = Status.FAILURE + continue + if not is_license_text_file( + found_license_texts[license_text_file]): + tags_without_text[license_tag] =\ + f"License text file '{license_text_file}' is not " +\ + "recognized as license text." + missing_texts_status[license_tag] = Status.FAILURE + continue + actual_license: Optional[str] = get_spdx_license_name( + found_license_texts[license_text_file]) + if actual_license is None: + tags_without_text[ + license_tag + ] = f"License text file '{license_text_file}'" +\ + " is not recognized as license text." + missing_texts_status[license_tag] = Status.FAILURE + continue + if actual_license != license_tag.get_license_id(): + tags_without_text[license_tag] =\ + f"License text file '{license_text_file}' is " +\ + f"of license {actual_license} but tag is " +\ + f"{license_tag.get_license_id()}." + missing_texts_status[license_tag] = Status.WARNING + files_with_wrong_tags[license_tag] = \ + {'actual_license': actual_license, + 'license_tag': license_tag.get_license_id()} + continue + return found_license_texts class LicensesInCodeCheck(Check): From 75c2d63d15701aac4d3550b0977e1b75d80b0fac Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Wed, 6 Mar 2024 13:14:19 +0100 Subject: [PATCH 24/36] restructured Check LicensesInCode and TextExists to work with attributes instead of local vars Signed-off-by: Anton Utz --- src/ros_license_toolkit/checks.py | 214 +++++++++++++++--------------- 1 file changed, 105 insertions(+), 109 deletions(-) diff --git a/src/ros_license_toolkit/checks.py b/src/ros_license_toolkit/checks.py index d8ee48b..cfd0c74 100644 --- a/src/ros_license_toolkit/checks.py +++ b/src/ros_license_toolkit/checks.py @@ -147,29 +147,82 @@ def _check(self, package: Package): class LicenseTextExistsCheck(Check): """This ensures that the license text file referenced by the tag exists.""" + def __init__(self: 'LicenseTextExistsCheck'): + Check.__init__(self) + self.license_tags_without_license_text: Dict[LicenseTag, str] = {} + self.missing_license_texts_status: Dict[LicenseTag, Status] = {} + self.files_with_wrong_tags: Dict[LicenseTag, Dict[str, str]] = {} + self.found_license_texts: Dict[str, Any] = {} + def _check(self, package: Package): if len(package.license_tags) == 0: self._failed("No license tag defined.") return - license_tags_without_license_text: Dict[LicenseTag, str] = {} - missing_license_texts_status: Dict[LicenseTag, Status] = {} - files_with_wrong_tags: Dict[LicenseTag, Dict[str, str]] = {} - - found_license_texts = check_licenses( - package, - license_tags_without_license_text, - missing_license_texts_status, - files_with_wrong_tags) - - if len(license_tags_without_license_text) > 0: - if max(missing_license_texts_status.values()) == Status.WARNING: + + self._check_licenses(package) + self._evaluate_results(package) + + def _check_licenses(self, package: Package) -> None: + '''checks each license tag for the corresponding license text. Also + detects inofficial licenses when tag is other than SPDX license file''' + self.found_license_texts = package.found_license_texts + for license_tag in package.license_tags.values(): + if not license_tag.has_license_text_file(): + self.license_tags_without_license_text[ + license_tag] = "No license text file defined." + self.missing_license_texts_status[license_tag] = Status.FAILURE + continue + license_text_file = license_tag.get_license_text_file() + if not os.path.exists( + os.path.join(package.abspath, license_text_file)): + self.license_tags_without_license_text[license_tag] =\ + f"License text file '{license_text_file}' does not exist." + self.missing_license_texts_status[license_tag] = Status.FAILURE + continue + if license_text_file not in self.found_license_texts: + self.license_tags_without_license_text[license_tag] =\ + f"License text file '{license_text_file}' not included" +\ + " in scan results." + self.missing_license_texts_status[license_tag] = Status.FAILURE + continue + if not is_license_text_file( + self.found_license_texts[license_text_file]): + self.license_tags_without_license_text[license_tag] =\ + f"License text file '{license_text_file}' is not " +\ + "recognized as license text." + self.missing_license_texts_status[license_tag] = Status.FAILURE + continue + actual_license: Optional[str] = get_spdx_license_name( + self.found_license_texts[license_text_file]) + if actual_license is None: + self.license_tags_without_license_text[ + license_tag + ] = f"License text file '{license_text_file}'" +\ + " is not recognized as license text." + self.missing_license_texts_status[license_tag] = Status.FAILURE + continue + if actual_license != license_tag.get_license_id(): + self.license_tags_without_license_text[license_tag] =\ + f"License text file '{license_text_file}' is " +\ + f"of license {actual_license} but tag is " +\ + f"{license_tag.get_license_id()}." + self.missing_license_texts_status[license_tag] = Status.WARNING + self.files_with_wrong_tags[license_tag] = \ + {'actual_license': actual_license, + 'license_tag': license_tag.get_license_id()} + continue + + def _evaluate_results(self, package: Package): + if len(self.license_tags_without_license_text) > 0: + if max(self.missing_license_texts_status.values()) \ + == Status.WARNING: self._warning( "Since they are not in the SPDX list, " "we can not check if these tags have the correct " "license text:\n" + "\n".join( [f" '{x[0]}': {x[1]}" for x in - license_tags_without_license_text.items()])) - for entry in files_with_wrong_tags.items(): + self.license_tags_without_license_text.items()])) + for entry in self.files_with_wrong_tags.items(): # if exactly one license text is found, # treat wrong license tag internally as this license # optional check for similarity between tag and file @@ -181,82 +234,33 @@ def _check(self, package: Package): "have a valid license text " "file:\n" + "\n".join( [f" '{x[0]}': {x[1]}" for x in - license_tags_without_license_text.items()])) + self.license_tags_without_license_text.items()])) self.verbose_output = red( "\n".join([f" '{x[0]}': {x[1]}" for x in - found_license_texts.items()])) + self.found_license_texts.items()])) else: self._success("All license tags have a valid license text file.") -def check_licenses(package: Package, - tags_without_text: Dict[LicenseTag, str], - missing_texts_status: Dict[LicenseTag, Status], - files_with_wrong_tags: Dict[LicenseTag, Dict[str, str]] - ) -> Dict[str, Any]: - '''checks each license tag for the corresponding license text. Also - detects inofficial licenses (When tag is other than SPDX license file)''' - found_license_texts: Dict[str, Any] = package.found_license_texts - for license_tag in package.license_tags.values(): - if not license_tag.has_license_text_file(): - tags_without_text[ - license_tag] = "No license text file defined." - missing_texts_status[license_tag] = Status.FAILURE - continue - license_text_file = license_tag.get_license_text_file() - if not os.path.exists( - os.path.join(package.abspath, license_text_file)): - tags_without_text[license_tag] =\ - f"License text file '{license_text_file}' does not exist." - missing_texts_status[license_tag] = Status.FAILURE - continue - if license_text_file not in found_license_texts: - tags_without_text[license_tag] =\ - f"License text file '{license_text_file}' not included" +\ - " in scan results." - missing_texts_status[license_tag] = Status.FAILURE - continue - if not is_license_text_file( - found_license_texts[license_text_file]): - tags_without_text[license_tag] =\ - f"License text file '{license_text_file}' is not " +\ - "recognized as license text." - missing_texts_status[license_tag] = Status.FAILURE - continue - actual_license: Optional[str] = get_spdx_license_name( - found_license_texts[license_text_file]) - if actual_license is None: - tags_without_text[ - license_tag - ] = f"License text file '{license_text_file}'" +\ - " is not recognized as license text." - missing_texts_status[license_tag] = Status.FAILURE - continue - if actual_license != license_tag.get_license_id(): - tags_without_text[license_tag] =\ - f"License text file '{license_text_file}' is " +\ - f"of license {actual_license} but tag is " +\ - f"{license_tag.get_license_id()}." - missing_texts_status[license_tag] = Status.WARNING - files_with_wrong_tags[license_tag] = \ - {'actual_license': actual_license, - 'license_tag': license_tag.get_license_id()} - continue - return found_license_texts - - class LicensesInCodeCheck(Check): """Check if all found licenses have a declaration in the package.xml.""" + def __init__(self: 'LicensesInCodeCheck'): + Check.__init__(self) + self.declared_licenses: Dict[str, LicenseTag] = {} + self.files_with_uncovered_licenses: Dict[str, List[str]] = {} + self.files_not_matched_by_any_license_tag: Dict[str, List[str]] = {} + self.files_with_inofficial_tag: Dict[str, List[str]] = {} + def _check(self, package: Package): if len(package.license_tags) == 0: self._failed('No license tag defined.') return - declared_licenses = package.license_tags - files_with_uncovered_licenses: Dict[str, List[str]] = {} - files_not_matched_by_any_license_tag: Dict[str, List[str]] = {} - files_with_inofficial_tag: Dict[str, List[str]] = {} + self.declared_licenses = package.license_tags + self._check_license_files(package) + self._evaluate_result(package) + def _check_license_files(self, package: Package) -> None: for fname, found_licenses in package.found_files_w_licenses.items(): if fname in package.get_license_files(): # the actual license text files are not relevant for this @@ -265,72 +269,64 @@ def _check(self, package: Package): 'detected_license_expression_spdx'] licenses = found_licenses_str.split(' AND ') for license_str in licenses: - if license_str not in declared_licenses: + if license_str not in self.declared_licenses: # this license has an inofficial tag if license_str in package.inofficial_license_tag.keys(): - if fname not in files_with_inofficial_tag: - files_with_inofficial_tag[fname] = [] - files_with_inofficial_tag[fname].append(license_str) - files_with_inofficial_tag[fname].append( + if fname not in self.files_with_inofficial_tag: + self.files_with_inofficial_tag[fname] = [] + self.files_with_inofficial_tag[fname].append( + license_str) + self.files_with_inofficial_tag[fname].append( package.inofficial_license_tag[license_str]) continue # this license is not declared by any license tag - if fname not in files_with_uncovered_licenses: - files_with_uncovered_licenses[fname] = [] - files_with_uncovered_licenses[fname].append( + if fname not in self.files_with_uncovered_licenses: + self.files_with_uncovered_licenses[fname] = [] + self.files_with_uncovered_licenses[fname].append( license_str) continue - if fname not in declared_licenses[ + if fname not in self.declared_licenses[ license_str].source_files: # this license is declared by a license tag but the file # is not listed in the source files of the license tag - if fname not in files_not_matched_by_any_license_tag: - files_not_matched_by_any_license_tag[fname] = [] - files_not_matched_by_any_license_tag[fname].append( + if fname not in self.files_not_matched_by_any_license_tag: + self.files_not_matched_by_any_license_tag[fname] = [] + self.files_not_matched_by_any_license_tag[fname].append( license_str) continue - self._evaluate_result(package, - files_with_uncovered_licenses, - files_not_matched_by_any_license_tag, - files_with_inofficial_tag) - - def _evaluate_result(self, - package: Package, - files_with_uncovered_licenses: Dict[str, List[str]], - files_not_matched_by_any_tag: Dict[str, List[str]], - files_with_inofficial_tag: Dict[str, List[str]] - ) -> None: - if files_with_uncovered_licenses: + + def _evaluate_result(self, package: Package) -> None: + if self.files_with_uncovered_licenses: info_str = '' info_str += '\nThe following files contain licenses that ' +\ 'are not covered by any license tag:\n' + '\n'.join( [f" '{x[0]}': {x[1]}" for x in - files_with_uncovered_licenses.items()]) + self.files_with_uncovered_licenses.items()]) self._print_info( info_str, - files_with_uncovered_licenses, - files_not_matched_by_any_tag, + self.files_with_uncovered_licenses, + self.files_not_matched_by_any_license_tag, package, ) - elif files_with_inofficial_tag: + elif self.files_with_inofficial_tag: info_str = '' info_str += 'For the following files, please change the ' +\ 'License Tag in the package file to SPDX format:\n' +\ '\n'.join( [f" '{x[0]}' is of {x[1][0]} but its Tag is {x[1][1]}." - for x in files_with_inofficial_tag.items()]) + for x in self.files_with_inofficial_tag.items()]) self._warning(info_str) - elif len(files_not_matched_by_any_tag) > 0: + elif len(self.files_not_matched_by_any_license_tag) > 0: info_str = '' info_str += '\nThe following files contain licenses that ' +\ 'are covered by a license tag but are not listed in ' +\ 'the source files of the license tag:\n' + '\n'.join( [f" '{x[0]}': {x[1]}" for x in - files_not_matched_by_any_tag.items()]) + self.files_not_matched_by_any_license_tag.items()]) self._print_info( info_str, - files_with_uncovered_licenses, - files_not_matched_by_any_tag, + self.files_with_uncovered_licenses, + self.files_not_matched_by_any_license_tag, package, ) else: From 6c9ba21675e488ca0cc8e8f36c1cb8badcd44ccd Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Wed, 6 Mar 2024 14:30:32 +0100 Subject: [PATCH 25/36] added test for two licenses with both tags not in spdx Signed-off-by: Anton Utz --- .../test_pkg_both_tags_not_spdx/BSD.LICENSE | 11 +++ .../test_pkg_both_tags_not_spdx/MPL.LICENSE | 78 +++++++++++++++++++ .../code_with_MPL.py | 32 ++++++++ .../test_pkg_both_tags_not_spdx/package.xml | 7 ++ test/systemtest/test_all_packages.py | 11 ++- test/systemtest/test_separate_pkgs.py | 8 ++ 6 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 test/_test_data/test_pkg_both_tags_not_spdx/BSD.LICENSE create mode 100644 test/_test_data/test_pkg_both_tags_not_spdx/MPL.LICENSE create mode 100644 test/_test_data/test_pkg_both_tags_not_spdx/code_with_MPL.py create mode 100644 test/_test_data/test_pkg_both_tags_not_spdx/package.xml diff --git a/test/_test_data/test_pkg_both_tags_not_spdx/BSD.LICENSE b/test/_test_data/test_pkg_both_tags_not_spdx/BSD.LICENSE new file mode 100644 index 0000000..d97fba8 --- /dev/null +++ b/test/_test_data/test_pkg_both_tags_not_spdx/BSD.LICENSE @@ -0,0 +1,11 @@ +Copyright 1995 Foo Bar + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/test/_test_data/test_pkg_both_tags_not_spdx/MPL.LICENSE b/test/_test_data/test_pkg_both_tags_not_spdx/MPL.LICENSE new file mode 100644 index 0000000..d28b6c7 --- /dev/null +++ b/test/_test_data/test_pkg_both_tags_not_spdx/MPL.LICENSE @@ -0,0 +1,78 @@ + + +MOZILLA PUBLIC LICENSE +Version 1.0 + + 1. Definitions. + 1.1. "Contributor" means each entity that creates or contributes to the creation of Modifications. + 1.2. "Contributor Version" means the combination of the Original Code, prior Modifications used by a Contributor, and the Modifications made by that particular Contributor. + 1.3. "Covered Code" means the Original Code or Modifications or the combination of the Original Code and Modifications, in each case including portions thereof. + 1.4. "Electronic Distribution Mechanism" means a mechanism generally accepted in the software development community for the electronic transfer of data. + 1.5. "Executable" means Covered Code in any form other than Source Code. + 1.6. "Initial Developer" means the individual or entity identified as the Initial Developer in the Source Code notice required by Exhibit A. + 1.7. "Larger Work" means a work which combines Covered Code or portions thereof with code not governed by the terms of this License. + 1.8. "License" means this document. + 1.9. "Modifications" means any addition to or deletion from the substance or structure of either the Original Code or any previous Modifications. When Covered Code is released as a series of files, a Modification is: + A. Any addition to or deletion from the contents of a file containing Original Code or previous Modifications. + B. Any new file that contains any part of the Original Code or previous Modifications. + 1.10. "Original Code" means Source Code of computer software code which is described in the Source Code notice required by Exhibit A as Original Code, and which, at the time of its release under this License is not already Covered Code governed by this License. + 1.11. "Source Code" means the preferred form of the Covered Code for making modifications to it, including all modules it contains, plus any associated interface definition files, scripts used to control compilation and installation of an Executable, or a list of source code differential comparisons against either the Original Code or another well known, available Covered Code of the Contributor's choice. The Source Code can be in a compressed or archival form, provided the appropriate decompression or de-archiving software is widely available for no charge. + 1.12. "You" means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License or a future version of this License issued under Section 6.1. For legal entities, "You" includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of fifty percent (50%) or more of the outstanding shares or beneficial ownership of such entity. + 2. Source Code License. + 2.1. The Initial Developer Grant. + The Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license, subject to third party intellectual property claims: + (a) to use, reproduce, modify, display, perform, sublicense and distribute the Original Code (or portions thereof) with or without Modifications, or as part of a Larger Work; and + (b) under patents now or hereafter owned or controlled by Initial Developer, to make, have made, use and sell ("Utilize") the Original Code (or portions thereof), but solely to the extent that any such patent is reasonably necessary to enable You to Utilize the Original Code (or portions thereof) and not to any greater extent that may be necessary to Utilize further Modifications or combinations. + 2.2. Contributor Grant. + Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license, subject to third party intellectual property claims: + (a) to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof) either on an unmodified basis, with other Modifications, as Covered Code or as part of a Larger Work; and + (b) under patents now or hereafter owned or controlled by Contributor, to Utilize the Contributor Version (or portions thereof), but solely to the extent that any such patent is reasonably necessary to enable You to Utilize the Contributor Version (or portions thereof), and not to any greater extent that may be necessary to Utilize further Modifications or combinations. + 3. Distribution Obligations. + 3.1. Application of License. + The Modifications which You create or to which You contribute are governed by the terms of this License, including without limitation Section 2.2. The Source Code version of Covered Code may be distributed only under the terms of this License or a future version of this License released under Section 6.1, and You must include a copy of this License with every copy of the Source Code You distribute. You may not offer or impose any terms on any Source Code version that alters or restricts the applicable version of this License or the recipients' rights hereunder. However, You may include an additional document offering the additional rights described in Section 3.5. + 3.2. Availability of Source Code. + Any Modification which You create or to which You contribute must be made available in Source Code form under the terms of this License either on the same media as an Executable version or via an accepted Electronic Distribution Mechanism to anyone to whom you made an Executable version available; and if made available via Electronic Distribution Mechanism, must remain available for at least twelve (12) months after the date it initially became available, or at least six (6) months after a subsequent version of that particular Modification has been made available to such recipients. You are responsible for ensuring that the Source Code version remains available even if the Electronic Distribution Mechanism is maintained by a third party. + 3.3. Description of Modifications. + You must cause all Covered Code to which you contribute to contain a file documenting the changes You made to create that Covered Code and the date of any change. You must include a prominent statement that the Modification is derived, directly or indirectly, from Original Code provided by the Initial Developer and including the name of the Initial Developer in (a) the Source Code, and (b) in any notice in an Executable version or related documentation in which You describe the origin or ownership of the Covered Code. + 3.4. Intellectual Property Matters + (a) Third Party Claims. + If You have knowledge that a party claims an intellectual property right in particular functionality or code (or its utilization under this License), you must include a text file with the source code distribution titled "LEGAL" which describes the claim and the party making the claim in sufficient detail that a recipient will know whom to contact. If you obtain such knowledge after You make Your Modification available as described in Section 3.2, You shall promptly modify the LEGAL file in all copies You make available thereafter and shall take other steps (such as notifying appropriate mailing lists or newsgroups) reasonably calculated to inform those who received the Covered Code that new knowledge has been obtained. + (b) Contributor APIs. + If Your Modification is an application programming interface and You own or control patents which are reasonably necessary to implement that API, you must also include this information in the LEGAL file. + 3.5. Required Notices. + You must duplicate the notice in Exhibit A in each file of the Source Code, and this License in any documentation for the Source Code, where You describe recipients' rights relating to Covered Code. If You created one or more Modification(s), You may add your name as a Contributor to the notice described in Exhibit A. If it is not possible to put such notice in a particular Source Code file due to its structure, then you must include such notice in a location (such as a relevant directory file) where a user would be likely to look for such a notice. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Code. However, You may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear than any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. + 3.6. Distribution of Executable Versions. + You may distribute Covered Code in Executable form only if the requirements of Section 3.1-3.5 have been met for that Covered Code, and if You include a notice stating that the Source Code version of the Covered Code is available under the terms of this License, including a description of how and where You have fulfilled the obligations of Section 3.2. The notice must be conspicuously included in any notice in an Executable version, related documentation or collateral in which You describe recipients' rights relating to the Covered Code. You may distribute the Executable version of Covered Code under a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable version does not attempt to limit or alter the recipient's rights in the Source Code version from the rights set forth in this License. If You distribute the Executable version under a different license You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or any Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. + 3.7. Larger Works. + You may create a Larger Work by combining Covered Code with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Code. + 4. Inability to Comply Due to Statute or Regulation. + If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Code due to statute or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be included in the LEGAL file described in Section 3.4 and must be included with all distributions of the Source Code. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. + 5. Application of this License. + This License applies to code to which the Initial Developer has attached the notice in Exhibit A, and to related Covered Code. + 6. Versions of the License. + 6.1. New Versions. + Netscape Communications Corporation ("Netscape") may publish revised and/or new versions of the License from time to time. Each version will be given a distinguishing version number. + 6.2. Effect of New Versions. + Once Covered Code has been published under a particular version of the License, You may always continue to use it under the terms of that version. You may also choose to use such Covered Code under the terms of any subsequent version of the License published by Netscape. No one other than Netscape has the right to modify the terms applicable to Covered Code created under this License. + 6.3. Derivative Works. + If you create or use a modified version of this License (which you may only do in order to apply it to code which is not already Covered Code governed by this License), you must (a) rename Your license so that the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape", "NPL" or any confusingly similar phrase do not appear anywhere in your license and (b) otherwise make it clear that your version of the license contains terms which differ from the Mozilla Public License and Netscape Public License. (Filling in the name of the Initial Developer, Original Code or Contributor in the notice described in Exhibit A shall not of themselves be deemed to be modifications of this License.) + 7. DISCLAIMER OF WARRANTY. + COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + 8. TERMINATION. + This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. All sublicenses to the Covered Code which are properly granted shall survive any termination of this License. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. + 9. LIMITATION OF LIABILITY. + UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO YOU OR ANY OTHER PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THAT EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + 10. U.S. GOVERNMENT END USERS. + The Covered Code is a "commercial item," as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer software" and "commercial computer software documentation," as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Code with only those rights set forth herein. + 11. MISCELLANEOUS. + This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by California law provisions (except to the extent applicable law, if any, provides otherwise), excluding its conflict-of-law provisions. With respect to disputes in which at least one party is a citizen of, or an entity chartered or registered to do business in, the United States of America: (a) unless otherwise agreed in writing, all disputes relating to this License (excepting any dispute relating to intellectual property rights) shall be subject to final and binding arbitration, with the losing party paying all costs of arbitration; (b) any arbitration relating to this Agreement shall be held in Santa Clara County, California, under the auspices of JAMS/EndDispute; and (c) any litigation relating to this Agreement shall be subject to the jurisdiction of the Federal Courts of the Northern District of California, with venue lying in Santa Clara County, California, with the losing party responsible for costs, including without limitation, court costs and reasonable attorneys fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. + 12. RESPONSIBILITY FOR CLAIMS. + Except in cases where another Contributor has failed to comply with Section 3.4, You are responsible for damages arising, directly or indirectly, out of Your utilization of rights under this License, based on the number of copies of Covered Code you made available, the revenues you received from utilizing such rights, and other relevant factors. You agree to work with affected parties to distribute responsibility on an equitable basis. + +EXHIBIT A. + +"The contents of this file are subject to the Mozilla Public License Version 1.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.mozilla.org/MPL/ + +Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. + +The Original Code is _____. The Initial Developer of the Original Code is _____. Portions created by _____ are Copyright (C) _____. All Rights Reserved. Contributor(s): _____." diff --git a/test/_test_data/test_pkg_both_tags_not_spdx/code_with_MPL.py b/test/_test_data/test_pkg_both_tags_not_spdx/code_with_MPL.py new file mode 100644 index 0000000..881bb15 --- /dev/null +++ b/test/_test_data/test_pkg_both_tags_not_spdx/code_with_MPL.py @@ -0,0 +1,32 @@ +# Copyright (C) 2000 An Author + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: + +# Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +import sys + +if __name__ == "__main__": + print("hi") + sys.exit(0) diff --git a/test/_test_data/test_pkg_both_tags_not_spdx/package.xml b/test/_test_data/test_pkg_both_tags_not_spdx/package.xml new file mode 100644 index 0000000..f304be0 --- /dev/null +++ b/test/_test_data/test_pkg_both_tags_not_spdx/package.xml @@ -0,0 +1,7 @@ + + + + test_pkg_both_tags_not_spdx + BSD + MPL + diff --git a/test/systemtest/test_all_packages.py b/test/systemtest/test_all_packages.py index f12ea17..0f93ddb 100644 --- a/test/systemtest/test_all_packages.py +++ b/test/systemtest/test_all_packages.py @@ -40,20 +40,19 @@ def test_all(self): print(stdout) print(stderr) self.assertIn(b"test_pkg_deep", stdout) - self.assertIn( - b"test_pkg_has_code_disjoint", stdout) - self.assertIn( - b"test_pkg_has_code_of_different", stdout) + self.assertIn(b"test_pkg_both_tags_not_spdx", stdout) + self.assertIn(b"test_pkg_has_code_disjoint", stdout) + self.assertIn(b"test_pkg_has_code_of_different", stdout) + self.assertIn(b"test_pkg_has_code_of_different_license", stdout) self.assertIn( b"test_pkg_has_code_of_different_license_and_tag", stdout) self.assertIn( b"test_pkg_has_code_of_different_license_and_wrong_tag", stdout) - self.assertIn(b"test_pkg_has_code_of_different_license", stdout) self.assertIn(b"test_pkg_name_not_in_spdx", stdout) self.assertIn(b"test_pkg_no_file_attribute", stdout) + self.assertIn(b"test_pkg_no_license", stdout) self.assertIn(b"test_pkg_no_license_file", stdout) self.assertIn(b"test_pkg_one_correct_one_license_file_missing", stdout) - self.assertIn(b"test_pkg_no_license", stdout) self.assertIn(b"test_pkg_spdx_name", stdout) self.assertIn(b"test_pkg_spdx_tag", stdout) self.assertIn(b"test_pkg_unknown_license", stdout) diff --git a/test/systemtest/test_separate_pkgs.py b/test/systemtest/test_separate_pkgs.py index 2412160..22dc539 100644 --- a/test/systemtest/test_separate_pkgs.py +++ b/test/systemtest/test_separate_pkgs.py @@ -48,6 +48,14 @@ def test_deep_package_folder(self): b"test_pkg_deep", stdout) remove_repo(repo_path) + def test_pkg_both_tags_not_spdx(self): + """ Test on a package that has two different Licenses that both + have a not SPDX conform Tag. License files and source files + are SPDX conform""" + self.assertEqual(os.EX_OK, main([ + "test/_test_data/test_pkg_both_tags_not_spdx" + ])) + def test_pkg_has_code_disjoint(self): """Test on a package with two disjoint sets of source files under a license different from the package main license.""" From 5af585d59149f324d1afe87e6196b125e6e812c2 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Wed, 6 Mar 2024 15:00:03 +0100 Subject: [PATCH 26/36] changed license from test to MPL Signed-off-by: Anton Utz --- .../code_with_MPL.py | 33 +++++++------------ 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/test/_test_data/test_pkg_both_tags_not_spdx/code_with_MPL.py b/test/_test_data/test_pkg_both_tags_not_spdx/code_with_MPL.py index 881bb15..cecdf14 100644 --- a/test/_test_data/test_pkg_both_tags_not_spdx/code_with_MPL.py +++ b/test/_test_data/test_pkg_both_tags_not_spdx/code_with_MPL.py @@ -1,29 +1,18 @@ # Copyright (C) 2000 An Author -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: +# The contents of this file are subject to the Mozilla Public License +# Version 1.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the +# License at http://www.mozilla.org/MPL/ -# Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. +# Software distributed under the License is distributed on an +# "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express +# or implied. See the License for the specific language governing +# rights and limitations under the License. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF -# THE POSSIBILITY OF SUCH DAMAGE. +# The Original Code is _____. The Initial Developer of the +# Original Code is _____. Portions created by _____ are +# Copyright (C) _____. All Rights Reserved. Contributor(s): _____. import sys From 89f05d9b02a344d3ef9ed2e048ca41b2921d29b7 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Wed, 6 Mar 2024 15:01:04 +0100 Subject: [PATCH 27/36] Added negative test for two not in spdx tags but one for own license file Signed-off-by: Anton Utz --- .../LICENSE | 3 + .../MPL.LICENSE | 78 +++++++++++++++++++ .../more_code_with_MPL.py | 21 +++++ .../package.xml | 7 ++ test/systemtest/test_all_packages.py | 1 + test/systemtest/test_separate_pkgs.py | 7 ++ 6 files changed, 117 insertions(+) create mode 100644 test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/LICENSE create mode 100644 test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/MPL.LICENSE create mode 100644 test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/more_code_with_MPL.py create mode 100644 test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/package.xml diff --git a/test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/LICENSE b/test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/LICENSE new file mode 100644 index 0000000..a52934c --- /dev/null +++ b/test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/LICENSE @@ -0,0 +1,3 @@ +Copyright 2017 Foo Bar + +This license is self created and therefore not in SPDX standard. (c) 2017 - 2018 diff --git a/test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/MPL.LICENSE b/test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/MPL.LICENSE new file mode 100644 index 0000000..d28b6c7 --- /dev/null +++ b/test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/MPL.LICENSE @@ -0,0 +1,78 @@ + + +MOZILLA PUBLIC LICENSE +Version 1.0 + + 1. Definitions. + 1.1. "Contributor" means each entity that creates or contributes to the creation of Modifications. + 1.2. "Contributor Version" means the combination of the Original Code, prior Modifications used by a Contributor, and the Modifications made by that particular Contributor. + 1.3. "Covered Code" means the Original Code or Modifications or the combination of the Original Code and Modifications, in each case including portions thereof. + 1.4. "Electronic Distribution Mechanism" means a mechanism generally accepted in the software development community for the electronic transfer of data. + 1.5. "Executable" means Covered Code in any form other than Source Code. + 1.6. "Initial Developer" means the individual or entity identified as the Initial Developer in the Source Code notice required by Exhibit A. + 1.7. "Larger Work" means a work which combines Covered Code or portions thereof with code not governed by the terms of this License. + 1.8. "License" means this document. + 1.9. "Modifications" means any addition to or deletion from the substance or structure of either the Original Code or any previous Modifications. When Covered Code is released as a series of files, a Modification is: + A. Any addition to or deletion from the contents of a file containing Original Code or previous Modifications. + B. Any new file that contains any part of the Original Code or previous Modifications. + 1.10. "Original Code" means Source Code of computer software code which is described in the Source Code notice required by Exhibit A as Original Code, and which, at the time of its release under this License is not already Covered Code governed by this License. + 1.11. "Source Code" means the preferred form of the Covered Code for making modifications to it, including all modules it contains, plus any associated interface definition files, scripts used to control compilation and installation of an Executable, or a list of source code differential comparisons against either the Original Code or another well known, available Covered Code of the Contributor's choice. The Source Code can be in a compressed or archival form, provided the appropriate decompression or de-archiving software is widely available for no charge. + 1.12. "You" means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License or a future version of this License issued under Section 6.1. For legal entities, "You" includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of fifty percent (50%) or more of the outstanding shares or beneficial ownership of such entity. + 2. Source Code License. + 2.1. The Initial Developer Grant. + The Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license, subject to third party intellectual property claims: + (a) to use, reproduce, modify, display, perform, sublicense and distribute the Original Code (or portions thereof) with or without Modifications, or as part of a Larger Work; and + (b) under patents now or hereafter owned or controlled by Initial Developer, to make, have made, use and sell ("Utilize") the Original Code (or portions thereof), but solely to the extent that any such patent is reasonably necessary to enable You to Utilize the Original Code (or portions thereof) and not to any greater extent that may be necessary to Utilize further Modifications or combinations. + 2.2. Contributor Grant. + Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license, subject to third party intellectual property claims: + (a) to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof) either on an unmodified basis, with other Modifications, as Covered Code or as part of a Larger Work; and + (b) under patents now or hereafter owned or controlled by Contributor, to Utilize the Contributor Version (or portions thereof), but solely to the extent that any such patent is reasonably necessary to enable You to Utilize the Contributor Version (or portions thereof), and not to any greater extent that may be necessary to Utilize further Modifications or combinations. + 3. Distribution Obligations. + 3.1. Application of License. + The Modifications which You create or to which You contribute are governed by the terms of this License, including without limitation Section 2.2. The Source Code version of Covered Code may be distributed only under the terms of this License or a future version of this License released under Section 6.1, and You must include a copy of this License with every copy of the Source Code You distribute. You may not offer or impose any terms on any Source Code version that alters or restricts the applicable version of this License or the recipients' rights hereunder. However, You may include an additional document offering the additional rights described in Section 3.5. + 3.2. Availability of Source Code. + Any Modification which You create or to which You contribute must be made available in Source Code form under the terms of this License either on the same media as an Executable version or via an accepted Electronic Distribution Mechanism to anyone to whom you made an Executable version available; and if made available via Electronic Distribution Mechanism, must remain available for at least twelve (12) months after the date it initially became available, or at least six (6) months after a subsequent version of that particular Modification has been made available to such recipients. You are responsible for ensuring that the Source Code version remains available even if the Electronic Distribution Mechanism is maintained by a third party. + 3.3. Description of Modifications. + You must cause all Covered Code to which you contribute to contain a file documenting the changes You made to create that Covered Code and the date of any change. You must include a prominent statement that the Modification is derived, directly or indirectly, from Original Code provided by the Initial Developer and including the name of the Initial Developer in (a) the Source Code, and (b) in any notice in an Executable version or related documentation in which You describe the origin or ownership of the Covered Code. + 3.4. Intellectual Property Matters + (a) Third Party Claims. + If You have knowledge that a party claims an intellectual property right in particular functionality or code (or its utilization under this License), you must include a text file with the source code distribution titled "LEGAL" which describes the claim and the party making the claim in sufficient detail that a recipient will know whom to contact. If you obtain such knowledge after You make Your Modification available as described in Section 3.2, You shall promptly modify the LEGAL file in all copies You make available thereafter and shall take other steps (such as notifying appropriate mailing lists or newsgroups) reasonably calculated to inform those who received the Covered Code that new knowledge has been obtained. + (b) Contributor APIs. + If Your Modification is an application programming interface and You own or control patents which are reasonably necessary to implement that API, you must also include this information in the LEGAL file. + 3.5. Required Notices. + You must duplicate the notice in Exhibit A in each file of the Source Code, and this License in any documentation for the Source Code, where You describe recipients' rights relating to Covered Code. If You created one or more Modification(s), You may add your name as a Contributor to the notice described in Exhibit A. If it is not possible to put such notice in a particular Source Code file due to its structure, then you must include such notice in a location (such as a relevant directory file) where a user would be likely to look for such a notice. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Code. However, You may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear than any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. + 3.6. Distribution of Executable Versions. + You may distribute Covered Code in Executable form only if the requirements of Section 3.1-3.5 have been met for that Covered Code, and if You include a notice stating that the Source Code version of the Covered Code is available under the terms of this License, including a description of how and where You have fulfilled the obligations of Section 3.2. The notice must be conspicuously included in any notice in an Executable version, related documentation or collateral in which You describe recipients' rights relating to the Covered Code. You may distribute the Executable version of Covered Code under a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable version does not attempt to limit or alter the recipient's rights in the Source Code version from the rights set forth in this License. If You distribute the Executable version under a different license You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or any Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. + 3.7. Larger Works. + You may create a Larger Work by combining Covered Code with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Code. + 4. Inability to Comply Due to Statute or Regulation. + If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Code due to statute or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be included in the LEGAL file described in Section 3.4 and must be included with all distributions of the Source Code. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. + 5. Application of this License. + This License applies to code to which the Initial Developer has attached the notice in Exhibit A, and to related Covered Code. + 6. Versions of the License. + 6.1. New Versions. + Netscape Communications Corporation ("Netscape") may publish revised and/or new versions of the License from time to time. Each version will be given a distinguishing version number. + 6.2. Effect of New Versions. + Once Covered Code has been published under a particular version of the License, You may always continue to use it under the terms of that version. You may also choose to use such Covered Code under the terms of any subsequent version of the License published by Netscape. No one other than Netscape has the right to modify the terms applicable to Covered Code created under this License. + 6.3. Derivative Works. + If you create or use a modified version of this License (which you may only do in order to apply it to code which is not already Covered Code governed by this License), you must (a) rename Your license so that the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape", "NPL" or any confusingly similar phrase do not appear anywhere in your license and (b) otherwise make it clear that your version of the license contains terms which differ from the Mozilla Public License and Netscape Public License. (Filling in the name of the Initial Developer, Original Code or Contributor in the notice described in Exhibit A shall not of themselves be deemed to be modifications of this License.) + 7. DISCLAIMER OF WARRANTY. + COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + 8. TERMINATION. + This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. All sublicenses to the Covered Code which are properly granted shall survive any termination of this License. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. + 9. LIMITATION OF LIABILITY. + UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO YOU OR ANY OTHER PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THAT EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + 10. U.S. GOVERNMENT END USERS. + The Covered Code is a "commercial item," as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer software" and "commercial computer software documentation," as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Code with only those rights set forth herein. + 11. MISCELLANEOUS. + This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by California law provisions (except to the extent applicable law, if any, provides otherwise), excluding its conflict-of-law provisions. With respect to disputes in which at least one party is a citizen of, or an entity chartered or registered to do business in, the United States of America: (a) unless otherwise agreed in writing, all disputes relating to this License (excepting any dispute relating to intellectual property rights) shall be subject to final and binding arbitration, with the losing party paying all costs of arbitration; (b) any arbitration relating to this Agreement shall be held in Santa Clara County, California, under the auspices of JAMS/EndDispute; and (c) any litigation relating to this Agreement shall be subject to the jurisdiction of the Federal Courts of the Northern District of California, with venue lying in Santa Clara County, California, with the losing party responsible for costs, including without limitation, court costs and reasonable attorneys fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. + 12. RESPONSIBILITY FOR CLAIMS. + Except in cases where another Contributor has failed to comply with Section 3.4, You are responsible for damages arising, directly or indirectly, out of Your utilization of rights under this License, based on the number of copies of Covered Code you made available, the revenues you received from utilizing such rights, and other relevant factors. You agree to work with affected parties to distribute responsibility on an equitable basis. + +EXHIBIT A. + +"The contents of this file are subject to the Mozilla Public License Version 1.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.mozilla.org/MPL/ + +Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. + +The Original Code is _____. The Initial Developer of the Original Code is _____. Portions created by _____ are Copyright (C) _____. All Rights Reserved. Contributor(s): _____." diff --git a/test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/more_code_with_MPL.py b/test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/more_code_with_MPL.py new file mode 100644 index 0000000..cecdf14 --- /dev/null +++ b/test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/more_code_with_MPL.py @@ -0,0 +1,21 @@ +# Copyright (C) 2000 An Author + +# The contents of this file are subject to the Mozilla Public License +# Version 1.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the +# License at http://www.mozilla.org/MPL/ + +# Software distributed under the License is distributed on an +# "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express +# or implied. See the License for the specific language governing +# rights and limitations under the License. + +# The Original Code is _____. The Initial Developer of the +# Original Code is _____. Portions created by _____ are +# Copyright (C) _____. All Rights Reserved. Contributor(s): _____. + +import sys + +if __name__ == "__main__": + print("hi") + sys.exit(0) diff --git a/test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/package.xml b/test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/package.xml new file mode 100644 index 0000000..ee09880 --- /dev/null +++ b/test/_test_data/test_pkg_both_tags_not_spdx_one_file_own/package.xml @@ -0,0 +1,7 @@ + + + + test_pkg_both_tags_not_spdx_one_file_own + Self Created License + MPL + diff --git a/test/systemtest/test_all_packages.py b/test/systemtest/test_all_packages.py index 0f93ddb..13ccc80 100644 --- a/test/systemtest/test_all_packages.py +++ b/test/systemtest/test_all_packages.py @@ -41,6 +41,7 @@ def test_all(self): print(stderr) self.assertIn(b"test_pkg_deep", stdout) self.assertIn(b"test_pkg_both_tags_not_spdx", stdout) + self.assertIn(b"test_pkg_both_tags_not_spdx_one_file_own", stdout) self.assertIn(b"test_pkg_has_code_disjoint", stdout) self.assertIn(b"test_pkg_has_code_of_different", stdout) self.assertIn(b"test_pkg_has_code_of_different_license", stdout) diff --git a/test/systemtest/test_separate_pkgs.py b/test/systemtest/test_separate_pkgs.py index 22dc539..3209038 100644 --- a/test/systemtest/test_separate_pkgs.py +++ b/test/systemtest/test_separate_pkgs.py @@ -56,6 +56,13 @@ def test_pkg_both_tags_not_spdx(self): "test/_test_data/test_pkg_both_tags_not_spdx" ])) + def test_pkg_both_tags_not_spdx_one_file_own(self): + """Test on a package that has two licenses. One is self-defined, other + one with not SPDX tag but therefore code and license file in SPDX""" + self.assertEqual(os.EX_DATAERR, main([ + "test/_test_data/test_pkg_both_tags_not_spdx_one_file_own" + ])) + def test_pkg_has_code_disjoint(self): """Test on a package with two disjoint sets of source files under a license different from the package main license.""" From f51e03ba32743d20fbe67ac51d929ebba9d50337 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Mon, 11 Mar 2024 11:11:31 +0100 Subject: [PATCH 28/36] added test for ignoring contents of readme Signed-off-by: Anton Utz --- .../test_pkg_ignore_readme_contents/LICENSE | 60 +++++++++++++++++++ .../test_pkg_ignore_readme_contents/README.md | 8 +++ .../code_with_license.py | 19 ++++++ .../package.xml | 6 ++ 4 files changed, 93 insertions(+) create mode 100644 test/_test_data/test_pkg_ignore_readme_contents/LICENSE create mode 100644 test/_test_data/test_pkg_ignore_readme_contents/README.md create mode 100644 test/_test_data/test_pkg_ignore_readme_contents/code_with_license.py create mode 100644 test/_test_data/test_pkg_ignore_readme_contents/package.xml diff --git a/test/_test_data/test_pkg_ignore_readme_contents/LICENSE b/test/_test_data/test_pkg_ignore_readme_contents/LICENSE new file mode 100644 index 0000000..feacae9 --- /dev/null +++ b/test/_test_data/test_pkg_ignore_readme_contents/LICENSE @@ -0,0 +1,60 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: +(a) You must give any other recipients of the Work or Derivative Works a copy of this License; and +(b) You must cause any modified files to carry prominent notices stating that You changed the files; and +(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and +(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/test/_test_data/test_pkg_ignore_readme_contents/README.md b/test/_test_data/test_pkg_ignore_readme_contents/README.md new file mode 100644 index 0000000..ffc2b19 --- /dev/null +++ b/test/_test_data/test_pkg_ignore_readme_contents/README.md @@ -0,0 +1,8 @@ +This readme contains some random license names that shall NOT be checked. + +The majority of this library is licensed under the Apache-2.0 licensed. However, certain parts are +licensed under different licenses: + - The queue used inside the communication structures is originally written by Cameron Desrochers + and is released under the BSD-2-Clause license. + - The semaphore implementation used inside the queue implementation is written by Jeff Preshing and + licensed under the zlib license \ No newline at end of file diff --git a/test/_test_data/test_pkg_ignore_readme_contents/code_with_license.py b/test/_test_data/test_pkg_ignore_readme_contents/code_with_license.py new file mode 100644 index 0000000..6d61d8a --- /dev/null +++ b/test/_test_data/test_pkg_ignore_readme_contents/code_with_license.py @@ -0,0 +1,19 @@ +# Copyright 2020 The Author + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +if __name__ == "__main__": + print("hi") + sys.exit(0) diff --git a/test/_test_data/test_pkg_ignore_readme_contents/package.xml b/test/_test_data/test_pkg_ignore_readme_contents/package.xml new file mode 100644 index 0000000..4999d73 --- /dev/null +++ b/test/_test_data/test_pkg_ignore_readme_contents/package.xml @@ -0,0 +1,6 @@ + + + + test_pkg_ignore_readme_contents + Apache-2.0 + From 4590a295205a35fabb356d5e561f61e30b65ca17 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Mon, 11 Mar 2024 14:56:25 +0100 Subject: [PATCH 29/36] added ignore file in json format to set all files, types and folders for ignoring Signed-off-by: Anton Utz --- ignore_in_scan.json | 12 ++++++++++++ src/ros_license_toolkit/common.py | 13 ++++++++++++- src/ros_license_toolkit/package.py | 6 +++++- .../test_pkg_ignore_readme_contents/README.txt | 8 ++++++++ .../test_pkg_ignore_readme_contents/package.xml | 2 +- test/systemtest/test_all_packages.py | 1 + test/systemtest/test_separate_pkgs.py | 8 ++++++++ 7 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 ignore_in_scan.json create mode 100644 test/_test_data/test_pkg_ignore_readme_contents/README.txt diff --git a/ignore_in_scan.json b/ignore_in_scan.json new file mode 100644 index 0000000..cee62e8 --- /dev/null +++ b/ignore_in_scan.json @@ -0,0 +1,12 @@ +{ + "ignored_files": [ + "README.*", + "package.xml", + "setup.py", + "setup.cfg", + "CMakeLists.txt" + ], + "ignored_folders": [ + ".git/*" + ] +} \ No newline at end of file diff --git a/src/ros_license_toolkit/common.py b/src/ros_license_toolkit/common.py index 809a1a0..6f31797 100644 --- a/src/ros_license_toolkit/common.py +++ b/src/ros_license_toolkit/common.py @@ -16,7 +16,8 @@ """Common utility functions.""" -from typing import Any, Dict +import json +from typing import Any, Dict, List REQUIRED_PERCENTAGE_OF_LICENSE_TEXT = 95.0 @@ -26,3 +27,13 @@ def is_license_text_file(scan_results: Dict[str, Any]) -> bool: return ( scan_results["percentage_of_license_text"] >= REQUIRED_PERCENTAGE_OF_LICENSE_TEXT) + + +def get_ignored_content() -> List[str]: + """Return all ignored Files and Folders from 'ignore_in_scan.json'""" + ignored_content: List[str] = [] + with open("ignore_in_scan.json", 'r', encoding="utf-8") as f: + data = json.loads(f.read()) + ignored_content.extend(data['ignored_files']) + ignored_content.extend(data['ignored_folders']) + return ignored_content diff --git a/src/ros_license_toolkit/package.py b/src/ros_license_toolkit/package.py index 6ab22bc..3dd9c58 100644 --- a/src/ros_license_toolkit/package.py +++ b/src/ros_license_toolkit/package.py @@ -27,6 +27,7 @@ from scancode.api import get_licenses from ros_license_toolkit.common import (REQUIRED_PERCENTAGE_OF_LICENSE_TEXT, + get_ignored_content, is_license_text_file) from ros_license_toolkit.copyright import get_copyright_strings_per_pkg from ros_license_toolkit.license_tag import LicenseTag @@ -101,6 +102,9 @@ def __init__(self, path: str, repo: Optional[Repo] = None): # be found out through declaration, this field contains the tag self.inofficial_license_tag: Dict[str, str] = {} + # All ignored files and folders + self._ignored_content: List[str] = get_ignored_content() + def _get_path_relative_to_pkg(self, path: str) -> str: """Get path relative to pkg root""" return os.path.relpath(path, self.abspath) @@ -134,7 +138,7 @@ def _run_scan_and_save_results(self): for (root, _, files) in os.walk(self.abspath): files_rel_to_pkg = [self._get_path_relative_to_pkg( os.path.join(root, f)) for f in files] - for pattern in IGNORED: + for pattern in self._ignored_content: matched = fnmatch.filter(files_rel_to_pkg, pattern) for m in matched: files_rel_to_pkg.remove(m) diff --git a/test/_test_data/test_pkg_ignore_readme_contents/README.txt b/test/_test_data/test_pkg_ignore_readme_contents/README.txt new file mode 100644 index 0000000..9cad38d --- /dev/null +++ b/test/_test_data/test_pkg_ignore_readme_contents/README.txt @@ -0,0 +1,8 @@ +This readme contains some random license names that shall NOT be checked, here as txt. + +The majority of this library is licensed under the Apache-2.0 licensed. However, certain parts are +licensed under different licenses: + - The queue used inside the communication structures is originally written by Cameron Desrochers + and is released under the BSD-2-Clause license. + - The semaphore implementation used inside the queue implementation is written by Jeff Preshing and + licensed under the zlib license \ No newline at end of file diff --git a/test/_test_data/test_pkg_ignore_readme_contents/package.xml b/test/_test_data/test_pkg_ignore_readme_contents/package.xml index 4999d73..c03fd28 100644 --- a/test/_test_data/test_pkg_ignore_readme_contents/package.xml +++ b/test/_test_data/test_pkg_ignore_readme_contents/package.xml @@ -2,5 +2,5 @@ test_pkg_ignore_readme_contents - Apache-2.0 + Apache-2.0 diff --git a/test/systemtest/test_all_packages.py b/test/systemtest/test_all_packages.py index 13ccc80..9a68db7 100644 --- a/test/systemtest/test_all_packages.py +++ b/test/systemtest/test_all_packages.py @@ -49,6 +49,7 @@ def test_all(self): b"test_pkg_has_code_of_different_license_and_tag", stdout) self.assertIn( b"test_pkg_has_code_of_different_license_and_wrong_tag", stdout) + self.assertIn(b"test_pkg_ignore_readme_contents", stdout) self.assertIn(b"test_pkg_name_not_in_spdx", stdout) self.assertIn(b"test_pkg_no_file_attribute", stdout) self.assertIn(b"test_pkg_no_license", stdout) diff --git a/test/systemtest/test_separate_pkgs.py b/test/systemtest/test_separate_pkgs.py index 3209038..e584e0c 100644 --- a/test/systemtest/test_separate_pkgs.py +++ b/test/systemtest/test_separate_pkgs.py @@ -26,6 +26,8 @@ class TestPkgs(unittest.TestCase): """Test different test packages.""" + # pylint: disable=too-many-public-methods + # Here it make sense to keep all tests in one place def test_deep_package_folder(self): """Call the linter on directories in three levels. @@ -92,6 +94,12 @@ def test_pkg_has_code_of_different_license_and_wrong_tag(self): ["test/_test_data/" "test_pkg_has_code_of_different_license_and_wrong_tag"])) + def test_pkg_ignore_readme_contents(self): + """Test on a package with readme files. READMEs mention licenses + that are not in package and shall therefore be ignored.""" + self.assertEqual(os.EX_OK, main( + ["test/_test_data/test_pkg_ignore_readme_contents"])) + def test_pkg_name_not_in_spdx(self): """Test on a package that has valid License file with BSD-3-Clause but its license tag BSD is not in SPDX format""" From 07f813e2a77b66938d34dd7905faf58ec62e88d6 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Mon, 11 Mar 2024 15:55:38 +0100 Subject: [PATCH 30/36] added test for hidden folder that should also be ignored (simulating git folder). Changed format of ignore file Signed-off-by: Anton Utz --- README.md | 14 ++++++++++++++ ignore_in_scan.json | 6 ++---- src/ros_license_toolkit/common.py | 6 +++--- .../.hidden/LICENSE | 8 ++++++++ test/systemtest/test_separate_pkgs.py | 19 +++++++++++++++++-- 5 files changed, 44 insertions(+), 9 deletions(-) create mode 100644 test/_test_data/test_pkg_ignore_readme_contents/.hidden/LICENSE diff --git a/README.md b/README.md index ea5ea96..e5652fa 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,20 @@ options: -q, --quiet disable most output ``` +Additionally, there is an option to ignore single files, folders and types of files. +In the File `ignore_in_scan.json` all of these can be entered into the `ignoring` Field. + +```json +{ + "ignoring": [ + "README.md", // Ignoring every README.md file + "package.xml", + "CMakeLists.txt", + ".git/*" //ignoring everything underneath .git/ + ] +} +``` + ### Using it as a GitHub action You can use `ros_license_toolkit` inside your GitHub workflow in order to check licenses in your diff --git a/ignore_in_scan.json b/ignore_in_scan.json index cee62e8..8ec7ea3 100644 --- a/ignore_in_scan.json +++ b/ignore_in_scan.json @@ -1,12 +1,10 @@ { - "ignored_files": [ + "ignoring": [ "README.*", "package.xml", "setup.py", "setup.cfg", - "CMakeLists.txt" - ], - "ignored_folders": [ + "CMakeLists.txt", ".git/*" ] } \ No newline at end of file diff --git a/src/ros_license_toolkit/common.py b/src/ros_license_toolkit/common.py index 6f31797..54a96bf 100644 --- a/src/ros_license_toolkit/common.py +++ b/src/ros_license_toolkit/common.py @@ -30,10 +30,10 @@ def is_license_text_file(scan_results: Dict[str, Any]) -> bool: def get_ignored_content() -> List[str]: - """Return all ignored Files and Folders from 'ignore_in_scan.json'""" + """Return all ignored patterns from 'ignore_in_scan.json'""" ignored_content: List[str] = [] with open("ignore_in_scan.json", 'r', encoding="utf-8") as f: data = json.loads(f.read()) - ignored_content.extend(data['ignored_files']) - ignored_content.extend(data['ignored_folders']) + f.close() + ignored_content.extend(data['ignoring']) return ignored_content diff --git a/test/_test_data/test_pkg_ignore_readme_contents/.hidden/LICENSE b/test/_test_data/test_pkg_ignore_readme_contents/.hidden/LICENSE new file mode 100644 index 0000000..3c8ebee --- /dev/null +++ b/test/_test_data/test_pkg_ignore_readme_contents/.hidden/LICENSE @@ -0,0 +1,8 @@ +This is also a license mentioning text, but it should be ignored since its under the .git directory. + +The majority of this library is licensed under the Apache-2.0 licensed. However, certain parts are +licensed under different licenses: + - The queue used inside the communication structures is originally written by Cameron Desrochers + and is released under the BSD-2-Clause license. + - The semaphore implementation used inside the queue implementation is written by Jeff Preshing and + licensed under the zlib license \ No newline at end of file diff --git a/test/systemtest/test_separate_pkgs.py b/test/systemtest/test_separate_pkgs.py index e584e0c..3933231 100644 --- a/test/systemtest/test_separate_pkgs.py +++ b/test/systemtest/test_separate_pkgs.py @@ -16,6 +16,8 @@ """This module tests different test packages.""" +import copy +import json import os import subprocess import unittest @@ -97,8 +99,21 @@ def test_pkg_has_code_of_different_license_and_wrong_tag(self): def test_pkg_ignore_readme_contents(self): """Test on a package with readme files. READMEs mention licenses that are not in package and shall therefore be ignored.""" - self.assertEqual(os.EX_OK, main( - ["test/_test_data/test_pkg_ignore_readme_contents"])) + with open("ignore_in_scan.json", 'r', encoding="utf-8") as f: + file_content = f.read() + f.close() + org_data = json.loads(file_content) + test_data = copy.deepcopy(org_data) + test_data['ignoring'].append('.hidden/*') + with open("ignore_in_scan.json", 'w', encoding="utf-8") as f: + json.dump(test_data, f, indent=4) + f.close() + test_result = main(["test/_test_data/test_pkg_ignore_readme_contents"]) + with open("ignore_in_scan.json", 'w', encoding="utf-8") as f: + json.dump(org_data, f, indent=4) + f.close() + # Asserting at end so file still gets reset + self.assertEqual(os.EX_OK, test_result) def test_pkg_name_not_in_spdx(self): """Test on a package that has valid License file with BSD-3-Clause From 4ee05fe98d60cfd1a1c96af1914ca480c68d8450 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Mon, 11 Mar 2024 16:03:17 +0100 Subject: [PATCH 31/36] merged two lines Signed-off-by: Anton Utz --- test/systemtest/test_separate_pkgs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/systemtest/test_separate_pkgs.py b/test/systemtest/test_separate_pkgs.py index 3933231..bee333d 100644 --- a/test/systemtest/test_separate_pkgs.py +++ b/test/systemtest/test_separate_pkgs.py @@ -100,9 +100,8 @@ def test_pkg_ignore_readme_contents(self): """Test on a package with readme files. READMEs mention licenses that are not in package and shall therefore be ignored.""" with open("ignore_in_scan.json", 'r', encoding="utf-8") as f: - file_content = f.read() + org_data = json.loads(f.read()) f.close() - org_data = json.loads(file_content) test_data = copy.deepcopy(org_data) test_data['ignoring'].append('.hidden/*') with open("ignore_in_scan.json", 'w', encoding="utf-8") as f: From 065ff4614c8a5656346d0faba2e81fd6c1fd1650 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Tue, 12 Mar 2024 10:59:54 +0100 Subject: [PATCH 32/36] changed check for LicenceTag to Fail if tag and file are not same license but both are SPDX Signed-off-by: Anton Utz --- src/ros_license_toolkit/checks.py | 8 +++++++- test/systemtest/test_separate_pkgs.py | 7 +++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/ros_license_toolkit/checks.py b/src/ros_license_toolkit/checks.py index cfd0c74..6e1bd96 100644 --- a/src/ros_license_toolkit/checks.py +++ b/src/ros_license_toolkit/checks.py @@ -206,7 +206,13 @@ def _check_licenses(self, package: Package) -> None: f"License text file '{license_text_file}' is " +\ f"of license {actual_license} but tag is " +\ f"{license_tag.get_license_id()}." - self.missing_license_texts_status[license_tag] = Status.WARNING + # If Tag and File both are in SPDX but don't match -> Error + if is_license_name_in_spdx_list(license_tag.get_license_id()): + self.missing_license_texts_status[license_tag] =\ + Status.FAILURE + else: + self.missing_license_texts_status[license_tag] =\ + Status.WARNING self.files_with_wrong_tags[license_tag] = \ {'actual_license': actual_license, 'license_tag': license_tag.get_license_id()} diff --git a/test/systemtest/test_separate_pkgs.py b/test/systemtest/test_separate_pkgs.py index 3209038..b718375 100644 --- a/test/systemtest/test_separate_pkgs.py +++ b/test/systemtest/test_separate_pkgs.py @@ -178,10 +178,9 @@ def test_pkg_with_multiple_licenses_one_referenced_incorrect(self): def test_pkg_wrong_license_file(self): """Test on a package with a license text file that does not match - the license declared in the package.xml.""" - process, stdout = open_subprocess("test_pkg_wrong_license_file") - self.assertEqual(os.EX_OK, process.returncode) - self.assertIn(b"WARNING", stdout) + the license declared in the package.xml, both tag and file in spdx""" + self.assertEqual(os.EX_DATAERR, main( + ["test/_test_data/test_pkg_wrong_license_file"])) def open_subprocess(test_data_name: str): From 3a99d0e6764931891bf666dd4874e1a82a4809f7 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Mon, 18 Mar 2024 09:59:59 +0100 Subject: [PATCH 33/36] linter issues Signed-off-by: Anton Utz --- src/ros_license_toolkit/common.py | 1 + src/ros_license_toolkit/package.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ros_license_toolkit/common.py b/src/ros_license_toolkit/common.py index 2aada8e..826504a 100644 --- a/src/ros_license_toolkit/common.py +++ b/src/ros_license_toolkit/common.py @@ -23,6 +23,7 @@ REQUIRED_PERCENTAGE_OF_LICENSE_TEXT = 95.0 + def get_spdx_license_name(scan_results: Dict[str, Any]) -> Optional[str]: """Get the SPDX license name from scan results.""" if scan_results['percentage_of_license_text'] \ diff --git a/src/ros_license_toolkit/package.py b/src/ros_license_toolkit/package.py index 001a54a..54c7299 100644 --- a/src/ros_license_toolkit/package.py +++ b/src/ros_license_toolkit/package.py @@ -26,7 +26,8 @@ from rospkg.common import PACKAGE_FILE from scancode.api import get_licenses -from ros_license_toolkit.common import get_spdx_license_name, get_ignored_content +from ros_license_toolkit.common import (get_ignored_content, + get_spdx_license_name) from ros_license_toolkit.copyright import get_copyright_strings_per_pkg from ros_license_toolkit.license_tag import LicenseTag from ros_license_toolkit.repo import NotARepoError, Repo From cecde49598be38074d41377709ed080ec7c90b4d Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Mon, 18 Mar 2024 10:35:17 +0100 Subject: [PATCH 34/36] deleted unnecessary attribute from pull request Signed-off-by: Anton Utz --- src/ros_license_toolkit/package.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/ros_license_toolkit/package.py b/src/ros_license_toolkit/package.py index 43a46d6..9b2e972 100644 --- a/src/ros_license_toolkit/package.py +++ b/src/ros_license_toolkit/package.py @@ -88,10 +88,6 @@ def __init__(self, path: str, repo: Optional[Repo] = None): # this is Optional, because it is only evaluated on the first call self._license_tags: Optional[Dict[str, LicenseTag]] = None - # If the tag is wrong (like BSD) but the actual license can - # be found out through declaration, this field contains the tag - self.inofficial_license_tag: Dict[str, str] = {} - def _get_path_relative_to_pkg(self, path: str) -> str: """Get path relative to pkg root""" return os.path.relpath(path, self.abspath) From 371d631de62eae4e2de9687eeaf69eeefe09718f Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Mon, 18 Mar 2024 14:39:47 +0100 Subject: [PATCH 35/36] introduced .scanignore file, keeping local var for ignoring default files. Signed-off-by: Anton Utz --- README.md | 19 +++++------ ignore_in_scan.json | 10 ------ src/ros_license_toolkit/common.py | 33 ++++++++++++++----- src/ros_license_toolkit/package.py | 11 +------ .../.hidden/LICENSE | 2 +- .../.scanignore | 4 +++ test/systemtest/test_separate_pkgs.py | 14 -------- 7 files changed, 39 insertions(+), 54 deletions(-) delete mode 100644 ignore_in_scan.json create mode 100644 test/_test_data/test_pkg_ignore_readme_contents/.scanignore diff --git a/README.md b/README.md index e5652fa..766274d 100644 --- a/README.md +++ b/README.md @@ -87,17 +87,14 @@ options: ``` Additionally, there is an option to ignore single files, folders and types of files. -In the File `ignore_in_scan.json` all of these can be entered into the `ignoring` Field. - -```json -{ - "ignoring": [ - "README.md", // Ignoring every README.md file - "package.xml", - "CMakeLists.txt", - ".git/*" //ignoring everything underneath .git/ - ] -} +If there exists a `.scanignore` in the **top level directory** of a package, +everything in it is going to be ignored. +The file entries work similar to a `.gitignore` file, including making comments with `#`. + +``` +.git/* # folder +README.txt # file +README.* # file pattern ``` ### Using it as a GitHub action diff --git a/ignore_in_scan.json b/ignore_in_scan.json deleted file mode 100644 index 8ec7ea3..0000000 --- a/ignore_in_scan.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "ignoring": [ - "README.*", - "package.xml", - "setup.py", - "setup.cfg", - "CMakeLists.txt", - ".git/*" - ] -} \ No newline at end of file diff --git a/src/ros_license_toolkit/common.py b/src/ros_license_toolkit/common.py index 826504a..988397e 100644 --- a/src/ros_license_toolkit/common.py +++ b/src/ros_license_toolkit/common.py @@ -16,12 +16,20 @@ """Common utility functions.""" -import json +import os from typing import Any, Dict, List, Optional REQUIRED_PERCENTAGE_OF_LICENSE_TEXT = 95.0 -REQUIRED_PERCENTAGE_OF_LICENSE_TEXT = 95.0 +# files we ignore in scan results +IGNORED = [ + ".scanignore", + "package.xml", + "setup.py", + "setup.cfg", + "CMakeLists.txt", + ".git/*", +] def get_spdx_license_name(scan_results: Dict[str, Any]) -> Optional[str]: @@ -32,11 +40,20 @@ def get_spdx_license_name(scan_results: Dict[str, Any]) -> Optional[str]: return None -def get_ignored_content() -> List[str]: - """Return all ignored patterns from 'ignore_in_scan.json'""" +def get_ignored_content(pkg_abspath: str) -> List[str]: + """Return all ignored patterns from '.scanignore' + and local IGNORED definition.""" ignored_content: List[str] = [] - with open("ignore_in_scan.json", 'r', encoding="utf-8") as f: - data = json.loads(f.read()) - f.close() - ignored_content.extend(data['ignoring']) + scanignore_path = pkg_abspath + "/.scanignore" + if os.path.exists(scanignore_path): + with open(scanignore_path, 'r', encoding="utf-8") as f: + for line in f: + line_contents = line.split('#') + ignore_pattern = line_contents[0].rstrip() + if len(ignore_pattern) > 0: + ignored_content.append(ignore_pattern) + f.close() + for pattern in IGNORED: + if pattern not in ignored_content: + ignored_content.append(pattern) return ignored_content diff --git a/src/ros_license_toolkit/package.py b/src/ros_license_toolkit/package.py index 54c7299..32be84a 100644 --- a/src/ros_license_toolkit/package.py +++ b/src/ros_license_toolkit/package.py @@ -32,15 +32,6 @@ from ros_license_toolkit.license_tag import LicenseTag from ros_license_toolkit.repo import NotARepoError, Repo -# files we ignore in scan results -IGNORED = [ - "package.xml", - "setup.py", - "setup.cfg", - "CMakeLists.txt", - ".git/*", -] - class PackageException(Exception): """Exception raised when a package is invalid.""" @@ -94,7 +85,7 @@ def __init__(self, path: str, repo: Optional[Repo] = None): self.inofficial_license_tag: Dict[str, str] = {} # All ignored files and folders - self._ignored_content: List[str] = get_ignored_content() + self._ignored_content: List[str] = get_ignored_content(self.abspath) def _get_path_relative_to_pkg(self, path: str) -> str: """Get path relative to pkg root""" diff --git a/test/_test_data/test_pkg_ignore_readme_contents/.hidden/LICENSE b/test/_test_data/test_pkg_ignore_readme_contents/.hidden/LICENSE index 3c8ebee..88a36a8 100644 --- a/test/_test_data/test_pkg_ignore_readme_contents/.hidden/LICENSE +++ b/test/_test_data/test_pkg_ignore_readme_contents/.hidden/LICENSE @@ -1,4 +1,4 @@ -This is also a license mentioning text, but it should be ignored since its under the .git directory. +This is also a license mentioning text, but it should be ignored since its under the .hidden directory. The majority of this library is licensed under the Apache-2.0 licensed. However, certain parts are licensed under different licenses: diff --git a/test/_test_data/test_pkg_ignore_readme_contents/.scanignore b/test/_test_data/test_pkg_ignore_readme_contents/.scanignore new file mode 100644 index 0000000..bd7ea7b --- /dev/null +++ b/test/_test_data/test_pkg_ignore_readme_contents/.scanignore @@ -0,0 +1,4 @@ +.hidden/* +README.md + +README.* # comment in any kind \ No newline at end of file diff --git a/test/systemtest/test_separate_pkgs.py b/test/systemtest/test_separate_pkgs.py index 1da5731..d74bb84 100644 --- a/test/systemtest/test_separate_pkgs.py +++ b/test/systemtest/test_separate_pkgs.py @@ -16,8 +16,6 @@ """This module tests different test packages.""" -import copy -import json import os import subprocess import unittest @@ -99,19 +97,7 @@ def test_pkg_has_code_of_different_license_and_wrong_tag(self): def test_pkg_ignore_readme_contents(self): """Test on a package with readme files. READMEs mention licenses that are not in package and shall therefore be ignored.""" - with open("ignore_in_scan.json", 'r', encoding="utf-8") as f: - org_data = json.loads(f.read()) - f.close() - test_data = copy.deepcopy(org_data) - test_data['ignoring'].append('.hidden/*') - with open("ignore_in_scan.json", 'w', encoding="utf-8") as f: - json.dump(test_data, f, indent=4) - f.close() test_result = main(["test/_test_data/test_pkg_ignore_readme_contents"]) - with open("ignore_in_scan.json", 'w', encoding="utf-8") as f: - json.dump(org_data, f, indent=4) - f.close() - # Asserting at end so file still gets reset self.assertEqual(os.EX_OK, test_result) def test_pkg_name_not_in_spdx(self): From 0a9573b6a1c67541656911dcb662d334f19a37c2 Mon Sep 17 00:00:00 2001 From: Anton Utz Date: Mon, 18 Mar 2024 14:40:32 +0100 Subject: [PATCH 36/36] Added list of default ignored items to readme Signed-off-by: Anton Utz --- README.md | 12 ++++++++++++ src/ros_license_toolkit/common.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 766274d..f0f413e 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,7 @@ Additionally, there is an option to ignore single files, folders and types of fi If there exists a `.scanignore` in the **top level directory** of a package, everything in it is going to be ignored. The file entries work similar to a `.gitignore` file, including making comments with `#`. +One Example for a custom `.scanignore` file: ``` .git/* # folder @@ -97,6 +98,17 @@ README.txt # file README.* # file pattern ``` +Per default, ros_license_toolkit ignores the following: + +``` +.scanignore +package.xml +setup.py +setup.cfg +CMakeLists.txt +.git/* +``` + ### Using it as a GitHub action You can use `ros_license_toolkit` inside your GitHub workflow in order to check licenses in your diff --git a/src/ros_license_toolkit/common.py b/src/ros_license_toolkit/common.py index 988397e..8784a98 100644 --- a/src/ros_license_toolkit/common.py +++ b/src/ros_license_toolkit/common.py @@ -28,7 +28,7 @@ "setup.py", "setup.cfg", "CMakeLists.txt", - ".git/*", + ".git/*" ]