diff --git a/how_to_update.md b/how_to_update.md index 41ec1ac..60c8752 100644 --- a/how_to_update.md +++ b/how_to_update.md @@ -1,5 +1,13 @@ # Note for me +## Requirements + +```bash +pip install bumpver build twine +``` + +## Steps + How to update the project: 1. Increment version diff --git a/pyproject.toml b/pyproject.toml index 6c0b097..bec0c66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "ros_license_toolkit" -version = "1.2.2" +version = "1.3.0" description = "Checks ROS packages for correct license declaration." readme = "README.md" authors = [ diff --git a/setup.cfg b/setup.cfg index 75cc4e0..2d5d77f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,9 @@ [flake8] -ignore = Q000, I100, I201, W503, W504 +ignore = Q000, I100, I201, W503 [bumpver] -current_version = "1.2.2" +current_version = "1.3.0" version_pattern = "MAJOR.MINOR.PATCH" commit = True tag = True diff --git a/src/ros_license_toolkit/__init__.py b/src/ros_license_toolkit/__init__.py index 4a59556..9dfaa42 100644 --- a/src/ros_license_toolkit/__init__.py +++ b/src/ros_license_toolkit/__init__.py @@ -1,2 +1,2 @@ """ROS License Toolkit.""" -__version__ = "1.2.2" +__version__ = "1.3.0" diff --git a/src/ros_license_toolkit/checks.py b/src/ros_license_toolkit/checks.py index 6e1bd96..8fee565 100644 --- a/src/ros_license_toolkit/checks.py +++ b/src/ros_license_toolkit/checks.py @@ -21,11 +21,10 @@ from pprint import pformat from typing import Any, Dict, List, Optional +from ros_license_toolkit.common import get_spdx_license_name from ros_license_toolkit.license_tag import (LicenseTag, is_license_name_in_spdx_list) -from ros_license_toolkit.package import (Package, PackageException, - get_spdx_license_name, - is_license_text_file) +from ros_license_toolkit.package import Package, PackageException from ros_license_toolkit.ui_elements import NO_REASON_STR, green, red, yellow @@ -160,11 +159,11 @@ def _check(self, package: Package): return self._check_licenses(package) - self._evaluate_results(package) + self._evaluate_results() def _check_licenses(self, package: Package) -> None: '''checks each license tag for the corresponding license text. Also - detects inofficial licenses when tag is other than SPDX license file''' + detects inofficial licenses when tag is not in the SPDX license list''' self.found_license_texts = package.found_license_texts for license_tag in package.license_tags.values(): if not license_tag.has_license_text_file(): @@ -185,7 +184,7 @@ def _check_licenses(self, package: Package) -> None: " in scan results." self.missing_license_texts_status[license_tag] = Status.FAILURE continue - if not is_license_text_file( + if not get_spdx_license_name( self.found_license_texts[license_text_file]): self.license_tags_without_license_text[license_tag] =\ f"License text file '{license_text_file}' is not " +\ @@ -218,7 +217,7 @@ def _check_licenses(self, package: Package) -> None: 'license_tag': license_tag.get_license_id()} continue - def _evaluate_results(self, package: Package): + def _evaluate_results(self): if len(self.license_tags_without_license_text) > 0: if max(self.missing_license_texts_status.values()) \ == Status.WARNING: @@ -228,12 +227,6 @@ def _evaluate_results(self, package: Package): "license text:\n" + "\n".join( [f" '{x[0]}': {x[1]}" for x in self.license_tags_without_license_text.items()])) - for entry in self.files_with_wrong_tags.items(): - # if exactly one license text is found, - # treat wrong license tag internally as this license - # optional check for similarity between tag and file - package.inofficial_license_tag[entry[1]['actual_license']]\ - = entry[1]['license_tag'] else: self._failed( "The following license tags do not " @@ -277,13 +270,17 @@ def _check_license_files(self, package: Package) -> None: for license_str in licenses: if license_str not in self.declared_licenses: # this license has an inofficial tag - if license_str in package.inofficial_license_tag.keys(): + inofficial_licenses = { + lic_tag.id_from_license_text: key + for key, lic_tag in package.license_tags.items() + if lic_tag.id_from_license_text != ''} + if license_str in inofficial_licenses.keys(): if fname not in self.files_with_inofficial_tag: self.files_with_inofficial_tag[fname] = [] self.files_with_inofficial_tag[fname].append( license_str) self.files_with_inofficial_tag[fname].append( - package.inofficial_license_tag[license_str]) + inofficial_licenses[license_str]) continue # this license is not declared by any license tag if fname not in self.files_with_uncovered_licenses: diff --git a/src/ros_license_toolkit/common.py b/src/ros_license_toolkit/common.py index 809a1a0..3f4e1e7 100644 --- a/src/ros_license_toolkit/common.py +++ b/src/ros_license_toolkit/common.py @@ -16,13 +16,14 @@ """Common utility functions.""" -from typing import Any, Dict +from typing import Any, Dict, Optional REQUIRED_PERCENTAGE_OF_LICENSE_TEXT = 95.0 -def is_license_text_file(scan_results: Dict[str, Any]) -> bool: - """Check if a file is a license text file.""" - return ( - scan_results["percentage_of_license_text"] >= - REQUIRED_PERCENTAGE_OF_LICENSE_TEXT) +def get_spdx_license_name(scan_results: Dict[str, Any]) -> Optional[str]: + """Get the SPDX license name from scan results.""" + if scan_results['percentage_of_license_text'] \ + >= REQUIRED_PERCENTAGE_OF_LICENSE_TEXT: + return scan_results['detected_license_expression_spdx'] + return None diff --git a/src/ros_license_toolkit/license_tag.py b/src/ros_license_toolkit/license_tag.py index 467b0bd..cdfdb94 100644 --- a/src/ros_license_toolkit/license_tag.py +++ b/src/ros_license_toolkit/license_tag.py @@ -22,7 +22,7 @@ import os import xml.etree.ElementTree as ET from glob import glob -from typing import List, Optional, Set +from typing import Any, Dict, List, Optional, Set from spdx.config import LICENSE_MAP @@ -60,13 +60,20 @@ def _eval_glob(glob_str: str, pkg_path: str) -> Set[str]: class LicenseTag: """A license tag found in a package.xml file.""" - def __init__(self, element: ET.Element, pkg_path: str): + def __init__(self, element: ET.Element, + pkg_path: str, + license_file_scan_results: Optional[Dict[str, Any]] = None): """Initialize a license tag from an XML element.""" self.element = element assert self.element.text is not None, "License tag must have text." - raw_license_name: str = str(self.element.text) # Name of the license (in SPDX tag format for comparability) + raw_license_name: str = str(self.element.text) + + # If the tag is wrong (like BSD) but the actual license can + # be found out through declaration, this field contains the tag + self.id_from_license_text: Optional[str] = None + try: self.id = to_spdx_license_tag(raw_license_name) except ValueError: @@ -74,6 +81,10 @@ def __init__(self, element: ET.Element, pkg_path: str): # we assume it is a custom license and use the name as-is. # This will be detected in `LicenseTagIsInSpdxListCheck`. self.id = raw_license_name + # If a file is linked to the tag, set its id for internal checks + if license_file_scan_results: + self.id_from_license_text = \ + get_id_from_license_text(license_file_scan_results) # Path to the file containing the license text # (relative to package root) @@ -139,3 +150,10 @@ def make_this_the_main_license(self, other_licenses: List["LicenseTag"]): continue source_files -= other_license.source_files self._source_files = source_files + + +def get_id_from_license_text(license_file_scan_result: Dict[str, Any]) -> str: + """Return the detected license id from the license declaration""" + if 'detected_license_expression_spdx' in license_file_scan_result: + return license_file_scan_result['detected_license_expression_spdx'] + return '' diff --git a/src/ros_license_toolkit/package.py b/src/ros_license_toolkit/package.py index 6ab22bc..43a46d6 100644 --- a/src/ros_license_toolkit/package.py +++ b/src/ros_license_toolkit/package.py @@ -26,8 +26,7 @@ from rospkg.common import PACKAGE_FILE from scancode.api import get_licenses -from ros_license_toolkit.common import (REQUIRED_PERCENTAGE_OF_LICENSE_TEXT, - is_license_text_file) +from ros_license_toolkit.common import get_spdx_license_name from ros_license_toolkit.copyright import get_copyright_strings_per_pkg from ros_license_toolkit.license_tag import LicenseTag from ros_license_toolkit.repo import NotARepoError, Repo @@ -42,14 +41,6 @@ ] -def get_spdx_license_name(scan_results: Dict[str, Any]) -> Optional[str]: - """Get the SPDX license name from scan results.""" - if scan_results['percentage_of_license_text'] >=\ - REQUIRED_PERCENTAGE_OF_LICENSE_TEXT: - return scan_results['detected_license_expression_spdx'] - return None - - class PackageException(Exception): """Exception raised when a package is invalid.""" @@ -143,7 +134,7 @@ def _run_scan_and_save_results(self): fpath = os.path.join(self.abspath, fname) # Path relative to package root scan_results = get_licenses(fpath) - if is_license_text_file(scan_results): + if get_spdx_license_name(scan_results): self._found_license_texts[fname ] = scan_results else: @@ -210,6 +201,20 @@ def _check_single_license_tag_without_file_attribute(self): tag.license_text_file = potential_license_files[0] break + def _check_for_single_tag_without_file(self): + """Set the id_from_license_text if only one tag and one + declaration exist.""" + if len(self._license_tags) == 1 and len(self.found_license_texts) == 1: + license_tag_key = next(iter(self._license_tags.keys())) + id_from_text = self._license_tags[ + license_tag_key].id_from_license_text + if id_from_text is None: + only_file_id = self.found_license_texts[ + next(iter(self.found_license_texts))][ + 'detected_license_expression_spdx'] + self._license_tags[license_tag_key].id_from_license_text = \ + only_file_id + @property def license_tags(self) -> Dict[str, LicenseTag]: """Get all license tags in the package.xml file.""" @@ -217,11 +222,20 @@ def license_tags(self) -> Dict[str, LicenseTag]: return self._license_tags self._license_tags = {} for license_tag in self.package_xml.iterfind('license'): - tag = LicenseTag(license_tag, self.abspath) + license_file_scan_result = None + if 'file' in license_tag.attrib: + license_file = license_tag.attrib['file'] + if license_file in self.found_license_texts: + license_file_scan_result = \ + self.found_license_texts[license_file] + license_file_scan_result['filename'] = license_file + tag = LicenseTag(license_tag, + self.abspath, license_file_scan_result) self._license_tags[tag.get_license_id()] = tag self._check_single_license_tag_without_source_files() self._check_single_license_tag_without_file_attribute() + self._check_for_single_tag_without_file() return self._license_tags diff --git a/src/ros_license_toolkit/repo.py b/src/ros_license_toolkit/repo.py index 1f2b34b..03e25b2 100644 --- a/src/ros_license_toolkit/repo.py +++ b/src/ros_license_toolkit/repo.py @@ -24,7 +24,7 @@ import git from scancode.api import get_licenses -from ros_license_toolkit.common import is_license_text_file +from ros_license_toolkit.common import get_spdx_license_name # how many folders up to search for a repo REPO_SEARCH_DEPTH = 5 @@ -80,13 +80,16 @@ def __init__(self, package_path: str): if not os.path.isfile(fpath): continue scan_results = get_licenses(fpath) - if is_license_text_file(scan_results): - self.license_text_files[fpath] = scan_results + if get_spdx_license_name(scan_results): + if 'ros_license_toolkit/LICENSE' not in fpath: + self.license_text_files[fpath] = scan_results # get the remote url self.remote_url: Optional[str] = None if len(repo.remotes) > 0: - self.remote_url = repo.remotes[0].url + # Ignore package + if 'ros_license_toolkit' not in repo.remotes[0].url: + self.remote_url = repo.remotes[0].url def __eq__(self, __o) -> bool: """Check if two repos are the same.""" diff --git a/test/_test_data/test_pkg_tag_not_spdx/LICENSE b/test/_test_data/test_pkg_tag_not_spdx/LICENSE new file mode 100644 index 0000000..008ba30 --- /dev/null +++ b/test/_test_data/test_pkg_tag_not_spdx/LICENSE @@ -0,0 +1,46 @@ +The Academic Free License + v. 2.0 + +This Academic Free License (the "License") applies to any original work of authorship (the "Original Work") whose owner (the "Licensor") has placed the following notice immediately following the copyright notice for the Original Work: +Licensed under the Academic Free License version 2.0 + +1) Grant of Copyright License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license to do the following: +a) to reproduce the Original Work in copies; + +b) to prepare derivative works ("Derivative Works") based upon the Original Work; + +c) to distribute copies of the Original Work and Derivative Works to the public; + +d) to perform the Original Work publicly; and + +e) to display the Original Work publicly. + +2) Grant of Patent License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license, under patent claims owned or controlled by the Licensor that are embodied in the Original Work as furnished by the Licensor, to make, use, sell and offer for sale the Original Work and Derivative Works. + +3) Grant of Source Code License. The term "Source Code" means the preferred form of the Original Work for making modifications to it and all available documentation describing how to modify the Original Work. Licensor hereby agrees to provide a machine-readable copy of the Source Code of the Original Work along with each copy of the Original Work that Licensor distributes. Licensor reserves the right to satisfy this obligation by placing a machine-readable copy of the Source Code in an information repository reasonably calculated to permit inexpensive and convenient access by You for as long as Licensor continues to distribute the Original Work, and by publishing the address of that information repository in a notice immediately following the copyright notice that applies to the Original Work. + +4) Exclusions From License Grant. Neither the names of Licensor, nor the names of any contributors to the Original Work, nor any of their trademarks or service marks, may be used to endorse or promote products derived from this Original Work without express prior written permission of the Licensor. Nothing in this License shall be deemed to grant any rights to trademarks, copyrights, patents, trade secrets or any other intellectual property of Licensor except as expressly stated herein. No patent license is granted to make, use, sell or offer to sell embodiments of any patent claims other than the licensed claims defined in Section 2. No right is granted to the trademarks of Licensor even if such marks are included in the Original Work. Nothing in this License shall be interpreted to prohibit Licensor from licensing under different terms from this License any Original Work that Licensor otherwise would have a right to license. + +5) This section intentionally omitted. + +6) Attribution Rights. You must retain, in the Source Code of any Derivative Works that You create, all copyright, patent or trademark notices from the Source Code of the Original Work, as well as any notices of licensing and any descriptive text identified therein as an "Attribution Notice." You must cause the Source Code for any Derivative Works that You create to carry a prominent Attribution Notice reasonably calculated to inform recipients that You have modified the Original Work. + +7) Warranty of Provenance and Disclaimer of Warranty. Licensor warrants that the copyright in and to the Original Work and the patent rights granted herein by Licensor are owned by the Licensor or are sublicensed to You under the terms of this License with the permission of the contributor(s) of those copyrights and patent rights. Except as expressly stated in the immediately proceeding sentence, the Original Work is provided under this License on an "AS IS" BASIS and WITHOUT WARRANTY, either express or implied, including, without limitation, the warranties of NON-INFRINGEMENT, MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK IS WITH YOU. This DISCLAIMER OF WARRANTY constitutes an essential part of this License. No license to Original Work is granted hereunder except under this disclaimer. + +8) Limitation of Liability. Under no circumstances and under no legal theory, whether in tort (including negligence), contract, or otherwise, shall the Licensor be liable to any person for any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or the use of the Original Work including, without limitation, damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses. This limitation of liability shall not apply to liability for death or personal injury resulting from Licensor's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. + +9) Acceptance and Termination. If You distribute copies of the Original Work or a Derivative Work, You must make a reasonable effort under the circumstances to obtain the express assent of recipients to the terms of this License. Nothing else but this License (or another written agreement between Licensor and You) grants You permission to create Derivative Works based upon the Original Work or to exercise any of the rights granted in Section 1 herein, and any attempt to do so except under the terms of this License (or another written agreement between Licensor and You) is expressly prohibited by U.S. copyright law, the equivalent laws of other countries, and by international treaty. Therefore, by exercising any of the rights granted to You in Section 1 herein, You indicate Your acceptance of this License and all of its terms and conditions. + +10) Termination for Patent Action. This License shall terminate automatically and You may no longer exercise any of the rights granted to You by this License as of the date You commence an action, including a cross-claim or counterclaim, for patent infringement (i) against Licensor with respect to a patent applicable to software or (ii) against any entity with respect to a patent applicable to the Original Work (but excluding combinations of the Original Work with other software or hardware). + +11) Jurisdiction, Venue and Governing Law. Any action or suit relating to this License may be brought only in the courts of a jurisdiction wherein the Licensor resides or in which Licensor conducts its primary business, and under the laws of that jurisdiction excluding its conflict-of-law provisions. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any use of the Original Work outside the scope of this License or after its termination shall be subject to the requirements and penalties of the U.S. Copyright Act, 17 U.S.C. � 101 et seq., the equivalent laws of other countries, and international treaty. This section shall survive the termination of this License. + +12) Attorneys Fees. In any action to enforce the terms of this License or seeking damages relating thereto, the prevailing party shall be entitled to recover its costs and expenses, including, without limitation, reasonable attorneys' fees and costs incurred in connection with such action, including any appeal of such action. This section shall survive the termination of this License. + +13) Miscellaneous. This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. + +14) Definition of "You" in This License. "You" throughout this License, whether in upper or lower case, means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with you. For purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +15) Right to Use. You may use the Original Work in all ways not otherwise restricted or conditioned by this License or by law, and Licensor promises not to interfere with or be responsible for such uses by You. + +This license is Copyright (C) 2003 Lawrence E. Rosen. All rights reserved. Permission is hereby granted to copy and distribute this license without modification. This license may not be modified without the express written permission of its copyright owner. \ No newline at end of file diff --git a/test/_test_data/test_pkg_tag_not_spdx/code_with_afl.py b/test/_test_data/test_pkg_tag_not_spdx/code_with_afl.py new file mode 100644 index 0000000..e0f473c --- /dev/null +++ b/test/_test_data/test_pkg_tag_not_spdx/code_with_afl.py @@ -0,0 +1,9 @@ +# Copyright (C) 2000 An Author + +# Licensed under the Academic Free License version 2.0 + +import sys + +if __name__ == "__main__": + print("hi") + sys.exit(0) diff --git a/test/_test_data/test_pkg_tag_not_spdx/package.xml b/test/_test_data/test_pkg_tag_not_spdx/package.xml new file mode 100644 index 0000000..45efa83 --- /dev/null +++ b/test/_test_data/test_pkg_tag_not_spdx/package.xml @@ -0,0 +1,7 @@ + + + + test_pkg_tag_not_spdx + AFL + + diff --git a/test/systemtest/test_all_packages.py b/test/systemtest/test_all_packages.py index 13ccc80..90a63d5 100644 --- a/test/systemtest/test_all_packages.py +++ b/test/systemtest/test_all_packages.py @@ -56,6 +56,7 @@ def test_all(self): self.assertIn(b"test_pkg_one_correct_one_license_file_missing", stdout) self.assertIn(b"test_pkg_spdx_name", stdout) self.assertIn(b"test_pkg_spdx_tag", stdout) + self.assertIn(b"test_pkg_tag_not_spdx", stdout) self.assertIn(b"test_pkg_unknown_license", stdout) self.assertIn(b"test_pkg_unknown_license_missing_file", stdout) self.assertIn(b"test_pkg_with_license_and_file", stdout) diff --git a/test/systemtest/test_separate_pkgs.py b/test/systemtest/test_separate_pkgs.py index b718375..3ade966 100644 --- a/test/systemtest/test_separate_pkgs.py +++ b/test/systemtest/test_separate_pkgs.py @@ -26,6 +26,8 @@ class TestPkgs(unittest.TestCase): """Test different test packages.""" + # pylint: disable=too-many-public-methods + # Here it make sense to keep all tests in one place def test_deep_package_folder(self): """Call the linter on directories in three levels. @@ -133,6 +135,15 @@ def test_pkg_spdx_tag(self): self.assertEqual(os.EX_OK, main( ["test/_test_data/test_pkg_spdx_tag"])) + def test_pkg_tag_not_spdx(self): + """Test on a package that has one linked declaration, one code file + but not in SPDX tag. Tag must be gotten from declaration.""" + process, stdout = open_subprocess("test_pkg_tag_not_spdx") + self.assertEqual(os.EX_OK, process.returncode) + self.assertIn(b"WARNING", stdout) + self.assertIn(b"'code_with_afl.py' is of AFL-2.0 but its Tag is AFL.", + stdout) + def test_pkg_unknown_license(self): """Test on a package with an unknown license declared in the package.xml."""