Merge pull request #3197 from nexB/fix-license-dump

Update license db generation
aboutcode-org · Jan 16, 2023 · 0d294bf · 0d294bf
2 parents 0aa964e + 567e9ff
commit 0d294bf
Show file tree

Hide file tree

Showing 44 changed files with 63 additions and 1,543 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -175,7 +175,7 @@ License detection:
 
   - See https://github.com/nexB/scancode-toolkit/issues/3049
 
-- There is a new ``--get-license-data`` scancode command line option to export
+- There is a new console script ``scancode-license-data`` to export
   license data in JSON, YAML and HTML, with indexes and a static website for use
   in the licensedb web site. This becomes the  API way to getr scancode license
   data.

diff --git a/setup-mini.cfg b/setup-mini.cfg
@@ -147,6 +147,7 @@ packages =
 console_scripts =
     scancode = scancode.cli:scancode
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
+    scancode-license-data = licensedcode.license_db:dump_scancode_license_data
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form:

diff --git a/setup.cfg b/setup.cfg
@@ -147,6 +147,7 @@ packages =
 console_scripts =
     scancode = scancode.cli:scancode
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
+    scancode-license-data = licensedcode.license_db:dump_scancode_license_data
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form:

diff --git a/src/licensedcode/cache.py b/src/licensedcode/cache.py
@@ -182,6 +182,12 @@ def load_or_build(
             # TODO: handle unable to lock in a nicer way
             raise
 
+    @property
+    def has_additional_licenses(self):
+        cache = get_cache()
+        if cache.additional_license_directory or cache.additional_license_plugins:
+            return True
+
 
 def build_index(
     licenses_db=None,

diff --git a/src/licensedcode/license_db.py b/src/licensedcode/license_db.py
@@ -31,7 +31,11 @@
 from os.path import join
 from distutils.dir_util import copy_tree
 
+import click
 import saneyaml
+
+from commoncode.cliutils import MISC_GROUP
+from commoncode.cliutils import PluggableCommandLineOption
 from jinja2 import Environment, FileSystemLoader
 from licensedcode.models import load_licenses
 from licensedcode.models import licenses_data_dir
@@ -126,13 +130,16 @@ def generate_details(output_path, environment, licenses, test=False):
 
     ``test`` is to generate a stable output for testing only
     """
+    from licensedcode.cache import get_cache
+    include_builtin = get_cache().has_additional_licenses
+
     if test:
         base_context_mapping = base_context_test
     else:
         base_context_mapping = base_context
     license_details_template = environment.get_template("license_details.html")
     for lic in licenses.values():
-        license_data = lic.to_dict(include_text=True)
+        license_data = lic.to_dict(include_text=False, include_builtin=include_builtin)
         html = license_details_template.render(
             **base_context_mapping,
             license=lic,
@@ -200,19 +207,40 @@ def generate(
     return count
 
 
-def dump_license_data(ctx, param, value):
+def scancode_license_data(path):
     """
-    Dump license data from scancode licenses to the directory ``value`` passed
+    Dump license data from scancode licenses to the directory ``path`` passed
     in from command line.
 
     Dumps data in JSON, YAML and HTML formats and also dumps the .LICENSE file
     with the license text and the data as YAML frontmatter.
     """
-    if not value or ctx.resilient_parsing:
-        return
-
-    import click
-    click.secho(f'Dumping license data to: {value}', err=True)
-    count = generate(build_location=value)
+    click.secho(f'Dumping license data to: {path}', err=True)
+    count = generate(build_location=path)
     click.secho(f'Done dumping #{count} licenses.', err=True)
-    ctx.exit(0)
+
+
+@click.command(name='scancode-license-data')
+@click.option(
+    '--path',
+    type=click.Path(exists=False, writable=True, file_okay=False, resolve_path=True, path_type=str),
+    metavar='DIR',
+    help='Dump the license data in this directory in the LicenseDB format and exit. '
+            'Creates the directory if it does not exist. ',
+    help_group=MISC_GROUP,
+    cls=PluggableCommandLineOption,
+)
+@click.help_option('-h', '--help')
+def dump_scancode_license_data(
+    path,
+    *args,
+    **kwargs,
+):
+    """
+    Dump scancode license data in various formats, and the licenseDB static website at `path`.
+    """
+    scancode_license_data(path=path)
+
+
+if __name__ == '__main__':
+    dump_scancode_license_data()
diff --git a/src/licensedcode/plugin_license.py b/src/licensedcode/plugin_license.py
@@ -12,11 +12,9 @@
 from functools import partial
 
 import attr
-import click
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.cliutils import SCAN_GROUP
 from commoncode.cliutils import SCAN_OPTIONS_GROUP
-from commoncode.cliutils import MISC_GROUP
 from plugincode.scan import ScanPlugin
 from plugincode.scan import scan_impl
 
@@ -33,7 +31,6 @@
 from licensedcode.detection import LicenseDetectionFromResult
 from licensedcode.detection import LicenseMatchFromResult
 from licensedcode.detection import UniqueDetection
-from licensedcode.license_db import dump_license_data
 from packagedcode.utils import combine_expressions
 from scancode.api import SCANCODE_LICENSEDB_URL
 
@@ -121,19 +118,7 @@ class LicenseScanner(ScanPlugin):
             required_options=['license'],
             help='[EXPERIMENTAL] Detect unknown licenses. ',
             help_group=SCAN_OPTIONS_GROUP,
-        ),
-
-        # TODO: consider creating a separate comamnd line option exe instead
-        PluggableCommandLineOption(
-            ('--dump-license-data',),
-            type=click.Path(exists=False, readable=True, file_okay=False, resolve_path=True, path_type=str),
-            metavar='DIR',
-            callback=dump_license_data,
-            help='Dump the license data in this directory in the LicenseDB format and exit. '
-            'Creates the directory if it does not exist. ',
-            help_group=MISC_GROUP,
-            is_eager=True,
-        ),
+        )
     ]
 
     def is_enabled(self, license, **kwargs):  # NOQA
@@ -176,20 +161,17 @@ def process_codebase(self, codebase, **kwargs):
         cche = cache.get_cache()
 
         cle = codebase.get_or_create_current_header()
-        has_additional_licenses = False
 
         if cche.additional_license_directory:
             cle.extra_data['additional_license_directory'] = cche.additional_license_directory
-            has_additional_licenses = True
 
         if cche.additional_license_plugins:
             cle.extra_data['additional_license_plugins'] = cche.additional_license_plugins
-            has_additional_licenses = True
 
-        if TRACE and has_additional_licenses:
+        if TRACE and cche.has_additional_licenses:
             logger_debug(
                 f'add_referenced_filenames_license_matches: additional_licenses',
-                f'has_additional_licenses: {has_additional_licenses}\n',
+                f'has_additional_licenses: {cche.has_additional_licenses}\n',
                 f'additional_license_directory: {cche.additional_license_directory}\n',
                 f'additional_license_plugins : {cche.additional_license_plugins}'
             )

diff --git a/src/licensedcode/templates/footer.html b/src/licensedcode/templates/footer.html
@@ -8,5 +8,6 @@
     </p>
     <p>Designed and built by <a href="https://www.nexb.com/" target="_blank">nexB</a>. Licensed under the <a href="cc-by-4.0.html">Creative Commons Attribution License 4.0 (CC-BY-4.0)</a>.</p>
     <p>Generated with <a href="https://github.com/nexB/scancode-toolkit" target="_blank">ScanCode toolkit</a> {{ scancode_version }} on {{ now }}.</p>
+    <p>This is updated daily automatically with latest updates from the develop branch of scancode-toolkit, if any.</p>
   </footer>
 </div>
diff --git a/tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.html b/tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.html
@@ -106,13 +106,6 @@
 
         </dd>
 
-        <dt style="">is_builtin</dt>
-        <dd>
-
-            True
-
-        </dd>
-
         <dt style="">is_exception</dt>
         <dd>
 
@@ -136,19 +129,6 @@
 
         </dd>
 
-        <dt style="">text</dt>
-        <dd>
-
-            The Free Software Foundation has exempted Bash from the requirement of
-Paragraph 2c of the General Public License.  This is to say, there is
-no requirement for Bash to print a notice when it is started
-interactively in the usual way.  We made this exception because users
-and standards expect shells not to print such messages.  This
-exception applies to any program that serves as a shell and that is
-based primarily on Bash as opposed to other GNU software.
-
-        </dd>
-
     </dl>
     <div class="text-bold">license_text</div>
     <pre id="license-text" class="code mt-1" style="white-space: pre-wrap;"><code>The Free Software Foundation has exempted Bash from the requirement of

diff --git a/tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.json b/tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.json
@@ -5,11 +5,9 @@
   "category": "Copyleft",
   "owner": "Free Software Foundation (FSF)",
   "notes": "this used with GPL 1.0 and 2.0. It was removed from the V3 text https://git.savannah.gnu.org/cgit/bash.git/commit/COPYING?id=3185942a5234e26ab13fa02f9c51d340cec514f8",
-  "is_builtin": true,
   "is_exception": true,
   "spdx_license_key": "LicenseRef-scancode-bash-exception-gpl-2.0",
   "text_urls": [
     "https://git.savannah.gnu.org/cgit/bash.git/plain/COPYING?h=bash-3.0-rc1&id=dd9e6dfa23d0dae4888f11fb8c6a27bc36d1b283"
-  ],
-  "text": "The Free Software Foundation has exempted Bash from the requirement of\nParagraph 2c of the General Public License.  This is to say, there is\nno requirement for Bash to print a notice when it is started\ninteractively in the usual way.  We made this exception because users\nand standards expect shells not to print such messages.  This\nexception applies to any program that serves as a shell and that is\nbased primarily on Bash as opposed to other GNU software."
+  ]
 }
diff --git a/tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.yml b/tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.yml
@@ -4,16 +4,7 @@ name: Bash exception to GPL
 category: Copyleft
 owner: Free Software Foundation (FSF)
 notes: this used with GPL 1.0 and 2.0. It was removed from the V3 text https://git.savannah.gnu.org/cgit/bash.git/commit/COPYING?id=3185942a5234e26ab13fa02f9c51d340cec514f8
-is_builtin: yes
 is_exception: yes
 spdx_license_key: LicenseRef-scancode-bash-exception-gpl-2.0
 text_urls:
   - https://git.savannah.gnu.org/cgit/bash.git/plain/COPYING?h=bash-3.0-rc1&id=dd9e6dfa23d0dae4888f11fb8c6a27bc36d1b283
-text: |
-  The Free Software Foundation has exempted Bash from the requirement of
-  Paragraph 2c of the General Public License.  This is to say, there is
-  no requirement for Bash to print a notice when it is started
-  interactively in the usual way.  We made this exception because users
-  and standards expect shells not to print such messages.  This
-  exception applies to any program that serves as a shell and that is
-  based primarily on Bash as opposed to other GNU software.
diff --git a/tests/licensedcode/data/license_db/license_dump/binary-linux-firmware.html b/tests/licensedcode/data/license_db/license_dump/binary-linux-firmware.html
@@ -106,52 +106,13 @@
 
         </dd>
 
-        <dt style="">is_builtin</dt>
-        <dd>
-
-            True
-
-        </dd>
-
         <dt style="">spdx_license_key</dt>
         <dd>
 
             LicenseRef-scancode-binary-linux-firmware
 
         </dd>
 
-        <dt style="">text</dt>
-        <dd>
-
-            Redistribution. Redistribution and use in binary form, without
-modification, are permitted provided that the following conditions are
-met:
-
-* Redistributions must reproduce the above copyright notice and the
-following disclaimer in the documentation and/or other materials
-provided with the distribution.
-
-* Neither the name of the Copyright Holder nor the names of its
-suppliers may be used to endorse or promote products derived from this
-software without specific prior written permission.
-
-* No reverse engineering, decompilation, or disassembly of this software
-is permitted.
-
-DISCLAIMER. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
-CONTRIBUTORS &#34;AS IS&#34; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
-BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
-FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
-NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE
-
-        </dd>
-
     </dl>
     <div class="text-bold">license_text</div>
     <pre id="license-text" class="code mt-1" style="white-space: pre-wrap;"><code>Redistribution. Redistribution and use in binary form, without

diff --git a/tests/licensedcode/data/license_db/license_dump/binary-linux-firmware.json b/tests/licensedcode/data/license_db/license_dump/binary-linux-firmware.json
@@ -5,7 +5,5 @@
   "category": "Proprietary Free",
   "owner": "Unspecified",
   "homepage_url": "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tree/",
-  "is_builtin": true,
-  "spdx_license_key": "LicenseRef-scancode-binary-linux-firmware",
-  "text": "Redistribution. Redistribution and use in binary form, without\nmodification, are permitted provided that the following conditions are\nmet:\n\n* Redistributions must reproduce the above copyright notice and the\nfollowing disclaimer in the documentation and/or other materials\nprovided with the distribution.\n\n* Neither the name of the Copyright Holder nor the names of its\nsuppliers may be used to endorse or promote products derived from this\nsoftware without specific prior written permission.\n\n* No reverse engineering, decompilation, or disassembly of this software\nis permitted.\n\nDISCLAIMER. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND\nCONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,\nBUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND\nFITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE\nCOPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,\nINCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\nNOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\nUSE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON\nANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\nTHIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE"
+  "spdx_license_key": "LicenseRef-scancode-binary-linux-firmware"
 }
diff --git a/tests/licensedcode/data/license_db/license_dump/binary-linux-firmware.yml b/tests/licensedcode/data/license_db/license_dump/binary-linux-firmware.yml
@@ -4,32 +4,4 @@ name: Binary-Only Linux Firmware License
 category: Proprietary Free
 owner: Unspecified
 homepage_url: https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tree/
-is_builtin: yes
 spdx_license_key: LicenseRef-scancode-binary-linux-firmware
-text: |
-  Redistribution. Redistribution and use in binary form, without
-  modification, are permitted provided that the following conditions are
-  met:
-
-  * Redistributions must reproduce the above copyright notice and the
-  following disclaimer in the documentation and/or other materials
-  provided with the distribution.
-
-  * Neither the name of the Copyright Holder nor the names of its
-  suppliers may be used to endorse or promote products derived from this
-  software without specific prior written permission.
-
-  * No reverse engineering, decompilation, or disassembly of this software
-  is permitted.
-
-  DISCLAIMER. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
-  CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
-  BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
-  FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
-  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-  USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-  ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE
diff --git a/tests/licensedcode/data/license_db/license_dump/bison-exception-2.0.html b/tests/licensedcode/data/license_db/license_dump/bison-exception-2.0.html
@@ -99,13 +99,6 @@
 
         </dd>
 
-        <dt style="">is_builtin</dt>
-        <dd>
-
-            True
-
-        </dd>
-
         <dt style="">is_exception</dt>
         <dd>
 
@@ -149,13 +142,6 @@
 
         </dd>
 
-        <dt style="">text</dt>
-        <dd>
-
-            As a special exception, when this file is copied by Bison into a Bison output file, you may use that output file without restriction. This special exception was added by the Free Software Foundation in version 1.24 of Bison.
-
-        </dd>
-
     </dl>
     <div class="text-bold">license_text</div>
     <pre id="license-text" class="code mt-1" style="white-space: pre-wrap;"><code>As a special exception, when this file is copied by Bison into a Bison output file, you may use that output file without restriction. This special exception was added by the Free Software Foundation in version 1.24 of Bison.</code></pre>

diff --git a/tests/licensedcode/data/license_db/license_dump/bison-exception-2.0.json b/tests/licensedcode/data/license_db/license_dump/bison-exception-2.0.json
@@ -4,10 +4,8 @@
   "name": "Bison exception to GPL 2.0 or later",
   "category": "Copyleft Limited",
   "owner": "Free Software Foundation (FSF)",
-  "is_builtin": true,
   "is_exception": true,
   "spdx_license_key": "LicenseRef-scancode-bison-exception-2.0",
   "faq_url": "http://www.gnu.org/software/bison/manual/bison.html#Conditions",
-  "standard_notice": "This library is free software; you can redistribute it and/or modify it\nunder the terms of the GNU General Public License as published by the Free\nSoftware Foundation; either version 2, or (at your option) any later\nversion.\nThis library is distributed in the hope that it will be useful, but WITHOUT\nANY WARRANTY; without even the implied warranty of MERCHANTABILITY or\nFITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for\nmore details.\nYou should have received a copy of the GNU General Public License along\nwith this library; see the file COPYING. If not, write to the Free Software\nFoundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.\nAs a special exception, when this file is copied by Bison into a Bison\noutput file, you may use that output file without restriction. This special\nexception was added by the Free Software Foundation in version 1.24 of\nBison.\n",
-  "text": "As a special exception, when this file is copied by Bison into a Bison output file, you may use that output file without restriction. This special exception was added by the Free Software Foundation in version 1.24 of Bison."
+  "standard_notice": "This library is free software; you can redistribute it and/or modify it\nunder the terms of the GNU General Public License as published by the Free\nSoftware Foundation; either version 2, or (at your option) any later\nversion.\nThis library is distributed in the hope that it will be useful, but WITHOUT\nANY WARRANTY; without even the implied warranty of MERCHANTABILITY or\nFITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for\nmore details.\nYou should have received a copy of the GNU General Public License along\nwith this library; see the file COPYING. If not, write to the Free Software\nFoundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.\nAs a special exception, when this file is copied by Bison into a Bison\noutput file, you may use that output file without restriction. This special\nexception was added by the Free Software Foundation in version 1.24 of\nBison.\n"
 }