Merge pull request #3201 from nexB/fix-license-dump

Remove license text from index.json of licenseDB

.VERSION

@@ -1,3 +1,5 @@
 refs=$Format:%D$
 commit=$Format:%H$
-abbrev_commit=$Format:%H$
+abbrev_commit=$Format:%h$
+committer_date=$Format:%cs$
+git_describe=$Format:%(describe)$

src/licensedcode/license_db.py

@@ -1,27 +1,11 @@
-# SPDX-License-Identifier: CC-BY-4.0 AND Apache-2.0
 #
-# https://github.com/nexB/scancode-licensedb
-# Copyright 2020 nexB Inc. and others.
+# Copyright (c) nexB Inc. and others. All rights reserved.
 # ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
 #
-# ScanCode LicenseDB data is licensed under the Creative Commons Attribution
-# License 4.0 (CC-BY-4.0).
-# Some licenses, such as the GNU GENERAL PUBLIC LICENSE, are subject to other licenses.
-# See the corresponding license text for the specific license conditions.
-#
-# ScanCode LicenseDB software is licensed under the Apache License version 2.0.
-# You may not use this software except in compliance with the License.
-# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software distributed
-# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
-# CONDITIONS OF ANY KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations under the License.
-#
-# ScanCode LicenseDB is generated with ScanCode Toolkit. The database and its contents
-# are provided on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-# either express or implied.
-# No content from ScanCode LicenseDB should be considered or used as legal advice.
-# Consult an attorney for any legal advice.
 
 import os
 import json
@@ -40,7 +24,7 @@
 from licensedcode.models import load_licenses
 from licensedcode.models import licenses_data_dir
 from scancode_config import __version__ as scancode_version
-
+from scancode_config import spdx_license_list_version
 
 TEMPLATES_DIR = os.path.join(dirname(__file__), 'templates')
 STATIC_DIR = os.path.join(dirname(__file__), 'static')
@@ -51,18 +35,19 @@ def write_file(path, filename, content):
 
 
 def now():
-    return datetime.now().strftime("%b %d, %Y")
-
+    return datetime.utcnow().strftime('%Y-%m-%dT')
 
 base_context = {
     "scancode_version": scancode_version,
     "now": now(),
+    "spdx_license_list_version": spdx_license_list_version,
 }
 
 
 base_context_test = {
     "scancode_version": "32.0.0b1",
     "now": "Dec 22, 2022",
+    "spdx_license_list_version": "3.19",
 }
 
 
@@ -98,7 +83,6 @@ def generate_indexes(output_path, environment, licenses, test=False):
             "other_spdx_license_keys": lic.other_spdx_license_keys,
             "is_exception": lic.is_exception,
             "is_deprecated": lic.is_deprecated,
-            "text": lic.text,
             "json": f"{key}.json",
             "yaml": f"{key}.yml",
             "html": f"{key}.html",
@@ -140,6 +124,7 @@ def generate_details(output_path, environment, licenses, test=False):
     license_details_template = environment.get_template("license_details.html")
     for lic in licenses.values():
         license_data = lic.to_dict(include_text=False, include_builtin=include_builtin)
+        license_data_with_text = lic.to_dict(include_text=True, include_builtin=include_builtin)
         html = license_details_template.render(
             **base_context_mapping,
             license=lic,
@@ -149,12 +134,12 @@ def generate_details(output_path, environment, licenses, test=False):
         write_file(
             output_path,
             f"{lic.key}.yml",
-            saneyaml.dump(license_data, indent=2)
+            saneyaml.dump(license_data_with_text, indent=2)
         )
         write_file(
             output_path,
             f"{lic.key}.json",
-            json.dumps(license_data, indent=2, sort_keys=False)
+            json.dumps(license_data_with_text, indent=2, sort_keys=False)
         )
         lic.dump(output_path)
 

src/licensedcode/templates/footer.html

@@ -6,8 +6,11 @@
       <a href="https://github.com/nexB/scancode-licensedb" target="_blank">GitHub</a> ·
       <a href="https://www.aboutcode.org/" target="_blank">AboutCode</a>
     </p>
-    <p>Designed and built by <a href="https://www.nexb.com/" target="_blank">nexB</a>. Licensed under the <a href="cc-by-4.0.html">Creative Commons Attribution License 4.0 (CC-BY-4.0)</a>.</p>
-    <p>Generated with <a href="https://github.com/nexB/scancode-toolkit" target="_blank">ScanCode toolkit</a> {{ scancode_version }} on {{ now }}.</p>
-    <p>This is updated daily automatically with latest updates from the develop branch of scancode-toolkit, if any.</p>
+    <p>Designed and built by <a href="https://www.nexb.com/" target="_blank">nexB</a>.
+    Licensed under the <a href="cc-by-4.0.html">Creative Commons Attribution License 4.0 (CC-BY-4.0)</a>.</p>
+    <p>Generated with <a href="https://github.com/nexB/scancode-toolkit" target="_blank">ScanCode toolkit</a>
+    {{ scancode_version }} on {{ now }}.</p>
+    <p>SPDX Licenses list version: {{ spdx_license_list_version }}</p>
+
   </footer>
 </div>

src/scancode_config.py

@@ -70,43 +70,80 @@ def _create_dir(location):
 ################################################################################
 # INVARIABLE INSTALLATION-SPECIFIC, BUILT-IN LOCATIONS AND FLAGS
 ################################################################################
-# these are guaranteed to be there and are entirely based on and relative to the
+# These are guaranteed to be there and are entirely based on and relative to the
 # current installation location. This is where the source code and static data
 # lives.
 
 
-# in case package is not installed or we do not have setutools/pkg_resources
-# on hand fall back to this version
-__version__ = '32.0.0rc1'
+system_temp_dir = tempfile.gettempdir()
+scancode_src_dir = dirname(__file__)
+scancode_root_dir = dirname(scancode_src_dir)
 
-# used to warn user when the version is out of date
-__release_date__ = datetime.datetime(2023, 1, 3)
+# USAGE MODE FLAG. If we have a .git dir, we are a git clone and developing
+_SCANCODE_DEV_MODE = os.path.exists(join(scancode_root_dir, '.git'))
 
-# See https://github.com/nexB/scancode-toolkit/issues/2653 for more information
-# on the data format version
-__output_format_version__ = '3.0.0'
+# There are multiple cases for versions, depending on where we come from:
+__version__ = ''
 
-#
-spdx_license_list_version = '3.19'
+# 1. - a git clone: we can use git describe which take consider the closest tag
+#   "git describe --dirty > SCANCODE_VERSION"
+# This is the common case when we develop, including dump the latestlicense db
+if _SCANCODE_DEV_MODE:
 
-try:
-    from pkg_resources import get_distribution, DistributionNotFound
+    from subprocess import check_output
+    from subprocess import STDOUT
+    from subprocess import CalledProcessError
+
+    # this may fail with exceptions
+    cmd = 'git', 'describe',
     try:
-        __version__ = get_distribution('scancode-toolkit').version
-    except DistributionNotFound:
+        output = check_output(cmd, stderr=STDOUT)
+        __version__ = output.decode('utf-8').strip()
+    except (Exception, CalledProcessError) as e:
         pass
-except ImportError:
-    pass
 
-system_temp_dir = tempfile.gettempdir()
-scancode_src_dir = dirname(__file__)
-scancode_root_dir = dirname(scancode_src_dir)
+# 2. - the scancode-tool binary or sources app archives: with a SCANCODE_VERSION
+#   file containing the git tag of the release with "git describe --tags > SCANCODE_VERSION"
+# - a tarball without an updated .VERSION, we cannot tell anything and we will use importlib metadata
+# - a wheel or sdist in which case we use the importlib metadata
+# we use importlib metadata in all these cases
+if not __version__:
+    try:
+        from importlib_metadata import version
+        __version__ = version('scancode-toolkit')
+    except Exception:
+        pass
 
-################################################################################
-# USAGE MODE FLAGS
-################################################################################
+# 3. - a tarball or zip archive from git with a .VERSION file with var substitution:
+#   .VERSION will contain:
+#      refs=HEAD -> fix-license-dump
+#      commit=5ccc92e69cffb503e9bedc7fce5a1dbb0fd851da
+#      abbrev_commit=5ccc92e69c
+#      committer_date=2023-01-16
+#      git_describe=v31.2.3-328-g5ccc92e69c
+#    otherwise it contains:
+#      refs=$Format:%D$
+#      commit=$Format:%H$
+#      abbrev_commit=$Format:%h$
+#      committer_date=$Format:%cs$
+#      git_describe=$Format:%(describe)$
+# NOTE: we do not handle this for now
+
+# 4. hardcoded This is the default, fallback version in case package is not installed or we
+# do not have a proper version otherwise.
+if not __version__:
+    __version__ = '32.0.0rc1'
+
+#######################
+# used to warn user when the version is out of date
+__release_date__ = datetime.datetime(2023, 1, 15)
 
-_SCANCODE_DEV_MODE = os.path.exists(join(scancode_root_dir, '.git'))
+# See https://github.com/nexB/scancode-toolkit/issues/2653 for more information
+# on the data format version
+__output_format_version__ = '3.0.0'
+
+#
+spdx_license_list_version = '3.19'
 
 ################################################################################
 # USAGE MODE-, INSTALLATION- and IMPORT- and RUN-SPECIFIC DIRECTORIES
@@ -171,6 +208,5 @@ def _create_dir(location):
 _prefix = 'scancode-tk-' + __version__ + '-'
 scancode_temp_dir = tempfile.mkdtemp(prefix=_prefix, dir=__scancode_temp_base_dir)
 
-
 # Used for tests to regenerate fixtures with regen=True
 REGEN_TEST_FIXTURES = os.getenv('SCANCODE_REGEN_TEST_FIXTURES', False)

tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.html

@@ -150,6 +150,7 @@
     </p>
     <p>Designed and built by <a href="https://www.nexb.com/" target="_blank">nexB</a>. Licensed under the <a href="cc-by-4.0.html">Creative Commons Attribution License 4.0 (CC-BY-4.0)</a>.</p>
     <p>Generated with <a href="https://github.com/nexB/scancode-toolkit" target="_blank">ScanCode toolkit</a> 32.0.0b1 on Dec 22, 2022.</p>
+    <p>This is updated daily automatically with latest updates from the develop branch of scancode-toolkit, if any.</p>
   </footer>
 </div>
 

tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.json

@@ -9,5 +9,6 @@
   "spdx_license_key": "LicenseRef-scancode-bash-exception-gpl-2.0",
   "text_urls": [
     "https://git.savannah.gnu.org/cgit/bash.git/plain/COPYING?h=bash-3.0-rc1&id=dd9e6dfa23d0dae4888f11fb8c6a27bc36d1b283"
-  ]
+  ],
+  "text": "The Free Software Foundation has exempted Bash from the requirement of\nParagraph 2c of the General Public License.  This is to say, there is\nno requirement for Bash to print a notice when it is started\ninteractively in the usual way.  We made this exception because users\nand standards expect shells not to print such messages.  This\nexception applies to any program that serves as a shell and that is\nbased primarily on Bash as opposed to other GNU software."
 }

tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.yml

@@ -8,3 +8,11 @@ is_exception: yes
 spdx_license_key: LicenseRef-scancode-bash-exception-gpl-2.0
 text_urls:
   - https://git.savannah.gnu.org/cgit/bash.git/plain/COPYING?h=bash-3.0-rc1&id=dd9e6dfa23d0dae4888f11fb8c6a27bc36d1b283
+text: |
+  The Free Software Foundation has exempted Bash from the requirement of
+  Paragraph 2c of the General Public License.  This is to say, there is
+  no requirement for Bash to print a notice when it is started
+  interactively in the usual way.  We made this exception because users
+  and standards expect shells not to print such messages.  This
+  exception applies to any program that serves as a shell and that is
+  based primarily on Bash as opposed to other GNU software.

tests/licensedcode/data/license_db/license_dump/binary-linux-firmware.html

@@ -153,6 +153,7 @@
     </p>
     <p>Designed and built by <a href="https://www.nexb.com/" target="_blank">nexB</a>. Licensed under the <a href="cc-by-4.0.html">Creative Commons Attribution License 4.0 (CC-BY-4.0)</a>.</p>
     <p>Generated with <a href="https://github.com/nexB/scancode-toolkit" target="_blank">ScanCode toolkit</a> 32.0.0b1 on Dec 22, 2022.</p>
+    <p>This is updated daily automatically with latest updates from the develop branch of scancode-toolkit, if any.</p>
   </footer>
 </div>
 

tests/licensedcode/data/license_db/license_dump/binary-linux-firmware.json

@@ -5,5 +5,6 @@
   "category": "Proprietary Free",
   "owner": "Unspecified",
   "homepage_url": "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tree/",
-  "spdx_license_key": "LicenseRef-scancode-binary-linux-firmware"
+  "spdx_license_key": "LicenseRef-scancode-binary-linux-firmware",
+  "text": "Redistribution. Redistribution and use in binary form, without\nmodification, are permitted provided that the following conditions are\nmet:\n\n* Redistributions must reproduce the above copyright notice and the\nfollowing disclaimer in the documentation and/or other materials\nprovided with the distribution.\n\n* Neither the name of the Copyright Holder nor the names of its\nsuppliers may be used to endorse or promote products derived from this\nsoftware without specific prior written permission.\n\n* No reverse engineering, decompilation, or disassembly of this software\nis permitted.\n\nDISCLAIMER. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND\nCONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,\nBUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND\nFITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE\nCOPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,\nINCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\nNOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\nUSE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON\nANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\nTHIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE"
 }

tests/licensedcode/data/license_db/license_dump/binary-linux-firmware.yml

@@ -5,3 +5,30 @@ category: Proprietary Free
 owner: Unspecified
 homepage_url: https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tree/
 spdx_license_key: LicenseRef-scancode-binary-linux-firmware
+text: |
+  Redistribution. Redistribution and use in binary form, without
+  modification, are permitted provided that the following conditions are
+  met:
+
+  * Redistributions must reproduce the above copyright notice and the
+  following disclaimer in the documentation and/or other materials
+  provided with the distribution.
+
+  * Neither the name of the Copyright Holder nor the names of its
+  suppliers may be used to endorse or promote products derived from this
+  software without specific prior written permission.
+
+  * No reverse engineering, decompilation, or disassembly of this software
+  is permitted.
+
+  DISCLAIMER. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+  CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
+  BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+  FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+  USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+  ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE

tests/licensedcode/data/license_db/license_dump/bison-exception-2.0.html

@@ -157,6 +157,7 @@
     </p>
     <p>Designed and built by <a href="https://www.nexb.com/" target="_blank">nexB</a>. Licensed under the <a href="cc-by-4.0.html">Creative Commons Attribution License 4.0 (CC-BY-4.0)</a>.</p>
     <p>Generated with <a href="https://github.com/nexB/scancode-toolkit" target="_blank">ScanCode toolkit</a> 32.0.0b1 on Dec 22, 2022.</p>
+    <p>This is updated daily automatically with latest updates from the develop branch of scancode-toolkit, if any.</p>
   </footer>
 </div>
 

tests/licensedcode/data/license_db/license_dump/bison-exception-2.0.json

@@ -7,5 +7,6 @@
   "is_exception": true,
   "spdx_license_key": "LicenseRef-scancode-bison-exception-2.0",
   "faq_url": "http://www.gnu.org/software/bison/manual/bison.html#Conditions",
-  "standard_notice": "This library is free software; you can redistribute it and/or modify it\nunder the terms of the GNU General Public License as published by the Free\nSoftware Foundation; either version 2, or (at your option) any later\nversion.\nThis library is distributed in the hope that it will be useful, but WITHOUT\nANY WARRANTY; without even the implied warranty of MERCHANTABILITY or\nFITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for\nmore details.\nYou should have received a copy of the GNU General Public License along\nwith this library; see the file COPYING. If not, write to the Free Software\nFoundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.\nAs a special exception, when this file is copied by Bison into a Bison\noutput file, you may use that output file without restriction. This special\nexception was added by the Free Software Foundation in version 1.24 of\nBison.\n"
+  "standard_notice": "This library is free software; you can redistribute it and/or modify it\nunder the terms of the GNU General Public License as published by the Free\nSoftware Foundation; either version 2, or (at your option) any later\nversion.\nThis library is distributed in the hope that it will be useful, but WITHOUT\nANY WARRANTY; without even the implied warranty of MERCHANTABILITY or\nFITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for\nmore details.\nYou should have received a copy of the GNU General Public License along\nwith this library; see the file COPYING. If not, write to the Free Software\nFoundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.\nAs a special exception, when this file is copied by Bison into a Bison\noutput file, you may use that output file without restriction. This special\nexception was added by the Free Software Foundation in version 1.24 of\nBison.\n",
+  "text": "As a special exception, when this file is copied by Bison into a Bison output file, you may use that output file without restriction. This special exception was added by the Free Software Foundation in version 1.24 of Bison."
 }

tests/licensedcode/data/license_db/license_dump/bison-exception-2.0.yml

@@ -22,3 +22,6 @@ standard_notice: |
   output file, you may use that output file without restriction. This special
   exception was added by the Free Software Foundation in version 1.24 of
   Bison.
+text: As a special exception, when this file is copied by Bison into a Bison output file, you
+  may use that output file without restriction. This special exception was added by the Free
+  Software Foundation in version 1.24 of Bison.

tests/licensedcode/data/license_db/license_dump/bison-exception-2.2.html

@@ -184,6 +184,7 @@
     </p>
     <p>Designed and built by <a href="https://www.nexb.com/" target="_blank">nexB</a>. Licensed under the <a href="cc-by-4.0.html">Creative Commons Attribution License 4.0 (CC-BY-4.0)</a>.</p>
     <p>Generated with <a href="https://github.com/nexB/scancode-toolkit" target="_blank">ScanCode toolkit</a> 32.0.0b1 on Dec 22, 2022.</p>
+    <p>This is updated daily automatically with latest updates from the develop branch of scancode-toolkit, if any.</p>
   </footer>
 </div>