Merge branch '2021-2-9-release-prep' into develop

.VERSION

@@ -0,0 +1,3 @@
+refs=$Format:%D$
+commit=$Format:%H$
+abbrev_commit=$Format:%H$

.bumpversion.cfg

@@ -1,6 +1,5 @@
 [bumpversion]
-current_version = 21.01.02
+current_version = 21.2.9
 files = setup.cfg src/scancode_config.py
 commit = False
 tag = False
-

.gitattributes

@@ -1,2 +1,4 @@
 # Ignore all Git auto CR/LF line endings conversions
 * binary
+# save the version details for git tarballs
+.VERSION  export-subst

CHANGELOG.rst

@@ -6,15 +6,21 @@ v21.x (next)
 
 
 
-v21.1.21
---------
+v21.2.9
+-------
+
+Security:
+
+ - Update vulnerable LXML to version 4.6.2 to fix
+   https://nvd.nist.gov/vuln/detail/CVE-2020-27783
+   This was detected thanks to https://github.com/nexb/vulnerablecode
 
 Operating system support:
 
  - Drop support for Python 2  #295
  - Drop support for 32 bits on Windows #335
  - Add support for Python 64 bits on Windows 64 bits #335
- - Add support for Python 3.6 to Python 3.9 on Linux, Windows and macOS.
+ - Add support for Python 3.6, 37, 3.8 and 3.9 on Linux, Windows and macOS.
    These are now tested on Azure.
  - Add deprecation message for native Windows support #2366
 
@@ -39,7 +45,7 @@ Copyright scanning:
 
  - Improve detection with minor grammar fixes
 
-Misc.
+Misc.:
 
  - Adopt a new calendar date-based versioning for scancode-toolkit version numbers
  - Update thirdparty dependencies and built-in plugins

INSTALL.rst

@@ -2,80 +2,94 @@
 Installation
 ============
 
-There are 4 main ways you can `install ScanCode <https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html>`_.
+There are a few ways you can `install ScanCode <https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html>`_.
+
+- Recommended standard install for everyone: Use a release download and install as an application
+
+- Advanced installation options:
+ - pip install a Python PyPI package
+ - from source code using a git clone
+ - using Docker
 
-- Installation as an Application: Downloading Releases (Recommended)
-- Docker Installation
-- Installation as a library: via pip
-- Installation from Source Code: Git Clone
 
 Prerequisites
 -------------
 
-Before installing ScanCode make sure you've installed the prerequisites properly. This mainly
-refers to installing the required Python interpreter (Python 3.6 is recommended).
+Before installing ScanCode make sure you have  installed these prerequisites.
+The main one is a Python interpreter.
+Python 3.6 is required for the standard installation.
 
 - For Linux(Ubuntu): ``sudo apt install python3.6-dev bzip2 xz-utils zlib1g libxml2-dev libxslt1-dev``
-- For MacOS: Install Python 3.6.8 from https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.6.pkg
-- For Windows: Install Python 3.6.8 from https://www.python.org/ftp/python/3.6.8/python-3.6.8.exe
+- For MacOS: Install Python 3.6.8 from https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg
+- For Windows: Install Python 3.6.8 from https://www.python.org/ftp/python/3.6.8/python-3.6.8-amd64.exe
+- For FreeBSD: (this ineeds to be documented)
 
-Refer `Prerequisites <https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html#prerequisites>`_ for detailed information on all different platforms and Python Versions.
+Refer `Prerequisites <https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html#prerequisites>`_
+for detailed information on all different operating systems and Python versions.
 
-Installation as an Application : Downloading Releases
------------------------------------------------------
 
-#. Download and extract the latest ScanCode release from https://github.com/nexB/scancode-toolkit/releases/
+Use a release download and install as an application
+----------------------------------------------------
 
-#. Open a terminal window and then `cd` to the extracted ScanCode directory. 
+- Download and extract the latest ScanCode release from
+  https://github.com/nexB/scancode-toolkit/releases/
 
-#. Run this command to self-configure and display the help-text.
+- Open a terminal window (or command prompt on Windows) and then `cd` to the
+  extracted ScanCode directory. 
+
+- Run this command to self-configure and display the initial command line help:
 
     - Linux/Mac : ``./scancode --help``
     - Windows : ``scancode --help``
 
-Docker Installation
--------------------
 
-#. Download the Source Code as an archive from the `GitHub releases <https://github.com/nexB/scancode-toolkit/releases>`_ and unzip it, or via `git clone`.
+Advanced installation: pip install a Python PyPI package
+--------------------------------------------------------
 
-#. Build the docker image from the `scancode-toolkit` directory.::
+- Create a virtual environment for Python 3.6 (of higher) and activate it::
 
-	docker build -t scancode-toolkit .
+    virtualenv -p /usr/bin/python3.6 venv-scancode && source venv-scancode/bin/activate
 
-#. Mount current working directory and run scan on mounted folder::
+- Run ``pip install scancode-toolkit[full]``
 
-    docker run -v $PWD/:/project scancode-toolkit -clpeui --json-pp /project/result.json /project
+Note that the ``[full]`` extra option is required to get a working installation
+except in some advanced use cases.
 
-Note that the parameters *before* ``scancode-toolkit`` are used for docker,
-those after will be forwarded to scancode.
 
-Installation as a library: via pip
-----------------------------------
 
-#. Create a Python 3.6 Virtual Environment and activate the same::
+Advanced installation: using Docker
+-----------------------------------
 
-    virtualenv -p /usr/bin/python3.6 venv-scancode && source venv-scancode/bin/activate
+- Download the Source Code as an archive from the `GitHub releases
+  <https://github.com/nexB/scancode-toolkit/releases>`_ and unzip it, or via
+  `git clone`.
 
-#. Run ``pip install scancode-toolkit[full]``
+- Build the docker image from the `scancode-toolkit` directory::
 
-Installation from Source Code: Git Clone
-----------------------------------------
+    docker build -t scancode-toolkit .
 
-#. Download the Source Code or Use Git Clone::
+- Mount current working directory and run a scan the mounted folder::
 
-    git clone https://github.com/nexB/scancode-toolkit.git
-    cd scancode-toolkit
+    docker run -v $PWD/:/project scancode-toolkit -clpeui --json-pp /project/result.json /project
+
+Note that the parameters *before* ``scancode-toolkit`` are used by docker and
+those after will be forwarded to scancode.
 
-#. You can jump to any checkpoint/Branch/Commit using the following command::
 
-    git checkout master
+Advanced installation: from source code using a git clone
+---------------------------------------------------------
 
-#. Run the Configure Script
+- Download the Source Code or Use Git Clone::
+
+    git clone https://github.com/nexB/scancode-toolkit.git
+    cd scancode-toolkit
 
-  - On Linux/Mac: ``./configure``
-  - On Windows: ``configure``
+- Run the configure script for development usage:
 
+  - On Linux/Mac: ``./configure --dev``
+  - On Windows: ``configure --dev``
 
-Note the `Commands will vary <https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html#commands-variation>`_ across different Installation methods and Platforms.
 
-If this displays the `Help Text <https://scancode-toolkit.readthedocs.io/en/latest/cli-reference/help-text-options.html#help-text>`_, you are all set to start using ScanCode.
+If this displays the `help text
+<https://scancode-toolkit.readthedocs.io/en/latest/cli-reference/help-text-options.html#help-text>`_,
+you are all set to start using ScanCode.

MANIFEST.in

@@ -1,18 +1,25 @@
 graft src
 graft thirdparty
 recursive-include etc configure.py
+graft etc/thirdparty
+recursive-include etc/thirdparty *
 
 include *.LICENSE
 include NOTICE
 include *.ABOUT
-include *.toml
-include *.rst
+
 include setup.*
 include configure*
+include requirements*
+include *.toml
+
 include extractcode*
 include scancode*
+
+
+include *.rst
 include Dockerfile
-include requirements*
+include .VERSION
 
 global-exclude *.py[co] __pycache__ *.*~
 

azure-pipelines.yml

@@ -205,7 +205,7 @@ jobs:
 ########################################################################
 # RELEASE on 3.6. Also check that we can pip install
 ########################################################################
-
+#
 #    - job: Build_release_archive_py3
 #      pool:
 #          vmImage: ubuntu-16.04
@@ -218,9 +218,6 @@ jobs:
 #                versionSpec: '3.6'
 #            displayName: 'Install Python 3.6'
 #
-#          - script: ./etc/release/scancode-create-release.sh
-#            displayName: 'Build installable releases'
-#
 #          - script: ./etc/release/scancode-test-pip-install.sh
 #            displayName: 'Test pip wheel installation'
 #

etc/release/README.rst

@@ -151,7 +151,7 @@ etc/thirdparty
   and save to thirdparty and update the ABOUT and LICENSE files as needed.
 
 * This virtualenv app contains also bundled pip, wheel and setuptools that are
-  essential for the correct operations to work.
+  essential for the installation to work.
 
 
 Other files 

etc/release/fetch_requirements.py

@@ -59,6 +59,10 @@
     is_flag=True,
     help='Allow requirements without pinned versions.',
 )
+@click.option('-s', '--only-sources',
+    is_flag=True,
+    help='Fetch only the corresponding source distributions.',
+)
 @click.help_option('-h', '--help')
 def fetch_requirements(
     requirements_file,
@@ -68,6 +72,7 @@ def fetch_requirements(
     with_sources,
     with_about,
     allow_unpinned,
+    only_sources,
 ):
     """
     Fetch and save to THIRDPARTY_DIR all the required wheels for pinned
@@ -85,21 +90,21 @@ def fetch_requirements(
     operating_systems = operating_system
     requirements_files = requirements_file
 
-    envs = itertools.product(python_versions, operating_systems)
-    envs = (utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in envs)
-
-    for env, reqf in itertools.product(envs, requirements_files):
-        for package, error in utils_thirdparty.fetch_wheels(
-            environment=env,
-            requirements_file=reqf,
-            allow_unpinned=allow_unpinned,
-            dest_dir=thirdparty_dir,
-        ):
-            if error:
-                print('Failed to fetch wheel:', package, ':', error)
+    if not only_sources:
+        envs = itertools.product(python_versions, operating_systems)
+        envs = (utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in envs)
+        for env, reqf in itertools.product(envs, requirements_files):
+            for package, error in utils_thirdparty.fetch_wheels(
+                environment=env,
+                requirements_file=reqf,
+                allow_unpinned=allow_unpinned,
+                dest_dir=thirdparty_dir,
+            ):
+                if error:
+                    print('Failed to fetch wheel:', package, ':', error)
 
     # optionally fetch sources
-    if with_sources:
+    if with_sources or only_sources:
         for reqf in requirements_files:
             for package, error in utils_thirdparty.fetch_sources(
                 requirements_file=reqf,
@@ -111,7 +116,11 @@ def fetch_requirements(
 
     if with_about:
         utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir)
-        utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
+        utils_thirdparty.find_problems(
+            dest_dir=thirdparty_dir,
+            report_missing_sources=with_sources or only_sources,
+            report_missing_wheels=not only_sources,
+        )
 
 
 if __name__ == '__main__':