Merge remote-tracking branch 'origin/main' into 23-publish-scan-to-fe…

…deratedcode

CHANGELOG.rst

@@ -8,6 +8,14 @@ v34.8.2 (unreleased)
   ``android_inspector``, which provides a pipeline for Android APK
   deploy-to-development analysis.
 
+- Remove the sleep time in the context of testing ``matchcode.poll_run_url_status``
+  to speed up the test.
+  https://github.com/aboutcode-org/scancode.io/issues/1411
+
+- Add ability to specify the CycloneDX output spec version using the ``output``
+  management command and providing the ``cyclonedx:VERSION`` syntax as format value.
+  https://github.com/aboutcode-org/scancode-action/issues/8
+
 v34.8.1 (2024-09-06)
 --------------------
 

README.rst

@@ -56,10 +56,189 @@ ScanCode.io should be considered or used as legal advice. Consult an Attorney
 for any legal advice.
 
 
+
+
 .. |ci-tests| image:: https://github.com/aboutcode-org/scancode.io/actions/workflows/ci.yml/badge.svg?branch=main
     :target: https://github.com/aboutcode-org/scancode.io/actions/workflows/ci.yml
     :alt: CI Tests Status
 
 .. |docs-rtd| image:: https://readthedocs.org/projects/scancodeio/badge/?version=latest
     :target: https://scancodeio.readthedocs.io/en/latest/?badge=latest
     :alt: Documentation Build Status
+
+
+Acknowledgements, Funding, Support and Sponsoring
+--------------------------------------------------------
+
+This project is funded, supported and sponsored by:
+
+- Generous support and contributions from users like you!
+- the European Commission NGI programme
+- the NLnet Foundation 
+- the Swiss State Secretariat for Education, Research and Innovation (SERI)
+- Google, including the Google Summer of Code and the Google Seasons of Doc programmes
+- Mercedes-Benz Group
+- Microsoft and Microsoft Azure
+- AboutCode ASBL
+- nexB Inc. 
+
+
+
+|europa|   |dgconnect| 
+
+|ngi|   |nlnet|   
+
+|aboutcode|  |nexb|
+
+
+This project was funded through the NGI0 Discovery Fund, a fund established by NLnet with financial
+support from the European Commission's Next Generation Internet programme, under the aegis of DG
+Communications Networks, Content and Technology under grant agreement No 825322.
+
+|ngidiscovery| https://nlnet.nl/project/vulnerabilitydatabase/
+
+
+This project is funded through the NGI0 Entrust Fund, a fund established by NLnet with financial
+support from the European Commission's Next Generation Internet programme, under the aegis of DG
+Communications Networks, Content and Technology under grant agreement No 101069594.
+
+|ngizeroentrust| https://nlnet.nl/project/FederatedSoftwareMetadata/
+
+
+This project was funded through the NGI0 Commons Fund, a fund established by NLnet with financial
+support from the European Commission's Next Generation Internet programme, under the aegis of DG
+Communications Networks, Content and Technology under grant agreement No 101135429. Additional
+funding is made available by the Swiss State Secretariat for Education, Research and Innovation
+(SERI). 
+
+|ngizerocommons| |swiss| https://nlnet.nl/project/FederatedCodeNext/
+
+
+This project was funded through the NGI0 Entrust Fund, a fund established by NLnet with financial
+support from the European Commission's Next Generation Internet programme, under the aegis of DG
+Communications Networks, Content and Technology under grant agreement No 101069594. 
+
+|ngizeroentrust| https://nlnet.nl/project/Back2source/
+
+
+This project was funded through the NGI0 Core Fund, a fund established by NLnet with financial
+support from the European Commission's Next Generation Internet programme, under the aegis of DG
+Communications Networks, Content and Technology under grant agreement No 101092990.
+
+|ngizerocore| https://nlnet.nl/project/Back2source-next/
+
+
+This project was funded through the NGI0 Core Fund, a fund established by NLnet with financial
+support from the European Commission's Next Generation Internet programme, under the aegis of DG
+Communications Networks, Content and Technology under grant agreement No 101092990. 
+
+|ngizerocore| https://nlnet.nl/project/FastScan/
+
+
+This project was funded through the NGI0 Commons Fund, a fund established by NLnet with financial
+support from the European Commission's Next Generation Internet programme, under the aegis of DG
+Communications Networks, Content and Technology under grant agreement No 101135429. Additional
+funding is made available by the Swiss State Secretariat for Education, Research and Innovation
+(SERI).
+
+|ngizerocommons| |swiss| https://nlnet.nl/project/MassiveFOSSscan/
+
+
+This project was funded through the NGI Assure Fund, a fund established by NLnet with financial
+support from the European Commission's Next Generation Internet programme, under the aegis of DG
+Communications Networks, Content and Technology under grant agreement No 957073.
+
+|ngiassure| https://nlnet.nl/project/FOSS-supplychain/
+
+
+This project was funded through the NGI0 Entrust Fund, a fund established by NLnet with financial
+support from the European Commission's Next Generation Internet programme, under the aegis of DG
+Communications Networks, Content and Technology under grant agreement No 101069594.
+
+|ngizeroentrust| https://nlnet.nl/project/FOSS-supplychain-II/
+
+
+This project was funded through the NGI0 Entrust Fund, a fund established by NLnet with financial
+support from the European Commission's Next Generation Internet programme, under the aegis of DG
+Communications Networks, Content and Technology under grant agreement No 101069594. 
+
+|ngizeroentrust| https://nlnet.nl/project/purl2all/
+
+
+This project was funded through the NGI0 Entrust Fund, a fund established by NLnet with financial
+support from the European Commission's Next Generation Internet programme, under the aegis of DG
+Communications Networks, Content and Technology under grant agreement No 101069594. 
+
+|ngizeroentrust| https://nlnet.nl/project/purl2sym/
+
+
+.. |nlnet| image:: https://nlnet.nl/logo/banner.png
+    :target: https://nlnet.nl
+    :height: 50
+    :alt: NLnet foundation logo
+
+.. |ngi| image:: https://ngi.eu/wp-content/uploads/thegem-logos/logo_8269bc6efcf731d34b6385775d76511d_1x.png
+    :target: https://ngi.eu35
+    :height: 50
+    :alt: NGI logo
+
+.. |nexb| image:: https://nexb.com/wp-content/uploads/2022/04/nexB.svg
+    :target: https://nexb.com
+    :height: 30
+    :alt: nexB logo
+
+.. |europa| image:: https://ngi.eu/wp-content/uploads/sites/77/2017/10/bandiera_stelle.png
+    :target: http://ec.europa.eu/index_en.htm
+    :height: 40
+    :alt: Europa logo
+
+.. |aboutcode| image:: https://aboutcode.org/wp-content/uploads/2023/10/AboutCode.svg
+    :target: https://aboutcode.org/
+    :height: 30
+    :alt: AboutCode logo
+
+.. |swiss| image:: https://www.sbfi.admin.ch/sbfi/en/_jcr_content/logo/image.imagespooler.png/1493119032540/logo.png
+    :target: https://www.sbfi.admin.ch/sbfi/en/home/seri/seri.html
+    :height: 40
+    :alt: Swiss logo
+
+.. |dgconnect| image:: https://commission.europa.eu/themes/contrib/oe_theme/dist/ec/images/logo/positive/logo-ec--en.svg
+    :target: https://commission.europa.eu/about-european-commission/departments-and-executive-agencies/communications-networks-content-and-technology_en
+    :height: 40
+    :alt: EC DG Connect logo
+
+.. |ngizerocore| image:: https://nlnet.nl/image/logos/NGI0_tag.svg
+    :target: https://nlnet.nl/core
+    :height: 40
+    :alt: NGI Zero Core Logo
+
+.. |ngizerocommons| image:: https://nlnet.nl/image/logos/NGI0_tag.svg
+    :target: https://nlnet.nl/commonsfund/
+    :height: 40
+    :alt: NGI Zero Commons Logo
+
+.. |ngizeropet| image:: https://nlnet.nl/image/logos/NGI0PET_tag.svg
+    :target: https://nlnet.nl/PET
+    :height: 40
+    :alt: NGI Zero PET logo
+
+.. |ngizeroentrust| image:: https://nlnet.nl/image/logos/NGI0Entrust_tag.svg
+    :target: https://nlnet.nl/entrust
+    :height: 38
+    :alt: NGI Zero Entrust logo
+
+.. |ngiassure| image:: https://nlnet.nl/image/logos/NGIAssure_tag.svg
+    :target: https://nlnet.nl/image/logos/NGIAssure_tag.svg
+    :height: 32
+    :alt: NGI Assure logo
+
+.. |ngidiscovery| image:: https://nlnet.nl/image/logos/NGI0Discovery_tag.svg
+    :target: https://nlnet.nl/discovery/
+    :height: 40
+    :alt: NGI Discovery logo
+
+
+
+
+
+

docs/application-settings.rst

@@ -398,6 +398,12 @@ location on disk using::
 
     SCANCODEIO_NETRC_LOCATION="~/.netrc"
 
+If you are deploying ScanCode.io using Docker and you wish to use a netrc file,
+you can provide it to the Docker container by moving the netrc file to
+``/etc/scancodeio/.netrc`` and then updating the ``.env`` file with the line::
+
+    SCANCODEIO_NETRC_LOCATION="/etc/scancodeio/.netrc"
+
 .. _scancodeio_settings_skopeo_credentials:
 
 SCANCODEIO_SKOPEO_CREDENTIALS

docs/command-line-interface.rst

@@ -264,6 +264,9 @@ Optional arguments:
 Refer to :ref:`Mount projects workspace <mount_projects_workspace_volume>` to access
 your outputs on the host machine when running with Docker.
 
+.. tip:: To specify a CycloneDX spec version (default to latest), use the syntax
+  ``cyclonedx:VERSION`` as format value. For example: ``--format cyclonedx:1.5``.
+
 `$ scanpipe check-compliance --project PROJECT`
 -----------------------------------------------
 

etc/thirdparty/virtualenv.pyz.ABOUT

@@ -1,7 +1,7 @@
 about_resource: virtualenv.pyz
 name: get-virtualenv
-version: 20.26.3
-download_url: https://github.com/pypa/get-virtualenv/raw/20.26.3/public/virtualenv.pyz
+version: 20.27.0
+download_url: https://github.com/pypa/get-virtualenv/raw/20.27.0/public/virtualenv.pyz
 description: virtualenv is a tool to create isolated Python environments.
 homepage_url: https://github.com/pypa/virtualenv
 license_expression: lgpl-2.1-plus AND (bsd-new OR apache-2.0) AND mit AND python AND bsd-new
@@ -10,4 +10,4 @@ copyright: Copyright (c) The Python Software Foundation and others
 redistribute: yes
 attribute: yes
 track_changes: yes
-package_url: pkg:github/pypa/get-virtualenv@20.26.3#public/virtualenv.pyz
+package_url: pkg:github/pypa/get-virtualenv@20.27.0#public/virtualenv.pyz

scanpipe/management/commands/output.py

@@ -25,18 +25,25 @@
 from scanpipe.management.commands import ProjectCommand
 from scanpipe.pipes import output
 
+SUPPORTED_FORMATS = ["json", "csv", "xlsx", "attribution", "spdx", "cyclonedx"]
+
 
 class Command(ProjectCommand):
-    help = "Output project results as JSON, XLSX, SPDX, and CycloneDX."
+    help = "Output project results as JSON, XLSX, Attribution, SPDX, and CycloneDX."
+    print_to_stdout = False
 
     def add_arguments(self, parser):
         super().add_arguments(parser)
         parser.add_argument(
             "--format",
             default=["json"],
             nargs="+",
-            choices=["json", "csv", "xlsx", "spdx", "cyclonedx", "attribution"],
-            help="Specifies the output serialization format for the results.",
+            metavar=f"{{{','.join(SUPPORTED_FORMATS)}}}",
+            help=(
+                "Specifies the output format for the results. "
+                "To specify a CycloneDX spec version (default to latest), use the "
+                'syntax "cyclonedx:VERSION", e.g. "cyclonedx:1.5".'
+            ),
         )
         parser.add_argument(
             "--print",
@@ -46,33 +53,53 @@ def add_arguments(self, parser):
 
     def handle(self, *args, **options):
         super().handle(*args, **options)
-        print_to_stdout = options["print"]
+        self.print_to_stdout = options["print"]
         formats = options["format"]
 
-        if print_to_stdout and len(formats) > 1:
+        if self.print_to_stdout and len(formats) > 1:
             raise CommandError(
                 "--print cannot be used when multiple formats are provided."
             )
 
-        if print_to_stdout and ("xlsx" in formats or "csv" in formats):
+        if self.print_to_stdout and ("xlsx" in formats or "csv" in formats):
             raise CommandError("--print is not compatible with xlsx and csv formats.")
 
-        for format_ in formats:
-            output_function = {
-                "json": output.to_json,
-                "csv": output.to_csv,
-                "xlsx": output.to_xlsx,
-                "spdx": output.to_spdx,
-                "cyclonedx": output.to_cyclonedx,
-                "attribution": output.to_attribution,
-            }.get(format_)
-
-            output_file = output_function(self.project)
-
-            if isinstance(output_file, list):
-                output_file = "\n".join([str(path) for path in output_file])
-
-            if options["print"]:
-                self.stdout.write(output_file.read_text())
-            elif self.verbosity > 0:
-                self.stdout.write(str(output_file), self.style.SUCCESS)
+        for output_format in formats:
+            self.handle_output(output_format)
+
+    def handle_output(self, output_format):
+        output_kwargs = {}
+        if ":" in output_format:
+            output_format, version = output_format.split(":", maxsplit=1)
+            if output_format != "cyclonedx":
+                raise CommandError(
+                    'The ":" version syntax is only supported for the cyclonedx '
+                    "format."
+                )
+            output_kwargs["version"] = version
+
+        output_function = {
+            "json": output.to_json,
+            "csv": output.to_csv,
+            "xlsx": output.to_xlsx,
+            "spdx": output.to_spdx,
+            "cyclonedx": output.to_cyclonedx,
+            "attribution": output.to_attribution,
+        }.get(output_format)
+
+        if not output_function:
+            msg = f"Error: argument --format: invalid choice: '{output_format}'"
+            raise CommandError(msg)
+
+        try:
+            output_file = output_function(self.project, **output_kwargs)
+        except Exception as e:
+            raise CommandError(e)
+
+        if isinstance(output_file, list):
+            output_file = "\n".join([str(path) for path in output_file])
+
+        if self.print_to_stdout:
+            self.stdout.write(output_file.read_text())
+        elif self.verbosity > 0:
+            self.stdout.write(str(output_file), self.style.SUCCESS)

scanpipe/tests/pipes/test_matchcode.py

@@ -140,7 +140,7 @@ def test_scanpipe_pipes_matchcode_poll_run_url_status(
                 "status": run_status.SUCCESS,
             },
         ]
-        return_value = matchcode.poll_run_url_status(run_url)
+        return_value = matchcode.poll_run_url_status(run_url, sleep=0)
         self.assertEqual(True, return_value)
 
         # Failure
@@ -169,7 +169,7 @@ def test_scanpipe_pipes_matchcode_poll_run_url_status(
             },
         ]
         with self.assertRaises(Exception) as context:
-            matchcode.poll_run_url_status(run_url)
+            matchcode.poll_run_url_status(run_url, sleep=0)
         self.assertTrue("failure message" in str(context.exception))
 
         # Stopped
@@ -198,7 +198,7 @@ def test_scanpipe_pipes_matchcode_poll_run_url_status(
             },
         ]
         with self.assertRaises(Exception) as context:
-            matchcode.poll_run_url_status(run_url)
+            matchcode.poll_run_url_status(run_url, sleep=0)
         self.assertTrue("stop message" in str(context.exception))
 
         # Stale
@@ -227,7 +227,7 @@ def test_scanpipe_pipes_matchcode_poll_run_url_status(
             },
         ]
         with self.assertRaises(Exception) as context:
-            matchcode.poll_run_url_status(run_url)
+            matchcode.poll_run_url_status(run_url, sleep=0)
         self.assertTrue("stale message" in str(context.exception))
 
     def test_scanpipe_pipes_matchcode_map_match_results(self):