diff --git a/common.json b/common.json index 02233e65..f2ac2a82 100644 --- a/common.json +++ b/common.json @@ -4,7 +4,7 @@ "Jsonnet files should not include this file directly but use ci/common.jsonnet instead." ], - "mx_version": "7.25.14", + "mx_version": "7.26.0", "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet", "jdks": { diff --git a/src/mx/_impl/mx.py b/src/mx/_impl/mx.py index bb79a4d3..67d94943 100755 --- a/src/mx/_impl/mx.py +++ b/src/mx/_impl/mx.py @@ -3700,7 +3700,7 @@ def download_file_exists(urls): def download_file_with_sha1(name, path, urls, sha1, sha1path, resolve, mustExist, ext=None, sources=False, canSymlink=True): return download_file_with_digest(name, path, urls, sha1, resolve, mustExist, ext, sources, canSymlink) -def download_file_with_digest(name, path, urls, digest, resolve, mustExist, ext=None, sources=False, canSymlink=True): +def download_file_with_digest(name, path, urls, digest, resolve, mustExist, ext=None, sources=False, canSymlink=True, supported_hash_algorithms=None): """ Downloads an entity from a URL in the list `urls` (tried in order) to `path`, checking the digest of the result against `digest` (if not "NOCHECK") @@ -3710,6 +3710,24 @@ def download_file_with_digest(name, path, urls, digest, resolve, mustExist, ext= check_digest = digest and digest.value != 'NOCHECK' canSymlink = canSymlink and can_symlink() + if supported_hash_algorithms is None: + # Legacy usage of download_file_with_digest without enforcing strong hash. + # Check algorithm against the newest allowlist, but warn only for backwards compatibility. + algos = mx_compat.getMxCompatibility(version).get_supported_hash_algorithms() + if digest.name in algos: + warn(f'Deprecated use of dowload_file_with_digest without supported_hash_algorithms argument.\nVerifying download of {name} with strong hash {digest.name}, but this is not enforced.\nConsider bumping mxversion in suite.py to at least 7.27.0 to get rid of this warning.') + else: + warn(f'Verifying download of {name} with unsupported or weak hash algorithm {digest.name}.\nThe recommended algorithms are {algos}.') + elif 'all' in supported_hash_algorithms: + # Concious decision of the programmer that the hash check in this usage of `download_file_with_digest` is not security relevant. + # So we don't need to enforce a strong hash algorithm here. + pass + else: + if not check_digest: + abort(f'Refusing download of {name} without checking digest.') + if digest.name not in supported_hash_algorithms: + abort(f'Refusing download of {name} with unsupported or weak hash algorithm {digest.name}.\nThe recommended algorithms are {supported_hash_algorithms}.') + if len(urls) == 0 and not check_digest: return path @@ -8306,7 +8324,8 @@ def get_urls(self): return self.urls def get_path(self, resolve): - return download_file_with_digest(self.name, self.path, self.urls, self.digest, resolve, not self.optional, ext=self.ext, canSymlink=True) + compat = self.suite.getMxCompatibility() + return download_file_with_digest(self.name, self.path, self.urls, self.digest, resolve, not self.optional, ext=self.ext, canSymlink=True, supported_hash_algorithms=compat.get_supported_hash_algorithms()) def getArchivableResults(self, use_relpath=True, single=False): path = realpath(self.get_path(False)) @@ -8636,7 +8655,8 @@ def is_available(self): def get_path(self, resolve): bootClassPathAgent = hasattr(self, 'bootClassPathAgent') and getattr(self, 'bootClassPathAgent').lower() == 'true' - return download_file_with_digest(self.name, self.path, self.urls, self.digest, resolve, not self.optional, canSymlink=not bootClassPathAgent) + compat = self.suite.getMxCompatibility() + return download_file_with_digest(self.name, self.path, self.urls, self.digest, resolve, not self.optional, canSymlink=not bootClassPathAgent, supported_hash_algorithms=compat.get_supported_hash_algorithms()) def _check_download_needed(self): if not _check_file_with_digest(self.path, self.digest): @@ -8649,7 +8669,8 @@ def _check_download_needed(self): def get_source_path(self, resolve): if self.sourcePath is None: return None - return download_file_with_digest(self.name, self.sourcePath, self.sourceUrls, self.sourceDigest, resolve, len(self.sourceUrls) != 0, sources=True) + compat = self.suite.getMxCompatibility() + return download_file_with_digest(self.name, self.sourcePath, self.sourceUrls, self.sourceDigest, resolve, len(self.sourceUrls) != 0, sources=True, supported_hash_algorithms=compat.get_supported_hash_algorithms()) def classpath_repr(self, resolve=True): path = self.get_path(resolve) @@ -18173,7 +18194,7 @@ def alarm_handler(signum, frame): _CACHE_DIR = get_env('MX_CACHE_DIR', join(dot_mx_dir(), 'cache')) # The version must be updated for every PR (checked in CI) and the comment should reflect the PR's issue -version = VersionSpec("7.26.1") # GR-54784 +version = VersionSpec("7.27.0") # strong digest check _mx_start_datetime = datetime.utcnow() diff --git a/src/mx/_impl/mx_compat.py b/src/mx/_impl/mx_compat.py index 25762860..9dc76539 100644 --- a/src/mx/_impl/mx_compat.py +++ b/src/mx/_impl/mx_compat.py @@ -300,6 +300,13 @@ def gate_run_pyformat(self) -> bool: """ return False + def get_supported_hash_algorithms(self): + """ + Enforce a strong hash on all downloads. + Returns a list of acceptable hash algorithms, or None to accept all (the behavior of old mx versions). + """ + return None + class MxCompatibility520(MxCompatibility500): @staticmethod @@ -765,6 +772,15 @@ def gate_run_pyformat(self) -> bool: return True +class MxCompatibility727(MxCompatibility713): + @staticmethod + def version(): + return mx.VersionSpec("7.27.0") + + def get_supported_hash_algorithms(self): + return ['sha256', 'sha512', 'sha3_384', 'sha3_512'] + + def minVersion(): _ensureCompatLoaded() return list(_versionsMap)[0] diff --git a/src/mx/_impl/mx_fetchjdk.py b/src/mx/_impl/mx_fetchjdk.py index ee8c85e1..00aa8f80 100644 --- a/src/mx/_impl/mx_fetchjdk.py +++ b/src/mx/_impl/mx_fetchjdk.py @@ -104,7 +104,9 @@ def fetch_jdk(args): except Exception as e: #pylint: disable=broad-except mx.abort(f'Error retrieving {sha_url}: {e}') - mx.download_file_with_digest(artifact, archive_location, [url], digest, resolve=True, mustExist=True, sources=False) + # Disable enforcment of strong hash algorithm for this download. The hash comes from the same server as the file, + # so there is no extra security benefit from this check. This is only checking for corrupted downloads. + mx.download_file_with_digest(artifact, archive_location, [url], digest, resolve=True, mustExist=True, sources=False, supported_hash_algorithms=['all']) extractor = mx.Extractor.create(archive_location)