[GR-54741] Enforce strong hash algorithms on library and source downl…

…oads.

PullRequest: mx/1810

common.json

@@ -4,7 +4,7 @@
     "Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
   ],
 
-  "mx_version": "7.25.14",
+  "mx_version": "7.26.0",
 
   "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
   "jdks": {

src/mx/_impl/mx.py

@@ -3700,7 +3700,7 @@ def download_file_exists(urls):
 def download_file_with_sha1(name, path, urls, sha1, sha1path, resolve, mustExist, ext=None, sources=False, canSymlink=True):
     return download_file_with_digest(name, path, urls, sha1, resolve, mustExist, ext, sources, canSymlink)
 
-def download_file_with_digest(name, path, urls, digest, resolve, mustExist, ext=None, sources=False, canSymlink=True):
+def download_file_with_digest(name, path, urls, digest, resolve, mustExist, ext=None, sources=False, canSymlink=True, supported_hash_algorithms=None):
     """
     Downloads an entity from a URL in the list `urls` (tried in order) to `path`,
     checking the digest of the result against `digest` (if not "NOCHECK")
@@ -3710,6 +3710,24 @@ def download_file_with_digest(name, path, urls, digest, resolve, mustExist, ext=
     check_digest = digest and digest.value != 'NOCHECK'
     canSymlink = canSymlink and can_symlink()
 
+    if supported_hash_algorithms is None:
+        # Legacy usage of download_file_with_digest without enforcing strong hash.
+        # Check algorithm against the newest allowlist, but warn only for backwards compatibility.
+        algos = mx_compat.getMxCompatibility(version).get_supported_hash_algorithms()
+        if digest.name in algos:
+            warn(f'Deprecated use of dowload_file_with_digest without supported_hash_algorithms argument.\nVerifying download of {name} with strong hash {digest.name}, but this is not enforced.\nConsider bumping mxversion in suite.py to at least 7.27.0 to get rid of this warning.')
+        else:
+            warn(f'Verifying download of {name} with unsupported or weak hash algorithm {digest.name}.\nThe recommended algorithms are {algos}.')
+    elif 'all' in supported_hash_algorithms:
+        # Concious decision of the programmer that the hash check in this usage of `download_file_with_digest` is not security relevant.
+        # So we don't need to enforce a strong hash algorithm here.
+        pass
+    else:
+        if not check_digest:
+            abort(f'Refusing download of {name} without checking digest.')
+        if digest.name not in supported_hash_algorithms:
+            abort(f'Refusing download of {name} with unsupported or weak hash algorithm {digest.name}.\nThe recommended algorithms are {supported_hash_algorithms}.')
+
     if len(urls) == 0 and not check_digest:
         return path
 
@@ -8306,7 +8324,8 @@ def get_urls(self):
         return self.urls
 
     def get_path(self, resolve):
-        return download_file_with_digest(self.name, self.path, self.urls, self.digest, resolve, not self.optional, ext=self.ext, canSymlink=True)
+        compat = self.suite.getMxCompatibility()
+        return download_file_with_digest(self.name, self.path, self.urls, self.digest, resolve, not self.optional, ext=self.ext, canSymlink=True, supported_hash_algorithms=compat.get_supported_hash_algorithms())
 
     def getArchivableResults(self, use_relpath=True, single=False):
         path = realpath(self.get_path(False))
@@ -8636,7 +8655,8 @@ def is_available(self):
 
     def get_path(self, resolve):
         bootClassPathAgent = hasattr(self, 'bootClassPathAgent') and getattr(self, 'bootClassPathAgent').lower() == 'true'
-        return download_file_with_digest(self.name, self.path, self.urls, self.digest, resolve, not self.optional, canSymlink=not bootClassPathAgent)
+        compat = self.suite.getMxCompatibility()
+        return download_file_with_digest(self.name, self.path, self.urls, self.digest, resolve, not self.optional, canSymlink=not bootClassPathAgent, supported_hash_algorithms=compat.get_supported_hash_algorithms())
 
     def _check_download_needed(self):
         if not _check_file_with_digest(self.path, self.digest):
@@ -8649,7 +8669,8 @@ def _check_download_needed(self):
     def get_source_path(self, resolve):
         if self.sourcePath is None:
             return None
-        return download_file_with_digest(self.name, self.sourcePath, self.sourceUrls, self.sourceDigest, resolve, len(self.sourceUrls) != 0, sources=True)
+        compat = self.suite.getMxCompatibility()
+        return download_file_with_digest(self.name, self.sourcePath, self.sourceUrls, self.sourceDigest, resolve, len(self.sourceUrls) != 0, sources=True, supported_hash_algorithms=compat.get_supported_hash_algorithms())
 
     def classpath_repr(self, resolve=True):
         path = self.get_path(resolve)
@@ -18173,7 +18194,7 @@ def alarm_handler(signum, frame):
 _CACHE_DIR = get_env('MX_CACHE_DIR', join(dot_mx_dir(), 'cache'))
 
 # The version must be updated for every PR (checked in CI) and the comment should reflect the PR's issue
-version = VersionSpec("7.26.1")  # GR-54784
+version = VersionSpec("7.27.0")  # strong digest check
 
 _mx_start_datetime = datetime.utcnow()
 

src/mx/_impl/mx_compat.py

@@ -300,6 +300,13 @@ def gate_run_pyformat(self) -> bool:
         """
         return False
 
+    def get_supported_hash_algorithms(self):
+        """
+        Enforce a strong hash on all downloads.
+        Returns a list of acceptable hash algorithms, or None to accept all (the behavior of old mx versions).
+        """
+        return None
+
 
 class MxCompatibility520(MxCompatibility500):
     @staticmethod
@@ -765,6 +772,15 @@ def gate_run_pyformat(self) -> bool:
         return True
 
 
+class MxCompatibility727(MxCompatibility713):
+    @staticmethod
+    def version():
+        return mx.VersionSpec("7.27.0")
+
+    def get_supported_hash_algorithms(self):
+        return ['sha256', 'sha512', 'sha3_384', 'sha3_512']
+
+
 def minVersion():
     _ensureCompatLoaded()
     return list(_versionsMap)[0]

src/mx/_impl/mx_fetchjdk.py

@@ -104,7 +104,9 @@ def fetch_jdk(args):
                 except Exception as e: #pylint: disable=broad-except
                     mx.abort(f'Error retrieving {sha_url}: {e}')
 
-                mx.download_file_with_digest(artifact, archive_location, [url], digest, resolve=True, mustExist=True, sources=False)
+                # Disable enforcment of strong hash algorithm for this download. The hash comes from the same server as the file,
+                # so there is no extra security benefit from this check. This is only checking for corrupted downloads.
+                mx.download_file_with_digest(artifact, archive_location, [url], digest, resolve=True, mustExist=True, sources=False, supported_hash_algorithms=['all'])
 
                 extractor = mx.Extractor.create(archive_location)