[GR-49282] Add cmakePreset and improve unpacking of packed resources.

PullRequest: mx/1684

ci/common.jsonnet

@@ -25,7 +25,9 @@ local common_json = import "../common.json";
   # ***************
   local variants(name) = [name, name + "Debug", name + "-llvm"],
   # gets the JDK major version from a labsjdk version string (e.g., "ce-21+35-jvmci-23.1-b15" -> 21)
-  local parse_labsjdk_version(version) =
+  local parse_labsjdk_version(jdk) =
+    if jdk.name == "jpg-jdk" then jdk.version else
+    local version = jdk.version;
     assert std.startsWith(version, "ce-") || std.startsWith(version, "ee-") : "Unsupported labsjdk version: " + version;
     local number_prefix(str) =
       if std.length(str) == 0 || std.length(std.findSubstr(str[0], "0123456789")) == 0 then
@@ -35,6 +37,12 @@ local common_json = import "../common.json";
       ;
     std.parseInt(number_prefix(version[3:]))
     ,
+  # gets the build_id from a labsjdk version string (e.g., "ce-21+35-jvmci-23.1-b15" -> 21)
+  local get_labsjdk_build_id(jdk) =
+    local _parts = std.split(jdk.version, "-");
+    local _version_build_id = std.split(_parts[1], "+");
+    _version_build_id[1]
+    ,
   local jdks_data = {
     oraclejdk11: jdk_base + common_json.jdks["oraclejdk11"] + { jdk_version:: 11 },
   } + {
@@ -50,10 +58,18 @@ local common_json = import "../common.json";
     [name]: jdk_base + common_json.jdks[name] + { jdk_version:: 21 }
     for name in ["oraclejdk21"] + variants("labsjdk-ce-21") + variants("labsjdk-ee-21")
   } + {
-    [name]: jdk_base + common_json.jdks[name] + { jdk_version:: parse_labsjdk_version(self.version), jdk_name:: "jdk-latest"}
-    for name in variants("labsjdk-ce-latest") + variants("labsjdk-ee-latest")
+    [name]: jdk_base + common_json.jdks[name] + { jdk_version:: parse_labsjdk_version(self), jdk_name:: "jdk-latest"}
+    for name in ["oraclejdk-latest"] + variants("labsjdk-ce-latest") + variants("labsjdk-ee-latest")
   },
   assert std.assertEqual(std.objectFields(common_json.jdks), std.objectFields(jdks_data)),
+  # Verify oraclejdk-latest and labsjdk-ee-latest versions match
+  assert
+    local _labsjdk = common_json.jdks["labsjdk-ee-latest"];
+    local _oraclejdk = common_json.jdks["oraclejdk-latest"];
+    local _ov = "ee-%s+%s" % [_oraclejdk.version, _oraclejdk.build_id];
+    local _lv = _labsjdk.version;
+    assert std.startsWith(_lv, _ov) : "update oraclejdk-latest to match labsjdk-ee-latest: %s+%s vs %s" % [_oraclejdk.version, _oraclejdk.build_id, _labsjdk.version];
+    true,
 
   # The raw jdk data, the same as common_json.jdks + { jdk_version:: }
   jdks_data: jdks_data,
@@ -78,6 +94,8 @@ local common_json = import "../common.json";
 
     labsjdkLatestCE: self["labsjdk-ce-21"],
     labsjdkLatestEE: self["labsjdk-ee-21"],
+
+    oraclejdkLatest: self["oraclejdk-latest"],
   },
 
   # The devkits versions reflect those used to build the JVMCI JDKs (e.g., see devkit_platform_revisions in <jdk>/make/conf/jib-profiles.js)

common.json

@@ -42,12 +42,13 @@
     "labsjdk-ee-21Debug": {"name": "labsjdk",   "version": "ee-21+35-jvmci-23.1-b15-debug", "platformspecific": true },
     "labsjdk-ee-21-llvm": {"name": "labsjdk",   "version": "ee-21+35-jvmci-23.1-b15-sulong", "platformspecific": true },
 
-    "labsjdk-ce-latest":      {"name": "labsjdk",   "version": "ce-22+15-jvmci-b01", "platformspecific": true },
-    "labsjdk-ce-latestDebug": {"name": "labsjdk",   "version": "ce-22+15-jvmci-b01-debug", "platformspecific": true },
-    "labsjdk-ce-latest-llvm": {"name": "labsjdk",   "version": "ce-22+15-jvmci-b01-sulong", "platformspecific": true },
-    "labsjdk-ee-latest":      {"name": "labsjdk",   "version": "ee-22+15-jvmci-b01", "platformspecific": true },
-    "labsjdk-ee-latestDebug": {"name": "labsjdk",   "version": "ee-22+15-jvmci-b01-debug", "platformspecific": true },
-    "labsjdk-ee-latest-llvm": {"name": "labsjdk",   "version": "ee-22+15-jvmci-b01-sulong", "platformspecific": true }
+    "oraclejdk-latest":       {"name": "jpg-jdk",   "version": "22",      "build_id": "16", "release": true, "platformspecific": true, "extrabundles": ["static-libs"]},
+    "labsjdk-ce-latest":      {"name": "labsjdk",   "version": "ce-22+16-jvmci-b01", "platformspecific": true },
+    "labsjdk-ce-latestDebug": {"name": "labsjdk",   "version": "ce-22+16-jvmci-b01-debug", "platformspecific": true },
+    "labsjdk-ce-latest-llvm": {"name": "labsjdk",   "version": "ce-22+16-jvmci-b01-sulong", "platformspecific": true },
+    "labsjdk-ee-latest":      {"name": "labsjdk",   "version": "ee-22+16-jvmci-b01", "platformspecific": true },
+    "labsjdk-ee-latestDebug": {"name": "labsjdk",   "version": "ee-22+16-jvmci-b01-debug", "platformspecific": true },
+    "labsjdk-ee-latest-llvm": {"name": "labsjdk",   "version": "ee-22+16-jvmci-b01-sulong", "platformspecific": true }
   },
 
   "eclipse": {

mx.py

@@ -631,6 +631,7 @@ def __init__(self, parents=None):
         self.add_argument('--arch', action='store', dest='arch', help='force use of the specified architecture')
         self.add_argument('--multi-platform-layout-directories', action='store', help="Causes platform-dependent layout dir distribution to contain the union of the files from their declared platforms. "
                                 "Can be set to 'all' or to a comma-separated list of platforms.")
+        self.add_argument('--extra-cmake-arg', action='append', metavar='<arg>')
 
         if not is_windows():
             # Time outs are (currently) implemented with Unix specific functionality
@@ -8705,7 +8706,7 @@ def _is_sane_name(m):
 
     @staticmethod
     def create(src):
-        if src.endswith(".tar") or src.endswith(".tar.gz") or src.endswith(".tgz"):
+        if src.endswith(".tar") or src.endswith(".tar.gz") or src.endswith(".tgz") or src.endswith(".tar.bz2") or src.endswith(".tbz2"):
             return TarExtractor(src)
         if src.endswith(".zip") or src.endswith(".jar"):
             return ZipExtractor(src)
@@ -18830,7 +18831,7 @@ def alarm_handler(signum, frame):
         abort(1, killsig=signal.SIGINT)
 
 # The version must be updated for every PR (checked in CI) and the comment should reflect the PR's issue
-version = VersionSpec("6.50.3")  # build graph
+version = VersionSpec("6.51.0")  # GR-49282 mx cmake features and tar.bz2 support
 
 _mx_start_datetime = datetime.utcnow()
 _last_timestamp = _mx_start_datetime

mx_cmake.py

@@ -44,6 +44,8 @@ class CMakeNinjaProject(mx_native.NinjaProject):  # pylint: disable=too-many-anc
         cmakeConfig: dict, optional
             Additional arguments passed to CMake in the form '-D{key}={value}'.
             Path substitution is performed on the values.
+        cmakePreset: str, optional
+            The CMake preset to use.
         cmakeSubdir: str, optional
             Subdirectory of sourceDir that contains the CMakeLists.txt. If omitted, CMakeLists.txt is assumed to be in sourceDir.
         symlinkSource: bool, optional
@@ -73,6 +75,7 @@ def __init__(self, suite, name, deps, workingSets, subDir, ninja_targets=None, n
         super(CMakeNinjaProject, self).__init__(suite, name, subDir, [srcDir], deps, workingSets, d, results=results, output=output, **args)
         self.silent = not cmake_show_warnings
         self._cmake_config_raw = args.pop('cmakeConfig', {})
+        self._cmake_preset = args.pop('cmakePreset', None)
         self.buildDependencies += extraBuildDeps
         if self._cmake_toolchain:
             self.buildDependencies += [self._cmake_toolchain]
@@ -117,7 +120,10 @@ def _toolchain_config(self):
         return {"CMAKE_TOOLCHAIN_FILE": mx.join(tc.get_output(), 'cmake', 'toolchain.cmake') for tc in [self._cmake_toolchain] if tc}
 
     def cmake_config(self):
-        return [CMakeNinjaProject.config_entry(k, v) for k, v in sorted({**self._cmake_config_raw, **self._toolchain_config()}.items())]
+        cfgArgs = [CMakeNinjaProject.config_entry(k, v) for k, v in sorted({**self._cmake_config_raw, **self._toolchain_config()}.items())]
+        if self._cmake_preset:
+            cfgArgs = ["--preset", self._cmake_preset] + cfgArgs
+        return cfgArgs
 
     def sourceDir(self, create=False):
         src_dir = self.source_dirs()[0]
@@ -163,6 +169,8 @@ def generate_manifest(self, output_dir, filename, extra_cmake_config=None):
         cmake_config = self.cmake_config()
         if extra_cmake_config:
             cmake_config.extend(extra_cmake_config)
+        if mx._opts.extra_cmake_arg:
+            cmake_config.extend(mx._opts.extra_cmake_arg)
 
         # explicitly set ninja executable if not on path
         cmake_make_program = 'CMAKE_MAKE_PROGRAM'