Skip to content

Commit

Permalink
[GR-50256] Add an option to ignore errors on TempDir.__exit__().
Browse files Browse the repository at this point in the history 
PullRequest: mx/1717
  • Loading branch information
@ansalond
ansalond committed Nov 21, 2023
2 parents 999926a + 993dbfc commit 1b21203
Show file tree
Hide file tree
Showing 4 changed files with 135 additions and 105 deletions.
215 changes: 122 additions & 93 deletions ci/common.jsonnet
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ local common_json = import "../common.json";
"windows-jdk19": { packages+: { "devkit:VS2022-17.1.0+1": "==0" }},
"windows-jdk20": { packages+: { "devkit:VS2022-17.1.0+1": "==0" }},
"windows-jdk21": { packages+: { "devkit:VS2022-17.1.0+1": "==1" }},
"windows-jdk-latest": { packages+: { "devkit:VS2022-17.1.0+1": "==1" }},
"windows-jdk-latest": { packages+: { "devkit:VS2022-17.6.5+1": "==0" }},
"windows-jdkLatest": self["windows-jdk-latest"],
"linux-jdk17": { packages+: { "devkit:gcc11.2.0-OL6.4+1": "==0" }},
"linux-jdk19": { packages+: { "devkit:gcc11.2.0-OL6.4+1": "==0" }},
Expand All @@ -118,6 +118,40 @@ local common_json = import "../common.json";
# Dependencies
# ************
deps: {
# These dependencies are included in Build GraalVM platforms, but not in bare platforms

mx: {
environment+: {
MX_PYTHON: "python3.8",
},
packages+: {
python3: "==3.8.10",
"pip:ninja_syntax": common_json.pip.ninja_syntax,
mx: common_json.mx_version,
},
python_version: "3", # To use the correct virtualenv
},

common_catch_files: {
catch_files+: [
# Keep in sync with jdk.graal.compiler.debug.StandardPathUtilitiesProvider#DIAGNOSTIC_OUTPUT_DIRECTORY_MESSAGE_REGEXP
"Graal diagnostic output saved in '(?P<filename>[^']+)'",
# Keep in sync with jdk.graal.compiler.debug.DebugContext#DUMP_FILE_MESSAGE_REGEXP
"Dumping debug output to '(?P<filename>[^']+)'",
# Keep in sync with com.oracle.svm.hosted.NativeImageOptions#DEFAULT_ERROR_FILE_NAME
" (?P<filename>.+/svm_err_b_\\d+T\\d+\\.\\d+_pid\\d+\\.md)",
],
},

common_env: {
environment+: {
# Enforce experimental option checking in CI (GR-47922)
NATIVE_IMAGE_EXPERIMENTAL_OPTIONS_ARE_FATAL: "true",
},
},

# These dependencies are not included by default in any platform object

eclipse: {
downloads+: {
ECLIPSE: {
Expand Down Expand Up @@ -238,46 +272,7 @@ local common_json = import "../common.json";
}
},

# Hardware definitions and common fields
# **************************************
# Note that only platforms (os-arch) are exposed (not os and arch separately),
# because this is the simplest way to ensure correct usage and dependencies (e.g. ol7 in linux_amd64).
#
# To add extra "common" fields for your CI:
# * If you already have platforms objects, you could extend them like:
# linux_amd64: common.linux_amd64 + self.my_common,
# * Otherwise, just include your common object as well as one of the os-arch objects below in each job:
# { name: "myjob" } + common.linux_amd64 + self.my_common + ...
#
# This also means self.my_common should no longer include mx, etc as it is already included by the os-arch objects.
local mx = {
environment+: {
MX_PYTHON: "python3.8",
},
packages+: {
python3: "==3.8.10",
"pip:ninja_syntax": common_json.pip.ninja_syntax,
mx: common_json.mx_version,
},
python_version: "3", # To use the correct virtualenv
},

local common = mx + {
catch_files+: [
# Keep in sync with jdk.graal.compiler.debug.StandardPathUtilitiesProvider#DIAGNOSTIC_OUTPUT_DIRECTORY_MESSAGE_REGEXP
"Graal diagnostic output saved in '(?P<filename>[^']+)'",
# Keep in sync with jdk.graal.compiler.debug.DebugContext#DUMP_FILE_MESSAGE_REGEXP
"Dumping debug output to '(?P<filename>[^']+)'",
# Keep in sync with com.oracle.svm.hosted.NativeImageOptions#DEFAULT_ERROR_FILE_NAME
" (?P<filename>.+/svm_err_b_\\d+T\\d+\\.\\d+_pid\\d+\\.md)",
],
environment+: {
# Enforce experimental option checking in CI (GR-47922)
NATIVE_IMAGE_EXPERIMENTAL_OPTIONS_ARE_FATAL: "true",
},
},

// OS specific file handling
# OS specific file handling
os_utils:: {
local lib_format = {
"windows": "%s.dll",
Expand All @@ -295,36 +290,84 @@ local common_json = import "../common.json";
os_lib(name):: lib_format[self.os] % name,
},

local ol7 = {
docker+: {
image: "buildslave_ol7",
mount_modules: true,
},
# Utils
disable_proxies: {
setup+: [
["unset", "HTTP_PROXY", "HTTPS_PROXY", "FTP_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "ftp_proxy", "no_proxy"],
],
},
local ol8 = {
docker+: {
image: "buildslave_ol8",
mount_modules: true,

# Hardware definitions and common fields
# **************************************
# Note that only platforms (os-arch) are exposed (not os and arch separately),
# because this is the simplest way to ensure correct usage and dependencies (e.g. ol7 in linux_amd64).
#
# To add extra "common" fields for your CI:
# * If you already have platforms objects, you could extend them like:
# linux_amd64: common.linux_amd64 + self.my_common,
# * Otherwise, just include your common object as well as one of the os-arch objects below in each job:
# { name: "myjob" } + common.linux_amd64 + self.my_common + ...
#
# This also means self.my_common should no longer include mx, etc as it is already included by the os-arch objects.

# Bare platforms, just the bare minimum and nothing else. Also see Build GraalVM platforms below.
bare:: {
local ol7 = {
docker+: {
image: "buildslave_ol7",
mount_modules: true,
},
},
},
local ol9 = {
docker+: {
image: "buildslave_ol9",
mount_modules: true,
local ol8 = {
docker+: {
image: "buildslave_ol8",
mount_modules: true,
},
},
},
local ubuntu22 = {
docker+: {
image: "buildslave_ubuntu22",
mount_modules: true,
local ol9 = {
docker+: {
image: "buildslave_ol9",
mount_modules: true,
},
},
local ubuntu22 = {
docker+: {
image: "buildslave_ubuntu22",
mount_modules: true,
},
},

local linux = { os:: "linux", capabilities+: [self.os] },
local darwin = { os:: "darwin", capabilities+: [self.os] },
local windows = { os:: "windows", capabilities+: [self.os] },

local amd64 = { arch:: "amd64", capabilities+: [self.arch] },
local aarch64 = { arch:: "aarch64", capabilities+: [self.arch] },

local ol_distro = { os_distro:: "ol" },

linux_amd64: self.linux_amd64_ol7,
linux_amd64_ol7: linux + amd64 + ol7 + ol_distro,
linux_amd64_ol8: linux + amd64 + ol8 + ol_distro,
linux_amd64_ol9: linux + amd64 + ol9 + ol_distro,

linux_aarch64: self.linux_aarch64_ol7,
linux_aarch64_ol7: linux + aarch64 + ol7 + ol_distro,
linux_aarch64_ol8: linux + aarch64 + ol8 + ol_distro,
linux_aarch64_ol9: linux + aarch64 + ol9 + ol_distro,

linux_amd64_ubuntu: linux + amd64 + ubuntu22 + { os_distro:: "ubuntu" },

darwin_amd64: darwin + amd64,
darwin_aarch64: darwin + aarch64,

windows_amd64: windows + amd64,
windows_server_2016_amd64: windows + amd64 + { capabilities+: ["windows_server_2016"] },
},
local deps_linux = {
},
local deps_darwin = {
},
local deps_windows = {
},

# Build GraalVM platforms, they include the dependencies listed in `local common =` just below.
# They also include a devtoolset on Oracle Linux, to use the same system toolchain for all builds.
local common = self.deps.mx + self.deps.common_catch_files + self.deps.common_env,

local ol_devtoolset = {
packages+: (if self.arch == "aarch64" then {
Expand All @@ -334,35 +377,21 @@ local common_json = import "../common.json";
}),
},

local linux = deps_linux + common + { os:: "linux", capabilities+: [self.os] },
local darwin = deps_darwin + common + { os:: "darwin", capabilities+: [self.os] },
local windows = deps_windows + common + { os:: "windows", capabilities+: [self.os] },
local windows_server_2016 = windows + { capabilities+: ["windows_server_2016"] },

local amd64 = { arch:: "amd64", capabilities+: [self.arch] },
local aarch64 = { arch:: "aarch64", capabilities+: [self.arch] },
# Always include the devtoolset on Oracle Linux to use the same system toolchain for all builds
local ol_distro = { os_distro:: "ol" } + ol_devtoolset,
linux_amd64: self.linux_amd64_ol7,
linux_amd64_ol7: self.bare.linux_amd64_ol7 + common + ol_devtoolset,
linux_amd64_ol8: self.bare.linux_amd64_ol8 + common + ol_devtoolset,
linux_amd64_ol9: self.bare.linux_amd64_ol9 + common + ol_devtoolset,

linux_amd64: linux + amd64 + ol7 + ol_distro,
linux_amd64_ol8: linux + amd64 + ol8 + ol_distro,
linux_amd64_ol9: linux + amd64 + ol9 + ol_distro,
linux_aarch64: linux + aarch64 + ol_distro,
linux_aarch64_ol8: linux + aarch64 + ol8 + ol_distro,
linux_aarch64_ol9: linux + aarch64 + ol9 + ol_distro,
linux_aarch64: self.linux_aarch64_ol7,
linux_aarch64_ol7: self.bare.linux_aarch64_ol7 + common + ol_devtoolset,
linux_aarch64_ol8: self.bare.linux_aarch64_ol8 + common + ol_devtoolset,
linux_aarch64_ol9: self.bare.linux_aarch64_ol9 + common + ol_devtoolset,

linux_amd64_ubuntu: linux + amd64 + ubuntu22 + { os_distro:: "ubuntu" },
linux_amd64_ubuntu: self.bare.linux_amd64_ubuntu + common,

darwin_amd64: darwin + amd64,
darwin_aarch64: darwin + aarch64,
darwin_amd64: self.bare.darwin_amd64 + common,
darwin_aarch64: self.bare.darwin_aarch64 + common,

windows_amd64: windows + amd64,
windows_server_2016_amd64: windows_server_2016 + amd64,

# Utils
disable_proxies: {
setup+: [
["unset", "HTTP_PROXY", "HTTPS_PROXY", "FTP_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "ftp_proxy", "no_proxy"],
],
},
windows_amd64: self.bare.windows_amd64 + common,
windows_server_2016_amd64: self.bare.windows_server_2016_amd64 + common,
}
16 changes: 8 additions & 8 deletions common.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
],

"mx_version": "7.0.3",
"mx_version": "7.2.2",

"COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
"jdks": {
Expand Down Expand Up @@ -42,13 +42,13 @@
"labsjdk-ee-21Debug": {"name": "labsjdk", "version": "ee-21.0.1+11-jvmci-23.1-b22-debug", "platformspecific": true },
"labsjdk-ee-21-llvm": {"name": "labsjdk", "version": "ee-21.0.1+11-jvmci-23.1-b22-sulong", "platformspecific": true },

"oraclejdk-latest": {"name": "jpg-jdk", "version": "22", "build_id": "22", "release": true, "platformspecific": true, "extrabundles": ["static-libs"]},
"labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-22+22-jvmci-b01", "platformspecific": true },
"labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-22+22-jvmci-b01-debug", "platformspecific": true },
"labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-22+22-jvmci-b01-sulong", "platformspecific": true },
"labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-22+22-jvmci-b01", "platformspecific": true },
"labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-22+22-jvmci-b01-debug", "platformspecific": true },
"labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-22+22-jvmci-b01-sulong", "platformspecific": true }
"oraclejdk-latest": {"name": "jpg-jdk", "version": "22", "build_id": "23", "release": true, "platformspecific": true, "extrabundles": ["static-libs"]},
"labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-22+23-jvmci-b01", "platformspecific": true },
"labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-22+23-jvmci-b01-debug", "platformspecific": true },
"labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-22+23-jvmci-b01-sulong", "platformspecific": true },
"labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-22+23-jvmci-b01", "platformspecific": true },
"labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-22+23-jvmci-b01-debug", "platformspecific": true },
"labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-22+23-jvmci-b01-sulong", "platformspecific": true }
},

"eclipse": {
Expand Down
7 changes: 4 additions & 3 deletions src/mx/_impl/mx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16039,15 +16039,16 @@ def __exit__(self, exc_type, exc_value, traceback):


class TempDir(object):
def __init__(self, parent_dir=None):
def __init__(self, parent_dir=None, ignore_errors=False):
self.parent_dir = parent_dir
self.ignore_errors = ignore_errors

def __enter__(self):
self.tmp_dir = mkdtemp(dir=self.parent_dir)
return self.tmp_dir

def __exit__(self, exc_type, exc_value, traceback):
shutil.rmtree(self.tmp_dir)
rmtree(self.tmp_dir, ignore_errors=self.ignore_errors)


class TempDirCwd(TempDir):
Expand Down Expand Up @@ -19249,7 +19250,7 @@ def alarm_handler(signum, frame):
abort(1, killsig=signal.SIGINT)

# The version must be updated for every PR (checked in CI) and the comment should reflect the PR's issue
version = VersionSpec("7.3.0") # GR-50010 Support qualifying jdk-ids in mx fetch-jdk jdk-binaries.json
version = VersionSpec("7.3.1") # GR-50256 ignore_errors in TempDir.__exit__()

_mx_start_datetime = datetime.utcnow()
_last_timestamp = _mx_start_datetime
Expand Down
2 changes: 1 addition & 1 deletion src/mx/_impl/mx_downstream.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ def get_suite(name):
candidates = '\n- '.join(downstream_branch_candidates)
mx.log(f"The most recent merge performed by the CI on the active branch of the upstream repository is at revision '{upstream_commit}', which is part of the following branches:\n- {candidates}")
if not _checkout_upstream_revision(upstream_commit, downstream_branch_candidates, upstream_suite, downstream_suite):
mx.abort(f"Cannot find a revision of '{downstream_suite.vc_dir}' that imports revision '{upstream_commit}' of '{upstream_suite.name}")
mx.abort(f"Cannot find a revision of '{downstream_suite.vc_dir}' that imports revision '{upstream_commit}' of '{upstream_suite.name}'")


def _run_git_cmd(vc_dir, cmd, regex=None, abortOnError=True):
Expand Down

0 comments on commit 1b21203

Please sign in to comment.