From e07ac51615c39cf70aead1053fd9de9680ff390f Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Fri, 24 May 2024 17:12:36 +0200 Subject: [PATCH 01/20] adjust OpenSSL version regex and OpenSSL header version check in OpenSSL wrapper easyblock --- easybuild/easyblocks/o/openssl_wrapper.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 247a018d62..53cb27aa2c 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -101,7 +101,7 @@ def __init__(self, *args, **kwargs): # Regex pattern to find version strings in OpenSSL libraries and headers full_version_regex = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+[a-z]?') - openssl_version_regex = re.compile(r'OpenSSL\s+([0-9]+\.[0-9]+(\.[0-9]+[a-z]?)*)', re.M) + openssl_version_regex = re.compile(r'(?i)OpenSSL[\s_]+([0-9]+\.[0-9]+(\.[0-9]+[a-z]?)*)', re.M) # Libraries packaged in OpenSSL openssl_libs = ['libssl', 'libcrypto'] @@ -259,13 +259,23 @@ def __init__(self, *args, **kwargs): err_msg = "System OpenSSL header '%s' does not contain any recognizable version string" raise EasyBuildError(err_msg, opensslv_path) - if header_version == self.system_ssl['version']: + header_version_major = header_version.split('.')[0] + system_ssl_version_major = self.system_ssl['version'].split('.')[0] + + # Compare only major version of header for OpenSSL 3+ + # This is needed on Debian-based distros + if LooseVersion(self.version) >= LooseVersion('3'): + header_found = (header_version_major == system_ssl_version_major) + else: + header_found = (header_version == self.system_ssl['version']) + + if header_found: self.system_ssl['include'] = include_dir info_msg = "Found OpenSSL headers v%s in host system: %s" self.log.info(info_msg, header_version, self.system_ssl['include']) break else: - dbg_msg = "System OpenSSL header version '%s' doesn not match library version '%s'" + dbg_msg = "System OpenSSL header version '%s' does not match library version '%s'" self.log.debug(dbg_msg, header_version, self.system_ssl['version']) else: self.log.info("System OpenSSL header file %s not found", opensslv_path) From e67ce87b8b13dd74fcaa423832bbf98450753976 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 28 May 2024 00:08:37 +0200 Subject: [PATCH 02/20] add non-standard include path used by RHEL-based systems to install OpenSSL v3 headers alongside OpenSSL v1.1 --- easybuild/easyblocks/o/openssl_wrapper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 53cb27aa2c..85f3a343b3 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -242,6 +242,10 @@ def __init__(self, *args, **kwargs): # but version 1.1 can be installed in 'include/openssl11/openssl' as well, for example in CentOS 7 # prefer 'include/openssl' as long as the version of headers matches ssl_include_subdirs.append(os.path.join('openssl11', self.name.lower())) + elif LooseVersion(self.version) >= LooseVersion('3'): + # but version 3.x can be installed in 'include/openssl3/openssl' as well, for example in RHEL 8 derivatives + # prefer 'include/openssl' as long as the version of headers matches + ssl_include_subdirs.append(os.path.join('openssl3', self.name.lower())) ssl_include_dirs = [os.path.join(incd, subd) for incd in sys_include_dirs for subd in ssl_include_subdirs] ssl_include_dirs = [include for include in ssl_include_dirs if os.path.isdir(include)] From a8fffcb6bf02220252248ee36f2e5e04d8ead544 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 28 May 2024 02:23:01 +0200 Subject: [PATCH 03/20] add non-standard binary name used by RHEL-based systems that install OpenSSL v3 alongside OpenSSL v1.1 --- easybuild/easyblocks/o/openssl_wrapper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 85f3a343b3..eed4ec76cf 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -294,6 +294,9 @@ def __init__(self, *args, **kwargs): if self.majmin_version == '1.1': # prefer 'openssl11' over 'openssl' with v1.1 self.system_ssl['bin'] = which('openssl11') + elif LooseVersion(self.version) >= LooseVersion('3'): + # prefer 'openssl3' over 'openssl' with v3.x + self.system_ssl['bin'] = which('openssl3') if not self.system_ssl['bin']: self.system_ssl['bin'] = which('openssl') From bf408fad7736fc349739501496011f5bff193005 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 28 May 2024 01:38:39 +0200 Subject: [PATCH 04/20] consider all versions found from strings in OpenSSL libraries and pick the highest one --- easybuild/easyblocks/o/openssl_wrapper.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index eed4ec76cf..8416f16ca4 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -180,8 +180,14 @@ def __init__(self, *args, **kwargs): # filename lacks the full version, fallback to version strings within the library solib_strings = read_file(system_solib, mode="rb").decode('utf-8', 'replace') try: - openssl_version = openssl_version_regex.search(solib_strings).group(1) - except AttributeError: + openssl_support_versions = openssl_version_regex.findall(solib_strings) + openssl_support_versions.sort() + dbg_msg = "System OpenSSL library '%s' supports versions: %s" + dbg_msg_support_versions = ', '.join([''.join(v) for v in openssl_support_versions]) + self.log.debug(dbg_msg, system_solib, dbg_msg_support_versions) + # pick highest supported version + openssl_version = openssl_support_versions[-1][0] + except IndexError: dbg_msg = "Could not detect the full version of system OpenSSL library: %s" self.log.debug(dbg_msg, system_solib) # check that system version fulfills requirements From a0d42f342d89e7d697e60927b4fb990816b463db Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 28 May 2024 02:00:10 +0200 Subject: [PATCH 05/20] simplify code checking OpenSSL system libraries --- easybuild/easyblocks/o/openssl_wrapper.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 8416f16ca4..2438a79e04 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -168,11 +168,13 @@ def __init__(self, *args, **kwargs): # Check the system libraries of OpenSSL # Find library file and compare its version string + target_system_ssl = {} for idx, solibs in enumerate(system_versioned_libs): + target_system_ssl['version'] = '0' + target_system_ssl['libs'] = [] for solib in solibs: system_solib = find_library_path(solib) if system_solib: - openssl_version = '0' # get version of system library filename try: openssl_version = full_version_regex.search(os.path.realpath(system_solib)).group(0) @@ -194,22 +196,21 @@ def __init__(self, *args, **kwargs): if LooseVersion(openssl_version) >= LooseVersion(min_openssl_version): dbg_msg = "System OpenSSL library '%s' with version %s fulfills requested version %s" self.log.debug(dbg_msg, system_solib, openssl_version, min_openssl_version) - self.system_ssl['libs'].append(system_solib) + target_system_ssl['version'] = openssl_version + target_system_ssl['libs'].append(system_solib) else: dbg_msg = "System OpenSSL library '%s' with version %s is older than requested version %s" self.log.debug(dbg_msg, system_solib, openssl_version, min_openssl_version) else: # one of the OpenSSL libraries is missing, switch to next group of versioned libs - self.system_ssl['libs'] = [] break - if len(self.system_ssl['libs']) == len(openssl_libs): - # keep these libraries as possible targets for this installation - target_system_ssl_libs = system_versioned_libs[idx] + if len(target_system_ssl['libs']) == len(openssl_libs): + # target libraries found, ignore further options break - if len(self.system_ssl['libs']) == len(openssl_libs): - self.system_ssl['version'] = openssl_version + if len(target_system_ssl['libs']) == len(openssl_libs): + self.system_ssl.update(target_system_ssl) info_msg = "Found OpenSSL library version %s in host system: %s" self.log.info(info_msg, self.system_ssl['version'], os.path.dirname(self.system_ssl['libs'][0])) else: @@ -314,7 +315,7 @@ def __init__(self, *args, **kwargs): return # system OpenSSL is fine, change target libraries to the ones found in it - self.target_ssl_libs = target_system_ssl_libs + self.target_ssl_libs = [os.path.basename(solib) for solib in target_system_ssl["libs"]] self.log.info("Target system OpenSSL libraries: %s", self.target_ssl_libs) def fetch_step(self, *args, **kwargs): From c24b4f704ccaf3a16cce0788f89c1dc40383da8d Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 28 May 2024 02:21:14 +0200 Subject: [PATCH 06/20] replace case insensitive pattern with corresponding flag in re.compile --- easybuild/easyblocks/o/openssl_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 2438a79e04..80d6bb6cd6 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -101,7 +101,7 @@ def __init__(self, *args, **kwargs): # Regex pattern to find version strings in OpenSSL libraries and headers full_version_regex = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+[a-z]?') - openssl_version_regex = re.compile(r'(?i)OpenSSL[\s_]+([0-9]+\.[0-9]+(\.[0-9]+[a-z]?)*)', re.M) + openssl_version_regex = re.compile(r'OpenSSL[\s_]+([0-9]+\.[0-9]+(\.[0-9]+[a-z]?)*)', re.M | re.I) # Libraries packaged in OpenSSL openssl_libs = ['libssl', 'libcrypto'] From 40120e6c230bec7ed609cd9485b5486126b528ae Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 28 May 2024 02:28:06 +0200 Subject: [PATCH 07/20] system OpenSSL header version only needs to be higher than required minimum version in wrapper easyconfig --- easybuild/easyblocks/o/openssl_wrapper.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 80d6bb6cd6..ceddbfe430 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -270,17 +270,7 @@ def __init__(self, *args, **kwargs): err_msg = "System OpenSSL header '%s' does not contain any recognizable version string" raise EasyBuildError(err_msg, opensslv_path) - header_version_major = header_version.split('.')[0] - system_ssl_version_major = self.system_ssl['version'].split('.')[0] - - # Compare only major version of header for OpenSSL 3+ - # This is needed on Debian-based distros - if LooseVersion(self.version) >= LooseVersion('3'): - header_found = (header_version_major == system_ssl_version_major) - else: - header_found = (header_version == self.system_ssl['version']) - - if header_found: + if LooseVersion(header_version) >= LooseVersion(min_openssl_version): self.system_ssl['include'] = include_dir info_msg = "Found OpenSSL headers v%s in host system: %s" self.log.info(info_msg, header_version, self.system_ssl['include']) From c5b86271aff049e5336be2d0046d234ef781dfd3 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 29 May 2024 10:16:01 +0200 Subject: [PATCH 08/20] remove unused idx variable from OpenSSL wrapper --- easybuild/easyblocks/o/openssl_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index ceddbfe430..c213bea616 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -169,7 +169,7 @@ def __init__(self, *args, **kwargs): # Check the system libraries of OpenSSL # Find library file and compare its version string target_system_ssl = {} - for idx, solibs in enumerate(system_versioned_libs): + for solibs in system_versioned_libs: target_system_ssl['version'] = '0' target_system_ssl['libs'] = [] for solib in solibs: From b8c29bd13161d37bf7c032785a59c2dcfe75e8fa Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 29 May 2024 10:17:54 +0200 Subject: [PATCH 09/20] hide pkg-config commands from trace output in OpenSSL wrapper --- easybuild/easyblocks/o/openssl_wrapper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index c213bea616..67ad2dede6 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -464,7 +464,7 @@ def install_pc_files(self): # check suffixed names with v1.1 pc_name_suffix = pc_name + '11' pc_exists_cmd = "pkg-config --exists %s" % pc_name_suffix - if run_cmd(pc_exists_cmd, simple=True, log_ok=False, log_all=False): + if run_cmd(pc_exists_cmd, simple=True, log_ok=False, log_all=False, trace=False): self.log.info("%s exists", pc_name_suffix) pc_name = pc_name_suffix @@ -473,7 +473,7 @@ def install_pc_files(self): for require_type in ['Requires', 'Requires.private']: require_print = require_type.lower().replace('.', '-') pc_print_cmd = "pkg-config --print-%s %s" % (require_print, pc_name) - out, _ = run_cmd(pc_print_cmd, simple=False, log_ok=False) + out, _ = run_cmd(pc_print_cmd, simple=False, log_ok=False, trace=False) self.log.info("Output of '%s': %s", pc_print_cmd, out) if out: @@ -494,12 +494,12 @@ def install_pc_files(self): pc_file['cflags'] = "Cflags: -I${includedir}" # infer private libs through pkg-config pc_libs_cmd = "pkg-config --libs %s" % pc_name - out, _ = run_cmd(pc_libs_cmd, simple=False, log_ok=False) + out, _ = run_cmd(pc_libs_cmd, simple=False, log_ok=False, trace=False) self.log.info("Output of '%s': %s", pc_libs_cmd, out) linker_libs = out pc_libs_static_cmd = "pkg-config --libs --static %s" % pc_name - out, _ = run_cmd(pc_libs_static_cmd, simple=False, log_ok=False) + out, _ = run_cmd(pc_libs_static_cmd, simple=False, log_ok=False, trace=False) self.log.info("Output of '%s': %s", pc_libs_static_cmd, out) libs_priv = "%s " % out.rstrip() From 4073a3145b66fcf91c5f65f880798ef8f33343d2 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 29 May 2024 10:55:23 +0200 Subject: [PATCH 10/20] replace majmin_version with generation attribute in OpenSSL wrapper --- easybuild/easyblocks/o/openssl_wrapper.py | 34 ++++++++++------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 67ad2dede6..8a15528f2a 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -73,16 +73,15 @@ def __init__(self, *args, **kwargs): """Locate the installation files of OpenSSL in the host system""" super(EB_OpenSSL_wrapper, self).__init__(*args, **kwargs) - # Wrapper should have at least a major minor version numbers for OpenSSL before version 3+ - if LooseVersion(self.version) >= LooseVersion('3') and self.version.count('.') == 0: - self.majmin_version = self.version - else: + if LooseVersion(self.version) < LooseVersion('2'): try: subversions = self.version.split('.') - self.majmin_version = '%s.%s' % (subversions[0], subversions[1]) + self.generation = '%s.%s' % (subversions[0], subversions[1]) except (AttributeError, IndexError): - err_msg = "Wrapper OpenSSL version does not have any subversion: %s" + err_msg = "Wrapper for OpenSSL v1 version is missing a minor version: %s" raise EasyBuildError(err_msg, self.version) + elif LooseVersion(self.version) < LooseVersion('4'): + self.generation = "3" # Set minimum OpenSSL version min_openssl_version = self.cfg.get('minimum_openssl_version') @@ -117,10 +116,6 @@ def __init__(self, *args, **kwargs): LINUX: ('so.1.1', ), DARWIN: ('1.1.dylib', ), }, - '3.0': { - LINUX: ('so.3', ), - DARWIN: ('3.dylib', ), - }, '3': { LINUX: ('so.3', ), DARWIN: ('3.dylib', ), @@ -128,16 +123,16 @@ def __init__(self, *args, **kwargs): } os_type = get_os_type() - if self.majmin_version in openssl_libext and os_type in openssl_libext[self.majmin_version]: + if self.generation in openssl_libext and os_type in openssl_libext[self.generation]: # generate matrix of versioned .so filenames system_versioned_libs = [ ['%s.%s' % (lib, ext) for lib in openssl_libs] - for ext in openssl_libext[self.majmin_version][os_type] + for ext in openssl_libext[self.generation][os_type] ] self.log.info("Matrix of version library names: %s", system_versioned_libs) else: - err_msg = "Don't know name of OpenSSL system library for version %s and OS type %s" - raise EasyBuildError(err_msg, self.majmin_version, os_type) + err_msg = "OpenSSL system library for version %s and OS type %s is unsupported" + raise EasyBuildError(err_msg, self.generation, os_type) # by default target the first option of each OpenSSL library, # which corresponds to installation from source @@ -148,10 +143,9 @@ def __init__(self, *args, **kwargs): openssl_engines = { '1.0': 'engines', '1.1': 'engines-1.1', - '3.0': 'engines-3', '3': 'engines-3', } - self.target_ssl_engine = openssl_engines[self.majmin_version] + self.target_ssl_engine = openssl_engines[self.generation] # Paths to system libraries and headers of OpenSSL self.system_ssl = { @@ -245,11 +239,11 @@ def __init__(self, *args, **kwargs): # headers are located in 'include/openssl' by default ssl_include_subdirs = ['openssl'] - if self.majmin_version == '1.1': + if self.generation == '1.1': # but version 1.1 can be installed in 'include/openssl11/openssl' as well, for example in CentOS 7 # prefer 'include/openssl' as long as the version of headers matches ssl_include_subdirs.append(os.path.join('openssl11', self.name.lower())) - elif LooseVersion(self.version) >= LooseVersion('3'): + elif self.generation == '3': # but version 3.x can be installed in 'include/openssl3/openssl' as well, for example in RHEL 8 derivatives # prefer 'include/openssl' as long as the version of headers matches ssl_include_subdirs.append(os.path.join('openssl3', self.name.lower())) @@ -408,7 +402,7 @@ def sanity_check_step(self): custom_commands = [ # make sure that version mentioned in output of 'openssl version' matches version we are using - "ssl_ver=$(openssl version); [ ${ssl_ver:8:%s} == '%s' ]" % (ssl_ver_comp_chars, self.majmin_version), + "ssl_ver=$(openssl version); [ ${ssl_ver:8:%s} == '%s' ]" % (ssl_ver_comp_chars, self.generation), ("echo | openssl s_client%s -connect github.com:443 -verify 9 " "| grep 'Verify return code: 0 (ok)'" % proxy_arg), ] @@ -460,7 +454,7 @@ def install_pc_files(self): # component name in system pkg-config pc_name = pc_comp - if self.majmin_version == '1.1': + if self.generation == '1.1': # check suffixed names with v1.1 pc_name_suffix = pc_name + '11' pc_exists_cmd = "pkg-config --exists %s" % pc_name_suffix From 2d73932cc0117afb7613541d6ba88959b248736c Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 29 May 2024 10:59:14 +0200 Subject: [PATCH 11/20] fix error message on failed header version check in OpenSSL wrapper --- easybuild/easyblocks/o/openssl_wrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 8a15528f2a..ee97a3040b 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -270,8 +270,8 @@ def __init__(self, *args, **kwargs): self.log.info(info_msg, header_version, self.system_ssl['include']) break else: - dbg_msg = "System OpenSSL header version '%s' does not match library version '%s'" - self.log.debug(dbg_msg, header_version, self.system_ssl['version']) + dbg_msg = "System OpenSSL header version '%s' does not fulfill minimum version requirement '%s'" + self.log.debug(dbg_msg, header_version, min_openssl_version) else: self.log.info("System OpenSSL header file %s not found", opensslv_path) From 476a7da99e5ef44433e4ce738fa46a288027722b Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 29 May 2024 11:40:48 +0200 Subject: [PATCH 12/20] check openssl executable version at detection time in OpenSSL wrapper --- easybuild/easyblocks/o/openssl_wrapper.py | 41 +++++++++++++++++------ 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index ee97a3040b..26a1fdaa74 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -282,20 +282,22 @@ def __init__(self, *args, **kwargs): raise EasyBuildError(err_msg, self.version, self.system_ssl['version']) # Check system OpenSSL binary - if self.majmin_version == '1.1': - # prefer 'openssl11' over 'openssl' with v1.1 - self.system_ssl['bin'] = which('openssl11') - elif LooseVersion(self.version) >= LooseVersion('3'): - # prefer 'openssl3' over 'openssl' with v3.x - self.system_ssl['bin'] = which('openssl3') + target_ssl_bins = ['openssl'] + if self.generation == '1.1': + target_ssl_bins.insert(0, 'openssl11') # prefer 'openssl11' over 'openssl' with v1.1 + elif self.generation == '3': + target_ssl_bins.insert(0, 'openssl3') # prefer 'openssl3' over 'openssl' with v3 - if not self.system_ssl['bin']: - self.system_ssl['bin'] = which('openssl') + for target_bin in target_ssl_bins: + target_bin_path, target_bin_version = self.get_openssl_bin_version(target_bin) + if target_bin_version == self.system_ssl['version']: + self.system_ssl['bin'] = target_bin_path + break if self.system_ssl['bin']: - self.log.info("System OpenSSL binary found: %s", self.system_ssl['bin']) + self.log.info("System OpenSSL binary for version %s found: %s", self.system_ssl['version'], self.system_ssl['bin']) else: - self.log.info("System OpenSSL binary not found!") + self.log.info("System OpenSSL binary for version %s not found!", self.system_ssl['version']) return # system OpenSSL is fine, change target libraries to the ones found in it @@ -409,6 +411,25 @@ def sanity_check_step(self): super(Bundle, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) + def get_openssl_bin_version(self, bin_name): + """Check OpenSSL executable version""" + bin_path = which(bin_name) + if not bin_path: + self.log.debug("OpenSSL executable '%s' not found", bin_name) + return None, None + + cmd = "%s version" % bin_path + out, _ = run_cmd(cmd, simple=False, log_ok=False, trace=False) + + try: + bin_version = out.split(' ')[1] + except (AttributeError, IndexError): + raise EasyBuildError("Failed to check version of OpenSSL executable: %s", bin_path) + else: + self.log.debug("Version of OpenSSL executable '%s': %s", bin_path, bin_version) + + return bin_path, bin_version + def install_pc_files(self): """Install pkg-config files for the wrapper""" From f54a5fb8dab0b20c16c89aea88ca6fa92fe529fa Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 29 May 2024 11:48:14 +0200 Subject: [PATCH 13/20] use pattern matching to sanity check version of openssl executbale in OpenSSL wrapper --- easybuild/easyblocks/o/openssl_wrapper.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 26a1fdaa74..1359e33857 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -397,14 +397,9 @@ def sanity_check_step(self): if proxy_parsed.netloc: proxy_arg = ' -proxy %s' % proxy_parsed.netloc - if LooseVersion(self.version) >= LooseVersion('3') and self.version.count('.') == 0: - ssl_ver_comp_chars = 1 - else: - ssl_ver_comp_chars = 3 - custom_commands = [ # make sure that version mentioned in output of 'openssl version' matches version we are using - "ssl_ver=$(openssl version); [ ${ssl_ver:8:%s} == '%s' ]" % (ssl_ver_comp_chars, self.generation), + '[[ "$(openssl version)" =~ ^OpenSSL.%s ]]' % self.generation, ("echo | openssl s_client%s -connect github.com:443 -verify 9 " "| grep 'Verify return code: 0 (ok)'" % proxy_arg), ] From bc1acde23b059f308a73685d94d8325577518d50 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 29 May 2024 11:54:34 +0200 Subject: [PATCH 14/20] fix codestyle issues in OpenSSL wrapper --- easybuild/easyblocks/o/openssl_wrapper.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 1359e33857..ad33adba2e 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -81,7 +81,7 @@ def __init__(self, *args, **kwargs): err_msg = "Wrapper for OpenSSL v1 version is missing a minor version: %s" raise EasyBuildError(err_msg, self.version) elif LooseVersion(self.version) < LooseVersion('4'): - self.generation = "3" + self.generation = "3" # Set minimum OpenSSL version min_openssl_version = self.cfg.get('minimum_openssl_version') @@ -295,7 +295,8 @@ def __init__(self, *args, **kwargs): break if self.system_ssl['bin']: - self.log.info("System OpenSSL binary for version %s found: %s", self.system_ssl['version'], self.system_ssl['bin']) + log_msg = "System OpenSSL binary for version %s found: %s" + self.log.info(log_msg, self.system_ssl['version'], self.system_ssl['bin']) else: self.log.info("System OpenSSL binary for version %s not found!", self.system_ssl['version']) return From b0cbd354f0cc574eb6080037b26c100dd7562593 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 29 May 2024 17:01:28 +0200 Subject: [PATCH 15/20] define version of OpenSSL wrapper from executable and allow mismatch between components --- easybuild/easyblocks/o/openssl_wrapper.py | 68 +++++++++++------------ 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index ad33adba2e..082fdaa09b 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -162,51 +162,48 @@ def __init__(self, *args, **kwargs): # Check the system libraries of OpenSSL # Find library file and compare its version string - target_system_ssl = {} for solibs in system_versioned_libs: - target_system_ssl['version'] = '0' - target_system_ssl['libs'] = [] + target_ssl_libs = [] for solib in solibs: system_solib = find_library_path(solib) - if system_solib: + if not system_solib: + # this OpenSSL library is missing, move on to next group of versioned libs + break + + try: # get version of system library filename + ssl_lib_version = full_version_regex.search(os.path.realpath(system_solib)).group(0) + except AttributeError: + # filename lacks the full version, fallback to version strings within the library + solib_strings = read_file(system_solib, mode="rb").decode('utf-8', 'replace') try: - openssl_version = full_version_regex.search(os.path.realpath(system_solib)).group(0) - except AttributeError: - # filename lacks the full version, fallback to version strings within the library - solib_strings = read_file(system_solib, mode="rb").decode('utf-8', 'replace') - try: - openssl_support_versions = openssl_version_regex.findall(solib_strings) - openssl_support_versions.sort() - dbg_msg = "System OpenSSL library '%s' supports versions: %s" - dbg_msg_support_versions = ', '.join([''.join(v) for v in openssl_support_versions]) - self.log.debug(dbg_msg, system_solib, dbg_msg_support_versions) - # pick highest supported version - openssl_version = openssl_support_versions[-1][0] - except IndexError: - dbg_msg = "Could not detect the full version of system OpenSSL library: %s" - self.log.debug(dbg_msg, system_solib) - # check that system version fulfills requirements - if LooseVersion(openssl_version) >= LooseVersion(min_openssl_version): - dbg_msg = "System OpenSSL library '%s' with version %s fulfills requested version %s" - self.log.debug(dbg_msg, system_solib, openssl_version, min_openssl_version) - target_system_ssl['version'] = openssl_version - target_system_ssl['libs'].append(system_solib) - else: - dbg_msg = "System OpenSSL library '%s' with version %s is older than requested version %s" - self.log.debug(dbg_msg, system_solib, openssl_version, min_openssl_version) + ssl_lib_support_versions = openssl_version_regex.findall(solib_strings) + ssl_lib_support_versions.sort() + dbg_msg = "System OpenSSL library '%s' supports versions: %s" + dbg_msg_support_versions = ', '.join([''.join(v) for v in ssl_lib_support_versions]) + self.log.debug(dbg_msg, system_solib, dbg_msg_support_versions) + # pick highest supported version + ssl_lib_version = ssl_lib_support_versions[-1][0] + except IndexError: + dbg_msg = "Could not detect the full version of system OpenSSL library: %s" + self.log.debug(dbg_msg, system_solib) + # check that system library version fulfills requirements + if LooseVersion(ssl_lib_version) >= LooseVersion(min_openssl_version): + dbg_msg = "System OpenSSL library '%s' with version %s fulfills requested version %s" + self.log.debug(dbg_msg, system_solib, ssl_lib_version, min_openssl_version) + target_ssl_libs.append(system_solib) else: - # one of the OpenSSL libraries is missing, switch to next group of versioned libs - break + dbg_msg = "System OpenSSL library '%s' with version %s is older than requested version %s" + self.log.debug(dbg_msg, system_solib, ssl_lib_version, min_openssl_version) - if len(target_system_ssl['libs']) == len(openssl_libs): + if len(target_ssl_libs) == len(openssl_libs): # target libraries found, ignore further options break - if len(target_system_ssl['libs']) == len(openssl_libs): - self.system_ssl.update(target_system_ssl) + if len(target_ssl_libs) == len(openssl_libs): + self.system_ssl['libs'] = target_ssl_libs info_msg = "Found OpenSSL library version %s in host system: %s" - self.log.info(info_msg, self.system_ssl['version'], os.path.dirname(self.system_ssl['libs'][0])) + self.log.info(info_msg, ssl_lib_version, os.path.dirname(self.system_ssl['libs'][0])) else: self.log.info("OpenSSL library not found in host system, falling back to OpenSSL in EasyBuild") return @@ -290,7 +287,8 @@ def __init__(self, *args, **kwargs): for target_bin in target_ssl_bins: target_bin_path, target_bin_version = self.get_openssl_bin_version(target_bin) - if target_bin_version == self.system_ssl['version']: + if LooseVersion(target_bin_version) >= LooseVersion(min_openssl_version): + self.system_ssl['version'] = target_bin_version self.system_ssl['bin'] = target_bin_path break From e40f78e7ee03dc1e5eaf477bf00ca016a5f650c7 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 29 May 2024 18:06:59 +0200 Subject: [PATCH 16/20] define maximum version for OpenSSL wrapper and start with executable detection --- easybuild/easyblocks/o/openssl_wrapper.py | 115 +++++++++++----------- 1 file changed, 58 insertions(+), 57 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 082fdaa09b..ebc1cf2e7e 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -84,19 +84,23 @@ def __init__(self, *args, **kwargs): self.generation = "3" # Set minimum OpenSSL version - min_openssl_version = self.cfg.get('minimum_openssl_version') - - if not min_openssl_version: - min_openssl_version = self.version - elif not isinstance(min_openssl_version, string_type): - min_openssl_version = str(min_openssl_version) - + self.min_version = self.cfg.get('minimum_openssl_version') + if not self.min_version: + self.min_version = self.version + elif not isinstance(self.min_version, string_type): + self.min_version = str(self.min_version) # Minimum OpenSSL version can only increase depth of wrapper version - if min_openssl_version.startswith(self.version): - self.log.debug("Requiring minimum OpenSSL version: %s", min_openssl_version) + if self.min_version.startswith(self.version): + self.log.debug("Requiring minimum OpenSSL version: %s", self.min_version) else: err_msg = "Requested minimum OpenSSL version '%s' does not fit in wrapper easyconfig version '%s'" - raise EasyBuildError(err_msg, min_openssl_version, self.version) + raise EasyBuildError(err_msg, self.min_version, self.version) + + # Set maximum OpenSSL version (increase smallest revision by 1) + max_version_parts = [int(subv) for subv in self.version.split('.')] + max_version_parts[-1] += 1 + self.max_version = '.'.join([str(subv) for subv in max_version_parts]) + self.log.debug("Restricting maximum OpenSSL version: %s", self.max_version) # Regex pattern to find version strings in OpenSSL libraries and headers full_version_regex = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+[a-z]?') @@ -134,20 +138,14 @@ def __init__(self, *args, **kwargs): err_msg = "OpenSSL system library for version %s and OS type %s is unsupported" raise EasyBuildError(err_msg, self.generation, os_type) - # by default target the first option of each OpenSSL library, - # which corresponds to installation from source - self.target_ssl_libs = system_versioned_libs[0] - self.log.info("Target OpenSSL libraries: %s", self.target_ssl_libs) - # folders containing engines libraries openssl_engines = { '1.0': 'engines', '1.1': 'engines-1.1', '3': 'engines-3', } - self.target_ssl_engine = openssl_engines[self.generation] - # Paths to system libraries and headers of OpenSSL + # Paths to system components of OpenSSL self.system_ssl = { 'bin': None, 'engines': None, @@ -160,6 +158,30 @@ def __init__(self, *args, **kwargs): self.log.info("Not wrapping system OpenSSL installation by user request") return + # Check system OpenSSL binary + target_ssl_bins = ['openssl'] + if self.generation == '1.1': + target_ssl_bins.insert(0, 'openssl11') # prefer 'openssl11' over 'openssl' with v1.1 + elif self.generation == '3': + target_ssl_bins.insert(0, 'openssl3') # prefer 'openssl3' over 'openssl' with v3 + + for ssl_bin in target_ssl_bins: + ssl_bin_path, ssl_bin_version = self.get_openssl_bin_version(ssl_bin) + if ssl_bin_path: + if LooseVersion(self.min_version) <= LooseVersion(ssl_bin_version) < LooseVersion(self.max_version): + self.system_ssl['version'] = ssl_bin_version + self.system_ssl['bin'] = ssl_bin_path + break + + if self.system_ssl['bin'] and self.system_ssl['version']: + log_msg = "System OpenSSL binary for version %s found: %s" + self.log.info(log_msg, self.system_ssl['version'], self.system_ssl['bin']) + else: + log_msg = "OpenSSL binary for version %s not found in host system! " + log_msg += "Falling back to building OpenSSL from source" + self.log.info(log_msg, self.version) + return + # Check the system libraries of OpenSSL # Find library file and compare its version string for solibs in system_versioned_libs: @@ -188,13 +210,13 @@ def __init__(self, *args, **kwargs): dbg_msg = "Could not detect the full version of system OpenSSL library: %s" self.log.debug(dbg_msg, system_solib) # check that system library version fulfills requirements - if LooseVersion(ssl_lib_version) >= LooseVersion(min_openssl_version): + if LooseVersion(self.min_version) <= LooseVersion(ssl_lib_version) < LooseVersion(self.max_version): dbg_msg = "System OpenSSL library '%s' with version %s fulfills requested version %s" - self.log.debug(dbg_msg, system_solib, ssl_lib_version, min_openssl_version) + self.log.debug(dbg_msg, system_solib, ssl_lib_version, self.min_version) target_ssl_libs.append(system_solib) else: dbg_msg = "System OpenSSL library '%s' with version %s is older than requested version %s" - self.log.debug(dbg_msg, system_solib, ssl_lib_version, min_openssl_version) + self.log.debug(dbg_msg, system_solib, ssl_lib_version, self.min_version) if len(target_ssl_libs) == len(openssl_libs): # target libraries found, ignore further options @@ -205,14 +227,16 @@ def __init__(self, *args, **kwargs): info_msg = "Found OpenSSL library version %s in host system: %s" self.log.info(info_msg, ssl_lib_version, os.path.dirname(self.system_ssl['libs'][0])) else: - self.log.info("OpenSSL library not found in host system, falling back to OpenSSL in EasyBuild") + log_msg = "OpenSSL library for version %s not found in host system! " + log_msg += "Falling back to building OpenSSL from source" + self.log.info(log_msg, self.system_ssl['version']) return # Directory with engine libraries lib_dir = os.path.dirname(self.system_ssl['libs'][0]) lib_engines_dir = [ - os.path.join(lib_dir, 'openssl', self.target_ssl_engine), - os.path.join(lib_dir, self.target_ssl_engine), + os.path.join(lib_dir, 'openssl', openssl_engines[self.generation]), + os.path.join(lib_dir, openssl_engines[self.generation]), ] for engines_path in lib_engines_dir: @@ -222,7 +246,9 @@ def __init__(self, *args, **kwargs): break if not self.system_ssl['engines']: - self.log.info("OpenSSL engines not found in host system, falling back to OpenSSL in EasyBuild") + log_msg = "OpenSSL engines for version %s not found in host system! " + log_msg += "Falling back to building OpenSSL from source" + self.log.info(log_msg, self.system_ssl['version']) return # Check system include paths for OpenSSL headers @@ -256,19 +282,19 @@ def __init__(self, *args, **kwargs): # check version reported by opensslv.h opensslv = read_file(opensslv_path) try: - header_version = openssl_version_regex.search(opensslv).group(1) + ssl_head_version = openssl_version_regex.search(opensslv).group(1) except AttributeError: err_msg = "System OpenSSL header '%s' does not contain any recognizable version string" raise EasyBuildError(err_msg, opensslv_path) - if LooseVersion(header_version) >= LooseVersion(min_openssl_version): + if LooseVersion(self.min_version) <= LooseVersion(ssl_head_version) < LooseVersion(self.max_version): self.system_ssl['include'] = include_dir info_msg = "Found OpenSSL headers v%s in host system: %s" - self.log.info(info_msg, header_version, self.system_ssl['include']) + self.log.info(info_msg, ssl_head_version, self.system_ssl['include']) break else: dbg_msg = "System OpenSSL header version '%s' does not fulfill minimum version requirement '%s'" - self.log.debug(dbg_msg, header_version, min_openssl_version) + self.log.debug(dbg_msg, ssl_head_version, self.min_version) else: self.log.info("System OpenSSL header file %s not found", opensslv_path) @@ -278,31 +304,6 @@ def __init__(self, *args, **kwargs): "source in EasyBuild by setting 'wrap_system_openssl = False' in the OpenSSL easyconfig.") raise EasyBuildError(err_msg, self.version, self.system_ssl['version']) - # Check system OpenSSL binary - target_ssl_bins = ['openssl'] - if self.generation == '1.1': - target_ssl_bins.insert(0, 'openssl11') # prefer 'openssl11' over 'openssl' with v1.1 - elif self.generation == '3': - target_ssl_bins.insert(0, 'openssl3') # prefer 'openssl3' over 'openssl' with v3 - - for target_bin in target_ssl_bins: - target_bin_path, target_bin_version = self.get_openssl_bin_version(target_bin) - if LooseVersion(target_bin_version) >= LooseVersion(min_openssl_version): - self.system_ssl['version'] = target_bin_version - self.system_ssl['bin'] = target_bin_path - break - - if self.system_ssl['bin']: - log_msg = "System OpenSSL binary for version %s found: %s" - self.log.info(log_msg, self.system_ssl['version'], self.system_ssl['bin']) - else: - self.log.info("System OpenSSL binary for version %s not found!", self.system_ssl['version']) - return - - # system OpenSSL is fine, change target libraries to the ones found in it - self.target_ssl_libs = [os.path.basename(solib) for solib in target_system_ssl["libs"]] - self.log.info("Target system OpenSSL libraries: %s", self.target_ssl_libs) - def fetch_step(self, *args, **kwargs): """Fetch sources if OpenSSL component is needed""" if not all(self.system_ssl[key] for key in ('bin', 'engines', 'include', 'libs')): @@ -370,15 +371,15 @@ def sanity_check_step(self): """Custom sanity check for OpenSSL wrapper.""" shlib_ext = get_shared_lib_ext() - ssl_libs = ['%s.%s' % (libso.split('.')[0], shlib_ext) for libso in self.target_ssl_libs] - ssl_libs.extend(self.target_ssl_libs) + ssl_libs = [os.path.basename(solib) for solib in self.system_ssl["libs"]] + ssl_libs.extend(['%s.%s' % (solib.split('.')[0], shlib_ext) for solib in ssl_libs]) ssl_files = [os.path.join('bin', self.name.lower())] ssl_files.extend(os.path.join('lib', libso) for libso in ssl_libs) ssl_dirs = [ os.path.join('include', self.name.lower()), - os.path.join('lib', self.target_ssl_engine), + os.path.join('lib', os.path.basename(self.system_ssl["engines"])), os.path.join('lib', 'pkgconfig'), ] @@ -444,7 +445,7 @@ def install_pc_files(self): 'libcrypto': { 'name': 'OpenSSL-libcrypto', 'description': 'OpenSSL cryptography library', - 'enginesdir': self.target_ssl_engine, + 'enginesdir': os.path.basename(self.system_ssl["engines"]), }, 'libssl': { 'name': 'OpenSSL-libssl', From 48bc68d276cbf60f730813b09431d23e85397a3a Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 29 May 2024 18:47:35 +0200 Subject: [PATCH 17/20] define target components per generation in OpenSSL wrapper --- easybuild/easyblocks/o/openssl_wrapper.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index ebc1cf2e7e..d758d684a0 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -145,6 +145,14 @@ def __init__(self, *args, **kwargs): '3': 'engines-3', } + # Define targets for this generation + # They are used to sanity check both installations from source or wrappers + self.generation_targets = { + 'bin': 'openssl', + 'engines': openssl_engines[self.generation], + 'libs': system_versioned_libs[0], # first set of libs match source installs + } + # Paths to system components of OpenSSL self.system_ssl = { 'bin': None, @@ -224,6 +232,7 @@ def __init__(self, *args, **kwargs): if len(target_ssl_libs) == len(openssl_libs): self.system_ssl['libs'] = target_ssl_libs + self.generation_targets['libs'] = [os.path.basename(solib) for solib in target_ssl_libs] info_msg = "Found OpenSSL library version %s in host system: %s" self.log.info(info_msg, ssl_lib_version, os.path.dirname(self.system_ssl['libs'][0])) else: @@ -371,15 +380,15 @@ def sanity_check_step(self): """Custom sanity check for OpenSSL wrapper.""" shlib_ext = get_shared_lib_ext() - ssl_libs = [os.path.basename(solib) for solib in self.system_ssl["libs"]] - ssl_libs.extend(['%s.%s' % (solib.split('.')[0], shlib_ext) for solib in ssl_libs]) + ssl_libs = ['%s.%s' % (solib.split('.')[0], shlib_ext) for solib in self.generation_targets['libs']] + ssl_libs.extend(self.generation_targets['libs']) ssl_files = [os.path.join('bin', self.name.lower())] ssl_files.extend(os.path.join('lib', libso) for libso in ssl_libs) ssl_dirs = [ os.path.join('include', self.name.lower()), - os.path.join('lib', os.path.basename(self.system_ssl["engines"])), + os.path.join('lib', self.generation_targets["engines"]), os.path.join('lib', 'pkgconfig'), ] @@ -445,7 +454,7 @@ def install_pc_files(self): 'libcrypto': { 'name': 'OpenSSL-libcrypto', 'description': 'OpenSSL cryptography library', - 'enginesdir': os.path.basename(self.system_ssl["engines"]), + 'enginesdir': self.generation_targets["engines"], }, 'libssl': { 'name': 'OpenSSL-libssl', From 3128f50e721513ddb3cd53d5e9c05617a5fedd89 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 29 May 2024 18:55:29 +0200 Subject: [PATCH 18/20] use generation target to detect and sanity check OpenSSL executable --- easybuild/easyblocks/o/openssl_wrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index d758d684a0..d776c43a02 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -167,7 +167,7 @@ def __init__(self, *args, **kwargs): return # Check system OpenSSL binary - target_ssl_bins = ['openssl'] + target_ssl_bins = [self.generation_targets['bin']] if self.generation == '1.1': target_ssl_bins.insert(0, 'openssl11') # prefer 'openssl11' over 'openssl' with v1.1 elif self.generation == '3': @@ -383,7 +383,7 @@ def sanity_check_step(self): ssl_libs = ['%s.%s' % (solib.split('.')[0], shlib_ext) for solib in self.generation_targets['libs']] ssl_libs.extend(self.generation_targets['libs']) - ssl_files = [os.path.join('bin', self.name.lower())] + ssl_files = [os.path.join('bin', self.generation_targets['bin'])] ssl_files.extend(os.path.join('lib', libso) for libso in ssl_libs) ssl_dirs = [ From 81b4294a2b8a9f2fedf90488ce246fc9a5f639b2 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 31 May 2024 15:44:00 +0000 Subject: [PATCH 19/20] Allow external PRRTE in openmpi easyblock --- easybuild/easyblocks/o/openmpi.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/o/openmpi.py b/easybuild/easyblocks/o/openmpi.py index 8460aad58a..e8261d0ca3 100644 --- a/easybuild/easyblocks/o/openmpi.py +++ b/easybuild/easyblocks/o/openmpi.py @@ -72,6 +72,8 @@ def config_opt_used(key, enable_opt=False): known_dependencies = ['CUDA', 'hwloc', 'libevent', 'libfabric', 'PMIx', 'UCX'] if LooseVersion(self.version) >= '4.1.4': known_dependencies.append('UCC') + if LooseVersion(self.version) >= '5.0.0': + known_dependencies.append('PRRTE') # Value to use for `--with-=` if the dependency is not specified in the easyconfig # No entry is interpreted as no option added at all @@ -85,6 +87,8 @@ def config_opt_used(key, enable_opt=False): # For these the default is to use an internal copy and not using any is not supported for dep in ('hwloc', 'libevent', 'PMIx'): unused_dep_value[dep] = 'internal' + if LooseVersion(self.version) >= '5.0.0': + unused_dep_value['PRRTE'] = 'internal' # handle dependencies for dep in known_dependencies: @@ -165,7 +169,8 @@ def sanity_check_step(self): bin_names = ['mpicc', 'mpicxx', 'mpif90', 'mpifort', 'mpirun', 'ompi_info', 'opal_wrapper'] if LooseVersion(self.version) >= LooseVersion('5.0.0'): - bin_names.append('prterun') + if not get_software_root('PRRTE'): + bin_names.append('prterun') else: bin_names.append('orterun') bin_files = [os.path.join('bin', x) for x in bin_names] @@ -173,13 +178,14 @@ def sanity_check_step(self): shlib_ext = get_shared_lib_ext() lib_names = ['mpi_mpifh', 'mpi', 'open-pal'] if LooseVersion(self.version) >= LooseVersion('5.0.0'): - lib_names.append('prrte') + if not get_software_root('PRRTE'): + lib_names.append('prrte') else: lib_names.extend(['ompitrace', 'open-rte']) lib_files = [os.path.join('lib', 'lib%s.%s' % (x, shlib_ext)) for x in lib_names] inc_names = ['mpi-ext', 'mpif-config', 'mpif', 'mpi', 'mpi_portable_platform'] - if LooseVersion(self.version) >= LooseVersion('5.0.0'): + if LooseVersion(self.version) >= LooseVersion('5.0.0') and not get_software_root('PRRTE'): inc_names.append('prte') inc_files = [os.path.join('include', x + '.h') for x in inc_names] From e29d315c759d9827141b1588342e4ff8f55ebbbe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 5 Jun 2024 11:10:52 +0200 Subject: [PATCH 20/20] replace use of run_cmd in OpenSSL wrapper easyblock with run_shell_cmd --- easybuild/easyblocks/o/openssl_wrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 0de6393ee1..22a50df87e 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -423,10 +423,10 @@ def get_openssl_bin_version(self, bin_name): return None, None cmd = "%s version" % bin_path - out, _ = run_cmd(cmd, simple=False, log_ok=False, trace=False) + res = run_shell_cmd(cmd, fail_on_error=False, hidden=True) try: - bin_version = out.split(' ')[1] + bin_version = res.output.split(' ')[1] except (AttributeError, IndexError): raise EasyBuildError("Failed to check version of OpenSSL executable: %s", bin_path) else: