diff --git a/easybuild/easyblocks/o/openmpi.py b/easybuild/easyblocks/o/openmpi.py index 8460aad58a..e8261d0ca3 100644 --- a/easybuild/easyblocks/o/openmpi.py +++ b/easybuild/easyblocks/o/openmpi.py @@ -72,6 +72,8 @@ def config_opt_used(key, enable_opt=False): known_dependencies = ['CUDA', 'hwloc', 'libevent', 'libfabric', 'PMIx', 'UCX'] if LooseVersion(self.version) >= '4.1.4': known_dependencies.append('UCC') + if LooseVersion(self.version) >= '5.0.0': + known_dependencies.append('PRRTE') # Value to use for `--with-=` if the dependency is not specified in the easyconfig # No entry is interpreted as no option added at all @@ -85,6 +87,8 @@ def config_opt_used(key, enable_opt=False): # For these the default is to use an internal copy and not using any is not supported for dep in ('hwloc', 'libevent', 'PMIx'): unused_dep_value[dep] = 'internal' + if LooseVersion(self.version) >= '5.0.0': + unused_dep_value['PRRTE'] = 'internal' # handle dependencies for dep in known_dependencies: @@ -165,7 +169,8 @@ def sanity_check_step(self): bin_names = ['mpicc', 'mpicxx', 'mpif90', 'mpifort', 'mpirun', 'ompi_info', 'opal_wrapper'] if LooseVersion(self.version) >= LooseVersion('5.0.0'): - bin_names.append('prterun') + if not get_software_root('PRRTE'): + bin_names.append('prterun') else: bin_names.append('orterun') bin_files = [os.path.join('bin', x) for x in bin_names] @@ -173,13 +178,14 @@ def sanity_check_step(self): shlib_ext = get_shared_lib_ext() lib_names = ['mpi_mpifh', 'mpi', 'open-pal'] if LooseVersion(self.version) >= LooseVersion('5.0.0'): - lib_names.append('prrte') + if not get_software_root('PRRTE'): + lib_names.append('prrte') else: lib_names.extend(['ompitrace', 'open-rte']) lib_files = [os.path.join('lib', 'lib%s.%s' % (x, shlib_ext)) for x in lib_names] inc_names = ['mpi-ext', 'mpif-config', 'mpif', 'mpi', 'mpi_portable_platform'] - if LooseVersion(self.version) >= LooseVersion('5.0.0'): + if LooseVersion(self.version) >= LooseVersion('5.0.0') and not get_software_root('PRRTE'): inc_names.append('prte') inc_files = [os.path.join('include', x + '.h') for x in inc_names] diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 91fa313b74..22a50df87e 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -72,35 +72,39 @@ def __init__(self, *args, **kwargs): """Locate the installation files of OpenSSL in the host system""" super(EB_OpenSSL_wrapper, self).__init__(*args, **kwargs) - # Wrapper should have at least a major minor version numbers for OpenSSL before version 3+ - if LooseVersion(self.version) >= LooseVersion('3') and self.version.count('.') == 0: - self.majmin_version = self.version - else: + if LooseVersion(self.version) < LooseVersion('2'): try: subversions = self.version.split('.') - self.majmin_version = '%s.%s' % (subversions[0], subversions[1]) + self.generation = '%s.%s' % (subversions[0], subversions[1]) except (AttributeError, IndexError): - err_msg = "Wrapper OpenSSL version does not have any subversion: %s" + err_msg = "Wrapper for OpenSSL v1 version is missing a minor version: %s" raise EasyBuildError(err_msg, self.version) + elif LooseVersion(self.version) < LooseVersion('4'): + self.generation = "3" # Set minimum OpenSSL version - min_openssl_version = self.cfg.get('minimum_openssl_version') - - if not min_openssl_version: - min_openssl_version = self.version - elif not isinstance(min_openssl_version, str): - min_openssl_version = str(min_openssl_version) + self.min_version = self.cfg.get('minimum_openssl_version') + if not self.min_version: + self.min_version = self.version + elif not isinstance(self.min_version, str): + self.min_version = str(self.min_version) # Minimum OpenSSL version can only increase depth of wrapper version - if min_openssl_version.startswith(self.version): - self.log.debug("Requiring minimum OpenSSL version: %s", min_openssl_version) + if self.min_version.startswith(self.version): + self.log.debug("Requiring minimum OpenSSL version: %s", self.min_version) else: err_msg = "Requested minimum OpenSSL version '%s' does not fit in wrapper easyconfig version '%s'" - raise EasyBuildError(err_msg, min_openssl_version, self.version) + raise EasyBuildError(err_msg, self.min_version, self.version) + + # Set maximum OpenSSL version (increase smallest revision by 1) + max_version_parts = [int(subv) for subv in self.version.split('.')] + max_version_parts[-1] += 1 + self.max_version = '.'.join([str(subv) for subv in max_version_parts]) + self.log.debug("Restricting maximum OpenSSL version: %s", self.max_version) # Regex pattern to find version strings in OpenSSL libraries and headers full_version_regex = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+[a-z]?') - openssl_version_regex = re.compile(r'OpenSSL\s+([0-9]+\.[0-9]+(\.[0-9]+[a-z]?)*)', re.M) + openssl_version_regex = re.compile(r'OpenSSL[\s_]+([0-9]+\.[0-9]+(\.[0-9]+[a-z]?)*)', re.M | re.I) # Libraries packaged in OpenSSL openssl_libs = ['libssl', 'libcrypto'] @@ -116,10 +120,6 @@ def __init__(self, *args, **kwargs): LINUX: ('so.1.1', ), DARWIN: ('1.1.dylib', ), }, - '3.0': { - LINUX: ('so.3', ), - DARWIN: ('3.dylib', ), - }, '3': { LINUX: ('so.3', ), DARWIN: ('3.dylib', ), @@ -127,32 +127,33 @@ def __init__(self, *args, **kwargs): } os_type = get_os_type() - if self.majmin_version in openssl_libext and os_type in openssl_libext[self.majmin_version]: + if self.generation in openssl_libext and os_type in openssl_libext[self.generation]: # generate matrix of versioned .so filenames system_versioned_libs = [ ['%s.%s' % (lib, ext) for lib in openssl_libs] - for ext in openssl_libext[self.majmin_version][os_type] + for ext in openssl_libext[self.generation][os_type] ] self.log.info("Matrix of version library names: %s", system_versioned_libs) else: - err_msg = "Don't know name of OpenSSL system library for version %s and OS type %s" - raise EasyBuildError(err_msg, self.majmin_version, os_type) - - # by default target the first option of each OpenSSL library, - # which corresponds to installation from source - self.target_ssl_libs = system_versioned_libs[0] - self.log.info("Target OpenSSL libraries: %s", self.target_ssl_libs) + err_msg = "OpenSSL system library for version %s and OS type %s is unsupported" + raise EasyBuildError(err_msg, self.generation, os_type) # folders containing engines libraries openssl_engines = { '1.0': 'engines', '1.1': 'engines-1.1', - '3.0': 'engines-3', '3': 'engines-3', } - self.target_ssl_engine = openssl_engines[self.majmin_version] - # Paths to system libraries and headers of OpenSSL + # Define targets for this generation + # They are used to sanity check both installations from source or wrappers + self.generation_targets = { + 'bin': 'openssl', + 'engines': openssl_engines[self.generation], + 'libs': system_versioned_libs[0], # first set of libs match source installs + } + + # Paths to system components of OpenSSL self.system_ssl = { 'bin': None, 'engines': None, @@ -165,55 +166,86 @@ def __init__(self, *args, **kwargs): self.log.info("Not wrapping system OpenSSL installation by user request") return + # Check system OpenSSL binary + target_ssl_bins = [self.generation_targets['bin']] + if self.generation == '1.1': + target_ssl_bins.insert(0, 'openssl11') # prefer 'openssl11' over 'openssl' with v1.1 + elif self.generation == '3': + target_ssl_bins.insert(0, 'openssl3') # prefer 'openssl3' over 'openssl' with v3 + + for ssl_bin in target_ssl_bins: + ssl_bin_path, ssl_bin_version = self.get_openssl_bin_version(ssl_bin) + if ssl_bin_path: + if LooseVersion(self.min_version) <= LooseVersion(ssl_bin_version) < LooseVersion(self.max_version): + self.system_ssl['version'] = ssl_bin_version + self.system_ssl['bin'] = ssl_bin_path + break + + if self.system_ssl['bin'] and self.system_ssl['version']: + log_msg = "System OpenSSL binary for version %s found: %s" + self.log.info(log_msg, self.system_ssl['version'], self.system_ssl['bin']) + else: + log_msg = "OpenSSL binary for version %s not found in host system! " + log_msg += "Falling back to building OpenSSL from source" + self.log.info(log_msg, self.version) + return + # Check the system libraries of OpenSSL # Find library file and compare its version string - for idx, solibs in enumerate(system_versioned_libs): + for solibs in system_versioned_libs: + target_ssl_libs = [] for solib in solibs: system_solib = find_library_path(solib) - if system_solib: - openssl_version = '0' + if not system_solib: + # this OpenSSL library is missing, move on to next group of versioned libs + break + + try: # get version of system library filename + ssl_lib_version = full_version_regex.search(os.path.realpath(system_solib)).group(0) + except AttributeError: + # filename lacks the full version, fallback to version strings within the library + solib_strings = read_file(system_solib, mode="rb").decode('utf-8', 'replace') try: - openssl_version = full_version_regex.search(os.path.realpath(system_solib)).group(0) - except AttributeError: - # filename lacks the full version, fallback to version strings within the library - solib_strings = read_file(system_solib, mode="rb").decode('utf-8', 'replace') - try: - openssl_version = openssl_version_regex.search(solib_strings).group(1) - except AttributeError: - dbg_msg = "Could not detect the full version of system OpenSSL library: %s" - self.log.debug(dbg_msg, system_solib) - # check that system version fulfills requirements - if LooseVersion(openssl_version) >= LooseVersion(min_openssl_version): - dbg_msg = "System OpenSSL library '%s' with version %s fulfills requested version %s" - self.log.debug(dbg_msg, system_solib, openssl_version, min_openssl_version) - self.system_ssl['libs'].append(system_solib) - else: - dbg_msg = "System OpenSSL library '%s' with version %s is older than requested version %s" - self.log.debug(dbg_msg, system_solib, openssl_version, min_openssl_version) + ssl_lib_support_versions = openssl_version_regex.findall(solib_strings) + ssl_lib_support_versions.sort() + dbg_msg = "System OpenSSL library '%s' supports versions: %s" + dbg_msg_support_versions = ', '.join([''.join(v) for v in ssl_lib_support_versions]) + self.log.debug(dbg_msg, system_solib, dbg_msg_support_versions) + # pick highest supported version + ssl_lib_version = ssl_lib_support_versions[-1][0] + except IndexError: + dbg_msg = "Could not detect the full version of system OpenSSL library: %s" + self.log.debug(dbg_msg, system_solib) + # check that system library version fulfills requirements + if LooseVersion(self.min_version) <= LooseVersion(ssl_lib_version) < LooseVersion(self.max_version): + dbg_msg = "System OpenSSL library '%s' with version %s fulfills requested version %s" + self.log.debug(dbg_msg, system_solib, ssl_lib_version, self.min_version) + target_ssl_libs.append(system_solib) else: - # one of the OpenSSL libraries is missing, switch to next group of versioned libs - self.system_ssl['libs'] = [] - break + dbg_msg = "System OpenSSL library '%s' with version %s is older than requested version %s" + self.log.debug(dbg_msg, system_solib, ssl_lib_version, self.min_version) - if len(self.system_ssl['libs']) == len(openssl_libs): - # keep these libraries as possible targets for this installation - target_system_ssl_libs = system_versioned_libs[idx] + if len(target_ssl_libs) == len(openssl_libs): + # target libraries found, ignore further options break - if len(self.system_ssl['libs']) == len(openssl_libs): - self.system_ssl['version'] = openssl_version + if len(target_ssl_libs) == len(openssl_libs): + self.system_ssl['libs'] = target_ssl_libs + self.generation_targets['libs'] = [os.path.basename(solib) for solib in target_ssl_libs] info_msg = "Found OpenSSL library version %s in host system: %s" - self.log.info(info_msg, self.system_ssl['version'], os.path.dirname(self.system_ssl['libs'][0])) + self.log.info(info_msg, ssl_lib_version, os.path.dirname(self.system_ssl['libs'][0])) else: - self.log.info("OpenSSL library not found in host system, falling back to OpenSSL in EasyBuild") + log_msg = "OpenSSL library for version %s not found in host system! " + log_msg += "Falling back to building OpenSSL from source" + self.log.info(log_msg, self.system_ssl['version']) return # Directory with engine libraries lib_dir = os.path.dirname(self.system_ssl['libs'][0]) lib_engines_dir = [ - os.path.join(lib_dir, 'openssl', self.target_ssl_engine), - os.path.join(lib_dir, self.target_ssl_engine), + os.path.join(lib_dir, 'openssl', openssl_engines[self.generation]), + os.path.join(lib_dir, openssl_engines[self.generation]), ] for engines_path in lib_engines_dir: @@ -223,7 +255,9 @@ def __init__(self, *args, **kwargs): break if not self.system_ssl['engines']: - self.log.info("OpenSSL engines not found in host system, falling back to OpenSSL in EasyBuild") + log_msg = "OpenSSL engines for version %s not found in host system! " + log_msg += "Falling back to building OpenSSL from source" + self.log.info(log_msg, self.system_ssl['version']) return # Check system include paths for OpenSSL headers @@ -237,10 +271,14 @@ def __init__(self, *args, **kwargs): # headers are located in 'include/openssl' by default ssl_include_subdirs = ['openssl'] - if self.majmin_version == '1.1': + if self.generation == '1.1': # but version 1.1 can be installed in 'include/openssl11/openssl' as well, for example in CentOS 7 # prefer 'include/openssl' as long as the version of headers matches ssl_include_subdirs.append(os.path.join('openssl11', self.name.lower())) + elif self.generation == '3': + # but version 3.x can be installed in 'include/openssl3/openssl' as well, for example in RHEL 8 derivatives + # prefer 'include/openssl' as long as the version of headers matches + ssl_include_subdirs.append(os.path.join('openssl3', self.name.lower())) ssl_include_dirs = [os.path.join(incd, subd) for incd in sys_include_dirs for subd in ssl_include_subdirs] ssl_include_dirs = [include for include in ssl_include_dirs if os.path.isdir(include)] @@ -253,19 +291,19 @@ def __init__(self, *args, **kwargs): # check version reported by opensslv.h opensslv = read_file(opensslv_path) try: - header_version = openssl_version_regex.search(opensslv).group(1) + ssl_head_version = openssl_version_regex.search(opensslv).group(1) except AttributeError: err_msg = "System OpenSSL header '%s' does not contain any recognizable version string" raise EasyBuildError(err_msg, opensslv_path) - if header_version == self.system_ssl['version']: + if LooseVersion(self.min_version) <= LooseVersion(ssl_head_version) < LooseVersion(self.max_version): self.system_ssl['include'] = include_dir info_msg = "Found OpenSSL headers v%s in host system: %s" - self.log.info(info_msg, header_version, self.system_ssl['include']) + self.log.info(info_msg, ssl_head_version, self.system_ssl['include']) break else: - dbg_msg = "System OpenSSL header version '%s' doesn not match library version '%s'" - self.log.debug(dbg_msg, header_version, self.system_ssl['version']) + dbg_msg = "System OpenSSL header version '%s' does not fulfill minimum version requirement '%s'" + self.log.debug(dbg_msg, ssl_head_version, self.min_version) else: self.log.info("System OpenSSL header file %s not found", opensslv_path) @@ -275,24 +313,6 @@ def __init__(self, *args, **kwargs): "source in EasyBuild by setting 'wrap_system_openssl = False' in the OpenSSL easyconfig.") raise EasyBuildError(err_msg, self.version, self.system_ssl['version']) - # Check system OpenSSL binary - if self.majmin_version == '1.1': - # prefer 'openssl11' over 'openssl' with v1.1 - self.system_ssl['bin'] = which('openssl11') - - if not self.system_ssl['bin']: - self.system_ssl['bin'] = which('openssl') - - if self.system_ssl['bin']: - self.log.info("System OpenSSL binary found: %s", self.system_ssl['bin']) - else: - self.log.info("System OpenSSL binary not found!") - return - - # system OpenSSL is fine, change target libraries to the ones found in it - self.target_ssl_libs = target_system_ssl_libs - self.log.info("Target system OpenSSL libraries: %s", self.target_ssl_libs) - def fetch_step(self, *args, **kwargs): """Fetch sources if OpenSSL component is needed""" if not all(self.system_ssl[key] for key in ('bin', 'engines', 'include', 'libs')): @@ -360,15 +380,15 @@ def sanity_check_step(self): """Custom sanity check for OpenSSL wrapper.""" shlib_ext = get_shared_lib_ext() - ssl_libs = ['%s.%s' % (libso.split('.')[0], shlib_ext) for libso in self.target_ssl_libs] - ssl_libs.extend(self.target_ssl_libs) + ssl_libs = ['%s.%s' % (solib.split('.')[0], shlib_ext) for solib in self.generation_targets['libs']] + ssl_libs.extend(self.generation_targets['libs']) - ssl_files = [os.path.join('bin', self.name.lower())] + ssl_files = [os.path.join('bin', self.generation_targets['bin'])] ssl_files.extend(os.path.join('lib', libso) for libso in ssl_libs) ssl_dirs = [ os.path.join('include', self.name.lower()), - os.path.join('lib', self.target_ssl_engine), + os.path.join('lib', self.generation_targets["engines"]), os.path.join('lib', 'pkgconfig'), ] @@ -386,20 +406,34 @@ def sanity_check_step(self): if proxy_parsed.netloc: proxy_arg = ' -proxy %s' % proxy_parsed.netloc - if LooseVersion(self.version) >= LooseVersion('3') and self.version.count('.') == 0: - ssl_ver_comp_chars = 1 - else: - ssl_ver_comp_chars = 3 - custom_commands = [ # make sure that version mentioned in output of 'openssl version' matches version we are using - "ssl_ver=$(openssl version); [ ${ssl_ver:8:%s} == '%s' ]" % (ssl_ver_comp_chars, self.majmin_version), + '[[ "$(openssl version)" =~ ^OpenSSL.%s ]]' % self.generation, ("echo | openssl s_client%s -connect github.com:443 -verify 9 " "| grep 'Verify return code: 0 (ok)'" % proxy_arg), ] super(Bundle, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) + def get_openssl_bin_version(self, bin_name): + """Check OpenSSL executable version""" + bin_path = which(bin_name) + if not bin_path: + self.log.debug("OpenSSL executable '%s' not found", bin_name) + return None, None + + cmd = "%s version" % bin_path + res = run_shell_cmd(cmd, fail_on_error=False, hidden=True) + + try: + bin_version = res.output.split(' ')[1] + except (AttributeError, IndexError): + raise EasyBuildError("Failed to check version of OpenSSL executable: %s", bin_path) + else: + self.log.debug("Version of OpenSSL executable '%s': %s", bin_path, bin_version) + + return bin_path, bin_version + def install_pc_files(self): """Install pkg-config files for the wrapper""" @@ -420,7 +454,7 @@ def install_pc_files(self): 'libcrypto': { 'name': 'OpenSSL-libcrypto', 'description': 'OpenSSL cryptography library', - 'enginesdir': self.target_ssl_engine, + 'enginesdir': self.generation_targets["engines"], }, 'libssl': { 'name': 'OpenSSL-libssl', @@ -445,11 +479,11 @@ def install_pc_files(self): # component name in system pkg-config pc_name = pc_comp - if self.majmin_version == '1.1': + if self.generation == '1.1': # check suffixed names with v1.1 pc_name_suffix = pc_name + '11' pc_exists_cmd = "pkg-config --exists %s" % pc_name_suffix - res = run_shell_cmd(pc_exists_cmd, fail_on_error=False) + res = run_shell_cmd(pc_exists_cmd, fail_on_error=False, hidden=True) if res.exit_code == 0: self.log.info("%s exists", pc_name_suffix) pc_name = pc_name_suffix @@ -459,7 +493,7 @@ def install_pc_files(self): for require_type in ['Requires', 'Requires.private']: require_print = require_type.lower().replace('.', '-') pc_print_cmd = "pkg-config --print-%s %s" % (require_print, pc_name) - res = run_shell_cmd(pc_print_cmd, fail_on_error=False) + res = run_shell_cmd(pc_print_cmd, fail_on_error=False, hidden=True) self.log.info("Output of '%s': %s", pc_print_cmd, res.output) if res.output: @@ -480,12 +514,12 @@ def install_pc_files(self): pc_file['cflags'] = "Cflags: -I${includedir}" # infer private libs through pkg-config pc_libs_cmd = "pkg-config --libs %s" % pc_name - res = run_shell_cmd(pc_libs_cmd, fail_on_error=False) + res = run_shell_cmd(pc_libs_cmd, fail_on_error=False, hidden=True) self.log.info("Output of '%s': %s", pc_libs_cmd, res.output) linker_libs = res.output pc_libs_static_cmd = "pkg-config --libs --static %s" % pc_name - res = run_shell_cmd(pc_libs_static_cmd, fail_on_error=False) + res = run_shell_cmd(pc_libs_static_cmd, fail_on_error=False, hidden=True) self.log.info("Output of '%s': %s", pc_libs_static_cmd, res.output) libs_priv = "%s " % res.output.rstrip()